From 69b03b06b7806dfeab7b433e55b07d1c6e8b4c3d Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Fri, 8 Jul 2016 15:47:46 -0700 Subject: [PATCH 1/2] - Add visibility macros. Hide boost symbols in arrow_io - Hack around Travis CI inability to use its boost static libraries - Use parquet_shared name - More informative verbose test logs - Fix some gtest-1.7.0 crankiness - Fix a valgrind shared_ptr possible memory leak stemming from static variable referenced at compile-time in libarrow_parquet - Fix a bunch of compiler warnings in release builds Change-Id: I4f519751becaf6fbdbc0e2fe79938dec13876b30 --- ci/travis_install_conda.sh | 1 - ci/travis_script_cpp.sh | 2 +- ci/travis_script_python.sh | 6 +- cpp/CMakeLists.txt | 218 +++++++++++++---------- cpp/build-support/run-test.sh | 10 +- cpp/conda.recipe/build.sh | 13 +- cpp/src/arrow/array.h | 5 +- cpp/src/arrow/builder.h | 3 +- cpp/src/arrow/column.h | 5 +- cpp/src/arrow/io/CMakeLists.txt | 53 ++++-- cpp/src/arrow/io/hdfs-io-test.cc | 2 +- cpp/src/arrow/io/hdfs.h | 17 +- cpp/src/arrow/io/libhdfs_shim.cc | 3 +- cpp/src/arrow/io/symbols.map | 18 ++ cpp/src/arrow/ipc/CMakeLists.txt | 2 +- cpp/src/arrow/parquet/CMakeLists.txt | 4 +- cpp/src/arrow/parquet/parquet-io-test.cc | 18 +- cpp/src/arrow/parquet/reader.cc | 2 +- cpp/src/arrow/parquet/reader.h | 6 +- cpp/src/arrow/parquet/schema.h | 10 +- cpp/src/arrow/parquet/writer.cc | 4 +- cpp/src/arrow/parquet/writer.h | 6 +- cpp/src/arrow/schema.h | 4 +- cpp/src/arrow/symbols.map | 15 ++ cpp/src/arrow/table.h | 6 +- cpp/src/arrow/type.h | 39 ++-- cpp/src/arrow/types/construct.h | 11 +- cpp/src/arrow/types/decimal.h | 3 +- cpp/src/arrow/types/list.h | 7 +- cpp/src/arrow/types/primitive.h | 13 +- cpp/src/arrow/types/string-test.cc | 8 +- cpp/src/arrow/types/string.cc | 11 +- cpp/src/arrow/types/string.h | 16 +- cpp/src/arrow/types/struct-test.cc | 8 +- cpp/src/arrow/types/struct.h | 5 +- cpp/src/arrow/util/CMakeLists.txt | 1 + cpp/src/arrow/util/buffer.h | 12 +- cpp/src/arrow/util/memory-pool-test.cc | 2 +- cpp/src/arrow/util/memory-pool.h | 6 +- cpp/src/arrow/util/status.cc | 3 + cpp/src/arrow/util/status.h | 4 +- cpp/src/arrow/util/visibility.h | 32 ++++ python/conda.recipe/build.sh | 15 +- python/src/pyarrow/adapters/builtin.h | 2 + python/src/pyarrow/adapters/pandas.h | 5 + python/src/pyarrow/common.h | 6 +- python/src/pyarrow/config.h | 4 + python/src/pyarrow/helpers.h | 3 + python/src/pyarrow/status.h | 4 +- python/src/pyarrow/visibility.h | 32 ++++ 50 files changed, 440 insertions(+), 245 deletions(-) create mode 100644 cpp/src/arrow/io/symbols.map create mode 100644 cpp/src/arrow/symbols.map create mode 100644 cpp/src/arrow/util/visibility.h create mode 100644 python/src/pyarrow/visibility.h diff --git a/ci/travis_install_conda.sh b/ci/travis_install_conda.sh index be7f59a4733..3a8f57bf8f1 100644 --- a/ci/travis_install_conda.sh +++ b/ci/travis_install_conda.sh @@ -25,4 +25,3 @@ conda install --yes conda-build jinja2 anaconda-client # faster builds, please conda install -y nomkl - diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh index 9cf4f8e3521..a3585507f0a 100755 --- a/ci/travis_script_cpp.sh +++ b/ci/travis_script_cpp.sh @@ -16,6 +16,6 @@ make lint # make check-clang-tidy # fi -ctest -L unittest +ctest -VV -L unittest popd diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 6d35785356a..4a377428ae4 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -7,7 +7,6 @@ PYTHON_DIR=$TRAVIS_BUILD_DIR/python # Re-use conda installation from C++ export MINICONDA=$TRAVIS_BUILD_DIR/miniconda export PATH="$MINICONDA/bin:$PATH" -export LD_LIBRARY_PATH="$MINICONDA/lib:$LD_LIBRARY_PATH" export PARQUET_HOME=$MINICONDA # Share environment with C++ @@ -32,12 +31,15 @@ python_version_tests() { # Expensive dependencies install from Continuum package repo conda install -y pip numpy pandas cython + conda install -y parquet-cpp arrow-cpp -c apache/channel/dev + # Other stuff pip install pip install -r requirements.txt export ARROW_HOME=$ARROW_CPP_INSTALL - python setup.py build_ext --inplace + python setup.py build_ext \ + --inplace python -m pytest -vv -r sxX pyarrow } diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 18b47599b93..d1a9e3a44d1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -44,12 +44,22 @@ endif(CCACHE_FOUND) # Top level cmake dir if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") + option(ARROW_BUILD_STATIC + "Build the libarrow static libraries" + ON) + + option(ARROW_BUILD_SHARED + "Build the libarrow shared libraries" + ON) + option(ARROW_PARQUET "Build the Parquet adapter and link to libparquet" OFF) + option(ARROW_TEST_MEMCHECK - "Run the test suite using valgrind --tool=memcheck" - OFF) + "Run the test suite using valgrind --tool=memcheck" + OFF) + option(ARROW_BUILD_TESTS "Build the Arrow googletest unit tests" ON) @@ -66,6 +76,10 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") "Build the Arrow IO extensions for the Hadoop file system" OFF) + option(ARROW_BOOST_USE_SHARED + "Rely on boost shared libraries where relevant" + ON) + option(ARROW_SSE3 "Build Arrow with SSE3" ON) @@ -169,21 +183,10 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang") # http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html # http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedef") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CLANG_OPTIONS}") endif() -# Sanity check linking option. -if (NOT ARROW_LINK) - set(ARROW_LINK "d") -elseif(NOT ("auto" MATCHES "^${ARROW_LINK}" OR - "dynamic" MATCHES "^${ARROW_LINK}" OR - "static" MATCHES "^${ARROW_LINK}")) - message(FATAL_ERROR "Unknown value for ARROW_LINK, must be auto|dynamic|static") -else() - # Remove all but the first letter. - string(SUBSTRING "${ARROW_LINK}" 0 1 ARROW_LINK) -endif() - # ASAN / TSAN / UBSAN include(san-config) @@ -203,61 +206,11 @@ if ("${ARROW_GENERATE_COVERAGE}") # For coverage to work properly, we need to use static linkage. Otherwise, # __gcov_flush() doesn't properly flush coverage from every module. # See http://stackoverflow.com/questions/28164543/using-gcov-flush-within-a-library-doesnt-force-the-other-modules-to-yield-gc - if("${ARROW_LINK}" STREQUAL "a") - message("Using static linking for coverage build") - set(ARROW_LINK "s") - elseif("${ARROW_LINK}" STREQUAL "d") - message(SEND_ERROR "Cannot use coverage with dynamic linking") - endif() -endif() - -# If we still don't know what kind of linking to perform, choose based on -# build type (developers like fast builds). -if ("${ARROW_LINK}" STREQUAL "a") - if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR - "${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG") - message("Using dynamic linking for ${CMAKE_BUILD_TYPE} builds") - set(ARROW_LINK "d") - else() - message("Using static linking for ${CMAKE_BUILD_TYPE} builds") - set(ARROW_LINK "s") + if(NOT ARROW_BUILD_STATIC) + message(SEND_ERROR "Coverage requires the static lib to be built") endif() endif() -# Are we using the gold linker? It doesn't work with dynamic linking as -# weak symbols aren't properly overridden, causing tcmalloc to be omitted. -# Let's flag this as an error in RELEASE builds (we shouldn't release a -# product like this). -# -# See https://sourceware.org/bugzilla/show_bug.cgi?id=16979 for details. -# -# The gold linker is only for ELF binaries, which OSX doesn't use. We can -# just skip. -if (NOT APPLE) - execute_process(COMMAND ${CMAKE_CXX_COMPILER} -Wl,--version OUTPUT_VARIABLE LINKER_OUTPUT) -endif () -if (LINKER_OUTPUT MATCHES "gold") - if ("${ARROW_LINK}" STREQUAL "d" AND - "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE") - message(SEND_ERROR "Cannot use gold with dynamic linking in a RELEASE build " - "as it would cause tcmalloc symbols to get dropped") - else() - message("Using gold linker") - endif() - set(ARROW_USING_GOLD 1) -else() - message("Using ld linker") -endif() - -# Having set ARROW_LINK due to build type and/or sanitizer, it's now safe to -# act on its value. -if ("${ARROW_LINK}" STREQUAL "d") - set(BUILD_SHARED_LIBS ON) - - # Position independent code is only necessary when producing shared objects. - add_definitions(-fPIC) -endif() - # set compile output directory string (TOLOWER ${CMAKE_BUILD_TYPE} BUILD_SUBDIR_NAME) @@ -290,6 +243,15 @@ set(LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}") include_directories(src) +############################################################ +# Visibility +############################################################ +# For generate_export_header() and add_compiler_export_flags(). +include(GenerateExportHeader) + +# Sets -fvisibility=hidden for gcc +add_compiler_export_flags() + ############################################################ # Benchmarking ############################################################ @@ -360,7 +322,7 @@ endfunction() # # Arguments after the test name will be passed to set_tests_properties(). function(ADD_ARROW_TEST REL_TEST_NAME) - if(NO_TESTS) + if(NO_TESTS OR NOT ARROW_BUILD_STATIC) return() endif() get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) @@ -377,13 +339,13 @@ function(ADD_ARROW_TEST REL_TEST_NAME) endif() if (ARROW_TEST_MEMCHECK) - SET_PROPERTY(TARGET ${TEST_NAME} - APPEND_STRING PROPERTY - COMPILE_FLAGS " -DARROW_VALGRIND") - add_test(${TEST_NAME} - valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH}) + SET_PROPERTY(TARGET ${TEST_NAME} + APPEND_STRING PROPERTY + COMPILE_FLAGS " -DARROW_VALGRIND") + add_test(${TEST_NAME} + valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH}) else() - add_test(${TEST_NAME} + add_test(${TEST_NAME} ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH}) endif() set_tests_properties(${TEST_NAME} PROPERTIES LABELS "unittest") @@ -427,19 +389,34 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") endif() - if(("${ARROW_LINK}" STREQUAL "s" AND ARG_STATIC_LIB) OR (NOT ARG_SHARED_LIB)) + if(ARG_STATIC_LIB AND ARG_SHARED_LIB) if(NOT ARG_STATIC_LIB) message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") endif() + + SET(AUG_LIB_NAME "${LIB_NAME}_static") + add_library(${AUG_LIB_NAME} STATIC IMPORTED) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") + message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}") + + SET(AUG_LIB_NAME "${LIB_NAME}_shared") + add_library(${AUG_LIB_NAME} SHARED IMPORTED) + set_target_properties(${AUG_LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") + message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}") + elseif(ARG_STATIC_LIB) add_library(${LIB_NAME} STATIC IMPORTED) set_target_properties(${LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}") message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}") - else() + elseif(ARG_SHARED_LIB) add_library(${LIB_NAME} SHARED IMPORTED) set_target_properties(${LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}") + else() + message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") endif() if(ARG_DEPS) @@ -538,9 +515,17 @@ endif() ############################################################ # Linker setup ############################################################ -set(ARROW_MIN_TEST_LIBS arrow arrow_test_main ${ARROW_BASE_LIBS}) +set(ARROW_MIN_TEST_LIBS + arrow_static + arrow_test_main + ${ARROW_BASE_LIBS}) + set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS}) -set(ARROW_BENCHMARK_LINK_LIBS arrow arrow_benchmark_main ${ARROW_BASE_LIBS}) + +set(ARROW_BENCHMARK_LINK_LIBS + arrow_static + arrow_benchmark_main + ${ARROW_BASE_LIBS}) ############################################################ # "make ctags" target @@ -576,14 +561,14 @@ endif (UNIX) if (UNIX) file(GLOB_RECURSE LINT_FILES - "${CMAKE_CURRENT_SOURCE_DIR}/src/*.h" - "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc" - ) + "${CMAKE_CURRENT_SOURCE_DIR}/src/*.h" + "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc" + ) FOREACH(item ${LINT_FILES}) - IF(NOT (item MATCHES "_generated.h")) + IF(NOT (item MATCHES "_generated.h")) LIST(APPEND FILTERED_LINT_FILES ${item}) - ENDIF() + ENDIF() ENDFOREACH(item ${LINT_FILES}) # Full lint @@ -628,7 +613,10 @@ endif() # Subdirectories ############################################################ -set(LIBARROW_LINK_LIBS +set(ARROW_LINK_LIBS +) + +set(ARROW_PRIVATE_LINK_LIBS ) set(ARROW_SRCS @@ -660,35 +648,67 @@ set(ARROW_SRCS src/arrow/util/status.cc ) -set(LIBARROW_LINKAGE "SHARED") - -add_library(arrow - ${LIBARROW_LINKAGE} +add_library(arrow_objlib OBJECT ${ARROW_SRCS} ) +# Necessary to make static linking into other shared libraries work properly +set_property(TARGET arrow_objlib PROPERTY POSITION_INDEPENDENT_CODE 1) + +if(NOT APPLE) + # Localize thirdparty symbols using a linker version script. This hides them + # from the client application. The OS X linker does not support the + # version-script option. + set(SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/arrow/symbols.map") +endif() + +if (ARROW_BUILD_SHARED) + add_library(arrow_shared SHARED $) + if(APPLE) + set_target_properties(arrow_shared PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + endif() + set_target_properties(arrow_shared + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" + LINK_FLAGS "${SHARED_LINK_FLAGS}" + OUTPUT_NAME "arrow") + target_link_libraries(arrow_shared + LINK_PUBLIC ${ARROW_LINK_LIBS} + LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS}) + + install(TARGETS arrow_shared + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib) +endif() + +if (ARROW_BUILD_STATIC) + add_library(arrow_static STATIC $) + set_target_properties(arrow_static + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" + OUTPUT_NAME "arrow") + + target_link_libraries(arrow_static + LINK_PUBLIC ${ARROW_LINK_LIBS} + LINK_PRIVATE ${ARROW_PRIVATE_LINK_LIBS}) + + install(TARGETS arrow_static + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib) +endif() + if (APPLE) - set_target_properties(arrow + set_target_properties(arrow_shared PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_NAME_DIR "@rpath") endif() -set_target_properties(arrow - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" -) -target_link_libraries(arrow ${LIBARROW_LINK_LIBS}) - add_subdirectory(src/arrow) add_subdirectory(src/arrow/io) add_subdirectory(src/arrow/util) add_subdirectory(src/arrow/types) -install(TARGETS arrow - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib) - #---------------------------------------------------------------------- # Parquet adapter library @@ -715,7 +735,7 @@ if(ARROW_IPC) include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR}) add_library(flatbuffers STATIC IMPORTED) set_target_properties(flatbuffers PROPERTIES - IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB}) + IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB}) add_subdirectory(src/arrow/ipc) endif() diff --git a/cpp/build-support/run-test.sh b/cpp/build-support/run-test.sh index 0e628e26ecd..f563da53679 100755 --- a/cpp/build-support/run-test.sh +++ b/cpp/build-support/run-test.sh @@ -79,16 +79,16 @@ function setup_sanitizers() { TSAN_OPTIONS="$TSAN_OPTIONS suppressions=$ROOT/build-support/tsan-suppressions.txt" TSAN_OPTIONS="$TSAN_OPTIONS history_size=7" export TSAN_OPTIONS - + # Enable leak detection even under LLVM 3.4, where it was disabled by default. # This flag only takes effect when running an ASAN build. ASAN_OPTIONS="$ASAN_OPTIONS detect_leaks=1" export ASAN_OPTIONS - + # Set up suppressions for LeakSanitizer LSAN_OPTIONS="$LSAN_OPTIONS suppressions=$ROOT/build-support/lsan-suppressions.txt" export LSAN_OPTIONS - + # Suppressions require symbolization. We'll default to using the symbolizer in # thirdparty. if [ -z "$ASAN_SYMBOLIZER_PATH" ]; then @@ -107,7 +107,7 @@ function run_test() { | $ROOT/build-support/asan_symbolize.py \ | c++filt \ | $ROOT/build-support/stacktrace_addr2line.pl $TEST_EXECUTABLE \ - | $pipe_cmd > $LOGFILE + | $pipe_cmd 2>&1 | tee $LOGFILE STATUS=$? # TSAN doesn't always exit with a non-zero exit code due to a bug: @@ -198,7 +198,7 @@ for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do fi done -if [ $RUN_TYPE = "test" ]; then +if [ $RUN_TYPE = "test" ]; then post_process_tests fi diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh index 7e60ccc911f..2f2b7482667 100644 --- a/cpp/conda.recipe/build.sh +++ b/cpp/conda.recipe/build.sh @@ -39,16 +39,17 @@ pwd source thirdparty/versions.sh export GTEST_HOME=`pwd`/thirdparty/$GTEST_BASEDIR -if [ `uname` == Linux ]; then - SHARED_LINKER_FLAGS='-static-libstdc++' -elif [ `uname` == Darwin ]; then - SHARED_LINKER_FLAGS='' -fi +# if [ `uname` == Linux ]; then +# SHARED_LINKER_FLAGS='-static-libstdc++' +# elif [ `uname` == Darwin ]; then +# SHARED_LINKER_FLAGS='' +# fi + +# -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \ cmake \ -DCMAKE_BUILD_TYPE=release \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_SHARED_LINKER_FLAGS=$SHARED_LINKER_FLAGS \ -DARROW_HDFS=on \ -DARROW_IPC=on \ -DARROW_PARQUET=on \ diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h index 76dc0f59814..c7ffb23ca18 100644 --- a/cpp/src/arrow/array.h +++ b/cpp/src/arrow/array.h @@ -24,6 +24,7 @@ #include "arrow/type.h" #include "arrow/util/bit-util.h" #include "arrow/util/macros.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -35,7 +36,7 @@ class Status; // // The base class is only required to have a null bitmap buffer if the null // count is greater than 0 -class Array { +class ARROW_EXPORT Array { public: Array(const std::shared_ptr& type, int32_t length, int32_t null_count = 0, const std::shared_ptr& null_bitmap = nullptr); @@ -83,7 +84,7 @@ class Array { }; // Degenerate null type Array -class NullArray : public Array { +class ARROW_EXPORT NullArray : public Array { public: NullArray(const std::shared_ptr& type, int32_t length) : Array(type, length, length, nullptr) {} diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h index 7d3f4398d73..5d9fb992ff0 100644 --- a/cpp/src/arrow/builder.h +++ b/cpp/src/arrow/builder.h @@ -25,6 +25,7 @@ #include "arrow/type.h" #include "arrow/util/macros.h" #include "arrow/util/status.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -38,7 +39,7 @@ static constexpr int32_t MIN_BUILDER_CAPACITY = 1 << 5; // This class provides a facilities for incrementally building the null bitmap // (see Append methods) and as a side effect the current number of slots and // the null count. -class ArrayBuilder { +class ARROW_EXPORT ArrayBuilder { public: explicit ArrayBuilder(MemoryPool* pool, const TypePtr& type) : pool_(pool), diff --git a/cpp/src/arrow/column.h b/cpp/src/arrow/column.h index e409566e1f1..d5168cb032b 100644 --- a/cpp/src/arrow/column.h +++ b/cpp/src/arrow/column.h @@ -24,6 +24,7 @@ #include #include "arrow/type.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -34,7 +35,7 @@ typedef std::vector> ArrayVector; // A data structure managing a list of primitive Arrow arrays logically as one // large array -class ChunkedArray { +class ARROW_EXPORT ChunkedArray { public: explicit ChunkedArray(const ArrayVector& chunks); @@ -56,7 +57,7 @@ class ChunkedArray { // An immutable column data structure consisting of a field (type metadata) and // a logical chunked data array (which can be validated as all being the same // type). -class Column { +class ARROW_EXPORT Column { public: Column(const std::shared_ptr& field, const ArrayVector& chunks); Column(const std::shared_ptr& field, const std::shared_ptr& data); diff --git a/cpp/src/arrow/io/CMakeLists.txt b/cpp/src/arrow/io/CMakeLists.txt index 33b654f8190..b8c0e138afb 100644 --- a/cpp/src/arrow/io/CMakeLists.txt +++ b/cpp/src/arrow/io/CMakeLists.txt @@ -19,13 +19,18 @@ # arrow_io : Arrow IO interfaces set(ARROW_IO_LINK_LIBS - arrow + arrow_shared ) -set(ARROW_IO_PRIVATE_LINK_LIBS - boost_system - boost_filesystem -) +if (ARROW_BOOST_USE_SHARED) + set(ARROW_IO_PRIVATE_LINK_LIBS + boost_system_shared + boost_filesystem_shared) +else() + set(ARROW_IO_PRIVATE_LINK_LIBS + boost_system_static + boost_filesystem_static) +endif() set(ARROW_IO_TEST_LINK_LIBS arrow_io @@ -36,18 +41,18 @@ set(ARROW_IO_SRCS if(ARROW_HDFS) if(NOT THIRDPARTY_DIR) - message(FATAL_ERROR "THIRDPARTY_DIR not set") + message(FATAL_ERROR "THIRDPARTY_DIR not set") endif() if (DEFINED ENV{HADOOP_HOME}) - set(HADOOP_HOME $ENV{HADOOP_HOME}) + set(HADOOP_HOME $ENV{HADOOP_HOME}) else() - set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop") + set(HADOOP_HOME "${THIRDPARTY_DIR}/hadoop") endif() set(HDFS_H_PATH "${HADOOP_HOME}/include/hdfs.h") if (NOT EXISTS ${HDFS_H_PATH}) - message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}") + message(FATAL_ERROR "Did not find hdfs.h at ${HDFS_H_PATH}") endif() message(STATUS "Found hdfs.h at: " ${HDFS_H_PATH}) message(STATUS "Building libhdfs shim component") @@ -55,29 +60,39 @@ if(ARROW_HDFS) include_directories(SYSTEM "${HADOOP_HOME}/include") set(ARROW_HDFS_SRCS - hdfs.cc - libhdfs_shim.cc) + hdfs.cc + libhdfs_shim.cc) set_property(SOURCE ${ARROW_HDFS_SRCS} - APPEND_STRING PROPERTY - COMPILE_FLAGS "-DHAS_HADOOP") + APPEND_STRING PROPERTY + COMPILE_FLAGS "-DHAS_HADOOP") set(ARROW_IO_SRCS - ${ARROW_HDFS_SRCS} - ${ARROW_IO_SRCS}) + ${ARROW_HDFS_SRCS} + ${ARROW_IO_SRCS}) ADD_ARROW_TEST(hdfs-io-test) ARROW_TEST_LINK_LIBRARIES(hdfs-io-test - ${ARROW_IO_TEST_LINK_LIBS}) + ${ARROW_IO_TEST_LINK_LIBS}) endif() add_library(arrow_io SHARED ${ARROW_IO_SRCS} ) -target_link_libraries(arrow_io LINK_PUBLIC ${ARROW_IO_LINK_LIBS}) -target_link_libraries(arrow_io LINK_PRIVATE ${ARROW_IO_PRIVATE_LINK_LIBS}) +target_link_libraries(arrow_io + LINK_PUBLIC ${ARROW_IO_LINK_LIBS} + LINK_PRIVATE ${ARROW_IO_PRIVATE_LINK_LIBS}) + +if(NOT APPLE) + # Localize thirdparty symbols using a linker version script. This hides them + # from the client application. The OS X linker does not support the + # version-script option. + set(ARROW_IO_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map") +endif() -SET_TARGET_PROPERTIES(arrow_io PROPERTIES LINKER_LANGUAGE CXX) +SET_TARGET_PROPERTIES(arrow_io PROPERTIES + LINKER_LANGUAGE CXX + LINK_FLAGS "${ARROW_IO_LINK_FLAGS}") if (APPLE) set_target_properties(arrow_io diff --git a/cpp/src/arrow/io/hdfs-io-test.cc b/cpp/src/arrow/io/hdfs-io-test.cc index 11d67aeba20..d1bf140ae68 100644 --- a/cpp/src/arrow/io/hdfs-io-test.cc +++ b/cpp/src/arrow/io/hdfs-io-test.cc @@ -227,7 +227,7 @@ TEST_F(TestHdfsClient, ListDirectory) { // Do it again, appends! ASSERT_OK(client_->ListDirectory(scratch_dir_, &listing)); - ASSERT_EQ(6, listing.size()); + ASSERT_EQ(6, static_cast(listing.size())); // Argh, well, shouldn't expect the listing to be in any particular order for (size_t i = 0; i < listing.size(); ++i) { diff --git a/cpp/src/arrow/io/hdfs.h b/cpp/src/arrow/io/hdfs.h index a1972db9615..532e3c536a1 100644 --- a/cpp/src/arrow/io/hdfs.h +++ b/cpp/src/arrow/io/hdfs.h @@ -25,6 +25,7 @@ #include "arrow/io/interfaces.h" #include "arrow/util/macros.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -32,8 +33,6 @@ class Status; namespace io { -Status ConnectLibHdfs(); - class HdfsClient; class HdfsReadableFile; class HdfsWriteableFile; @@ -64,7 +63,7 @@ struct HdfsConnectionConfig { // TODO: Kerberos, etc. }; -class HdfsClient : public FileSystemClient { +class ARROW_EXPORT HdfsClient : public FileSystemClient { public: ~HdfsClient(); @@ -149,14 +148,14 @@ class HdfsClient : public FileSystemClient { friend class HdfsReadableFile; friend class HdfsWriteableFile; - class HdfsClientImpl; + class ARROW_NO_EXPORT HdfsClientImpl; std::unique_ptr impl_; HdfsClient(); DISALLOW_COPY_AND_ASSIGN(HdfsClient); }; -class HdfsReadableFile : public RandomAccessFile { +class ARROW_EXPORT HdfsReadableFile : public RandomAccessFile { public: ~HdfsReadableFile(); @@ -175,7 +174,7 @@ class HdfsReadableFile : public RandomAccessFile { Status Read(int32_t nbytes, int32_t* bytes_read, uint8_t* buffer) override; private: - class HdfsReadableFileImpl; + class ARROW_NO_EXPORT HdfsReadableFileImpl; std::unique_ptr impl_; friend class HdfsClient::HdfsClientImpl; @@ -184,7 +183,7 @@ class HdfsReadableFile : public RandomAccessFile { DISALLOW_COPY_AND_ASSIGN(HdfsReadableFile); }; -class HdfsWriteableFile : public WriteableFile { +class ARROW_EXPORT HdfsWriteableFile : public WriteableFile { public: ~HdfsWriteableFile(); @@ -197,7 +196,7 @@ class HdfsWriteableFile : public WriteableFile { Status Tell(int64_t* position) override; private: - class HdfsWriteableFileImpl; + class ARROW_NO_EXPORT HdfsWriteableFileImpl; std::unique_ptr impl_; friend class HdfsClient::HdfsClientImpl; @@ -207,6 +206,8 @@ class HdfsWriteableFile : public WriteableFile { DISALLOW_COPY_AND_ASSIGN(HdfsWriteableFile); }; +Status ARROW_EXPORT ConnectLibHdfs(); + } // namespace io } // namespace arrow diff --git a/cpp/src/arrow/io/libhdfs_shim.cc b/cpp/src/arrow/io/libhdfs_shim.cc index f75266536e5..003570d4fde 100644 --- a/cpp/src/arrow/io/libhdfs_shim.cc +++ b/cpp/src/arrow/io/libhdfs_shim.cc @@ -55,6 +55,7 @@ extern "C" { #include // NOLINT #include "arrow/util/status.h" +#include "arrow/util/visibility.h" namespace fs = boost::filesystem; @@ -496,7 +497,7 @@ static arrow::Status try_dlopen( namespace arrow { namespace io { -Status ConnectLibHdfs() { +Status ARROW_EXPORT ConnectLibHdfs() { static std::mutex lock; std::lock_guard guard(lock); diff --git a/cpp/src/arrow/io/symbols.map b/cpp/src/arrow/io/symbols.map new file mode 100644 index 00000000000..b4ad98cd7f2 --- /dev/null +++ b/cpp/src/arrow/io/symbols.map @@ -0,0 +1,18 @@ +{ + # Symbols marked as 'local' are not exported by the DSO and thus may not + # be used by client applications. + local: + # devtoolset / static-libstdc++ symbols + __cxa_*; + + extern "C++" { + # boost + boost::*; + + # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically + # links c++11 symbols into binaries so that the result may be executed on + # a system with an older libstdc++ which doesn't include the necessary + # c++11 symbols. + std::*; + }; +}; diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt index 383684f42f9..82634169ed9 100644 --- a/cpp/src/arrow/ipc/CMakeLists.txt +++ b/cpp/src/arrow/ipc/CMakeLists.txt @@ -48,4 +48,4 @@ add_custom_command( ) add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES}) -add_dependencies(arrow metadata_fbs) +add_dependencies(arrow_objlib metadata_fbs) diff --git a/cpp/src/arrow/parquet/CMakeLists.txt b/cpp/src/arrow/parquet/CMakeLists.txt index f00bb53c084..00f19b354e3 100644 --- a/cpp/src/arrow/parquet/CMakeLists.txt +++ b/cpp/src/arrow/parquet/CMakeLists.txt @@ -25,8 +25,8 @@ set(PARQUET_SRCS ) set(PARQUET_LIBS - arrow - ${PARQUET_SHARED_LIB} + arrow_shared + parquet_shared ) add_library(arrow_parquet SHARED diff --git a/cpp/src/arrow/parquet/parquet-io-test.cc b/cpp/src/arrow/parquet/parquet-io-test.cc index 572cae16e58..bfc27d26d63 100644 --- a/cpp/src/arrow/parquet/parquet-io-test.cc +++ b/cpp/src/arrow/parquet/parquet-io-test.cc @@ -411,7 +411,7 @@ class TestPrimitiveParquetIO : public TestParquetIO { public: typedef typename TestType::c_type T; - void TestFile(std::vector& values, int num_chunks, + void MakeTestFile(std::vector& values, int num_chunks, std::unique_ptr* file_reader) { std::shared_ptr schema = this->MakeSchema(Repetition::REQUIRED); std::unique_ptr file_writer = this->MakeWriter(schema); @@ -435,10 +435,10 @@ class TestPrimitiveParquetIO : public TestParquetIO { *file_reader = this->ReaderFromSink(); } - void TestSingleColumnRequiredTableRead(int num_chunks) { + void CheckSingleColumnRequiredTableRead(int num_chunks) { std::vector values(SMALL_SIZE, test_traits::value); std::unique_ptr file_reader; - ASSERT_NO_THROW(TestFile(values, num_chunks, &file_reader)); + ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader)); std::shared_ptr out; this->ReadTableFromFile(std::move(file_reader), &out); @@ -450,10 +450,10 @@ class TestPrimitiveParquetIO : public TestParquetIO { ExpectArray(values.data(), chunked_array->chunk(0).get()); } - void TestSingleColumnRequiredRead(int num_chunks) { + void CheckSingleColumnRequiredRead(int num_chunks) { std::vector values(SMALL_SIZE, test_traits::value); std::unique_ptr file_reader; - ASSERT_NO_THROW(TestFile(values, num_chunks, &file_reader)); + ASSERT_NO_THROW(MakeTestFile(values, num_chunks, &file_reader)); std::shared_ptr out; this->ReadSingleColumnFile(std::move(file_reader), &out); @@ -469,19 +469,19 @@ typedef ::testing::TypesTestSingleColumnRequiredRead(1); + this->CheckSingleColumnRequiredRead(1); } TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredTableRead) { - this->TestSingleColumnRequiredTableRead(1); + this->CheckSingleColumnRequiredTableRead(1); } TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedRead) { - this->TestSingleColumnRequiredRead(4); + this->CheckSingleColumnRequiredRead(4); } TYPED_TEST(TestPrimitiveParquetIO, SingleColumnRequiredChunkedTableRead) { - this->TestSingleColumnRequiredTableRead(4); + this->CheckSingleColumnRequiredTableRead(4); } } // namespace parquet diff --git a/cpp/src/arrow/parquet/reader.cc b/cpp/src/arrow/parquet/reader.cc index 7b05665b230..c7c400e9573 100644 --- a/cpp/src/arrow/parquet/reader.cc +++ b/cpp/src/arrow/parquet/reader.cc @@ -213,7 +213,7 @@ Status FlatColumnReader::Impl::ReadNonNullableBatch(typename ParquetType::c_type using ParquetCType = typename ParquetType::c_type; DCHECK(builder); - const ArrowCType* values_ptr; + const ArrowCType* values_ptr = nullptr; RETURN_NOT_OK( (ConvertPhysicalType(values, values_read, &values_ptr))); RETURN_NOT_OK(builder->Append(values_ptr, values_read)); diff --git a/cpp/src/arrow/parquet/reader.h b/cpp/src/arrow/parquet/reader.h index db7a15753d8..2c8a9dfd025 100644 --- a/cpp/src/arrow/parquet/reader.h +++ b/cpp/src/arrow/parquet/reader.h @@ -23,6 +23,8 @@ #include "parquet/api/reader.h" #include "parquet/api/schema.h" +#include "arrow/util/visibility.h" + namespace arrow { class Array; @@ -77,7 +79,7 @@ class FlatColumnReader; // // This is additionally complicated "chunky" repeated fields or very large byte // arrays -class FileReader { +class ARROW_EXPORT FileReader { public: FileReader(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileReader> reader); @@ -107,7 +109,7 @@ class FileReader { // // We also do not expose any internal Parquet details, such as row groups. This // might change in the future. -class FlatColumnReader { +class ARROW_EXPORT FlatColumnReader { public: virtual ~FlatColumnReader(); diff --git a/cpp/src/arrow/parquet/schema.h b/cpp/src/arrow/parquet/schema.h index 39bee059522..88b5977d223 100644 --- a/cpp/src/arrow/parquet/schema.h +++ b/cpp/src/arrow/parquet/schema.h @@ -25,6 +25,7 @@ #include "arrow/schema.h" #include "arrow/type.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -32,15 +33,16 @@ class Status; namespace parquet { -Status NodeToField(const ::parquet::schema::NodePtr& node, std::shared_ptr* out); +Status ARROW_EXPORT NodeToField( + const ::parquet::schema::NodePtr& node, std::shared_ptr* out); -Status FromParquetSchema( +Status ARROW_EXPORT FromParquetSchema( const ::parquet::SchemaDescriptor* parquet_schema, std::shared_ptr* out); -Status FieldToNode(const std::shared_ptr& field, +Status ARROW_EXPORT FieldToNode(const std::shared_ptr& field, const ::parquet::WriterProperties& properties, ::parquet::schema::NodePtr* out); -Status ToParquetSchema(const Schema* arrow_schema, +Status ARROW_EXPORT ToParquetSchema(const Schema* arrow_schema, const ::parquet::WriterProperties& properties, std::shared_ptr<::parquet::SchemaDescriptor>* out); diff --git a/cpp/src/arrow/parquet/writer.cc b/cpp/src/arrow/parquet/writer.cc index 63449bb20b1..0139edd3bb8 100644 --- a/cpp/src/arrow/parquet/writer.cc +++ b/cpp/src/arrow/parquet/writer.cc @@ -118,7 +118,7 @@ Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer, reinterpret_cast<::parquet::TypedColumnWriter*>(column_writer); if (writer->descr()->max_definition_level() == 0) { // no nulls, just dump the data - const ParquetCType* data_writer_ptr; + const ParquetCType* data_writer_ptr = nullptr; RETURN_NOT_OK((ConvertPhysicalType( data_ptr, length, &data_writer_ptr))); PARQUET_CATCH_NOT_OK(writer->WriteBatch(length, nullptr, nullptr, data_writer_ptr)); @@ -128,7 +128,7 @@ Status FileWriter::Impl::TypedWriteBatch(::parquet::ColumnWriter* column_writer, reinterpret_cast(def_levels_buffer_.mutable_data()); if (data->null_count() == 0) { std::fill(def_levels_ptr, def_levels_ptr + length, 1); - const ParquetCType* data_writer_ptr; + const ParquetCType* data_writer_ptr = nullptr; RETURN_NOT_OK((ConvertPhysicalType( data_ptr, length, &data_writer_ptr))); PARQUET_CATCH_NOT_OK( diff --git a/cpp/src/arrow/parquet/writer.h b/cpp/src/arrow/parquet/writer.h index cfd80d80b79..45d0fd59868 100644 --- a/cpp/src/arrow/parquet/writer.h +++ b/cpp/src/arrow/parquet/writer.h @@ -23,6 +23,8 @@ #include "parquet/api/schema.h" #include "parquet/api/writer.h" +#include "arrow/util/visibility.h" + namespace arrow { class Array; @@ -40,7 +42,7 @@ namespace parquet { * Start a new RowGroup/Chunk with NewRowGroup * Write column-by-column the whole column chunk */ -class FileWriter { +class ARROW_EXPORT FileWriter { public: FileWriter(MemoryPool* pool, std::unique_ptr<::parquet::ParquetFileWriter> writer); @@ -62,7 +64,7 @@ class FileWriter { * * The table shall only consist of nullable, non-repeated columns of primitive type. */ -Status WriteFlatTable(const Table* table, MemoryPool* pool, +Status ARROW_EXPORT WriteFlatTable(const Table* table, MemoryPool* pool, const std::shared_ptr<::parquet::OutputStream>& sink, int64_t chunk_size, const std::shared_ptr<::parquet::WriterProperties>& properties = ::parquet::default_writer_properties()); diff --git a/cpp/src/arrow/schema.h b/cpp/src/arrow/schema.h index a8b0d8444ac..4301968e015 100644 --- a/cpp/src/arrow/schema.h +++ b/cpp/src/arrow/schema.h @@ -22,11 +22,13 @@ #include #include +#include "arrow/util/visibility.h" + namespace arrow { struct Field; -class Schema { +class ARROW_EXPORT Schema { public: explicit Schema(const std::vector>& fields); diff --git a/cpp/src/arrow/symbols.map b/cpp/src/arrow/symbols.map new file mode 100644 index 00000000000..2ca8d730610 --- /dev/null +++ b/cpp/src/arrow/symbols.map @@ -0,0 +1,15 @@ +{ + # Symbols marked as 'local' are not exported by the DSO and thus may not + # be used by client applications. + local: + # devtoolset / static-libstdc++ symbols + __cxa_*; + + extern "C++" { + # devtoolset or -static-libstdc++ - the Red Hat devtoolset statically + # links c++11 symbols into binaries so that the result may be executed on + # a system with an older libstdc++ which doesn't include the necessary + # c++11 symbols. + std::*; + }; +}; diff --git a/cpp/src/arrow/table.h b/cpp/src/arrow/table.h index 756b2a19593..2088fdf0b64 100644 --- a/cpp/src/arrow/table.h +++ b/cpp/src/arrow/table.h @@ -23,6 +23,8 @@ #include #include +#include "arrow/util/visibility.h" + namespace arrow { class Array; @@ -33,7 +35,7 @@ class Status; // A row batch is a simpler and more rigid table data structure intended for // use primarily in shared memory IPC. It contains a schema (metadata) and a // corresponding vector of equal-length Arrow arrays -class RowBatch { +class ARROW_EXPORT RowBatch { public: // num_rows is a parameter to allow for row batches of a particular size not // having any materialized columns. Each array should have the same length as @@ -63,7 +65,7 @@ class RowBatch { }; // Immutable container of fixed-length columns conforming to a particular schema -class Table { +class ARROW_EXPORT Table { public: // If columns is zero-length, the table's number of rows is zero Table(const std::string& name, const std::shared_ptr& schema, diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h index 8fb41211ba9..4cb37fd1dea 100644 --- a/cpp/src/arrow/type.h +++ b/cpp/src/arrow/type.h @@ -24,6 +24,7 @@ #include #include "arrow/util/macros.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -101,7 +102,7 @@ struct Type { struct Field; -struct DataType { +struct ARROW_EXPORT DataType { Type::type type; std::vector> children_; @@ -133,7 +134,7 @@ typedef std::shared_ptr TypePtr; // A field is a piece of metadata that includes (for now) a name and a data // type -struct Field { +struct ARROW_EXPORT Field { // Field name std::string name; @@ -163,7 +164,7 @@ struct Field { typedef std::shared_ptr FieldPtr; template -struct PrimitiveType : public DataType { +struct ARROW_EXPORT PrimitiveType : public DataType { PrimitiveType() : DataType(Derived::type_enum) {} std::string ToString() const override; @@ -185,55 +186,55 @@ inline std::string PrimitiveType::ToString() const { \ static const char* name() { return NAME; } -struct NullType : public PrimitiveType { +struct ARROW_EXPORT NullType : public PrimitiveType { PRIMITIVE_DECL(NullType, void, NA, 0, "null"); }; -struct BooleanType : public PrimitiveType { +struct ARROW_EXPORT BooleanType : public PrimitiveType { PRIMITIVE_DECL(BooleanType, uint8_t, BOOL, 1, "bool"); }; -struct UInt8Type : public PrimitiveType { +struct ARROW_EXPORT UInt8Type : public PrimitiveType { PRIMITIVE_DECL(UInt8Type, uint8_t, UINT8, 1, "uint8"); }; -struct Int8Type : public PrimitiveType { +struct ARROW_EXPORT Int8Type : public PrimitiveType { PRIMITIVE_DECL(Int8Type, int8_t, INT8, 1, "int8"); }; -struct UInt16Type : public PrimitiveType { +struct ARROW_EXPORT UInt16Type : public PrimitiveType { PRIMITIVE_DECL(UInt16Type, uint16_t, UINT16, 2, "uint16"); }; -struct Int16Type : public PrimitiveType { +struct ARROW_EXPORT Int16Type : public PrimitiveType { PRIMITIVE_DECL(Int16Type, int16_t, INT16, 2, "int16"); }; -struct UInt32Type : public PrimitiveType { +struct ARROW_EXPORT UInt32Type : public PrimitiveType { PRIMITIVE_DECL(UInt32Type, uint32_t, UINT32, 4, "uint32"); }; -struct Int32Type : public PrimitiveType { +struct ARROW_EXPORT Int32Type : public PrimitiveType { PRIMITIVE_DECL(Int32Type, int32_t, INT32, 4, "int32"); }; -struct UInt64Type : public PrimitiveType { +struct ARROW_EXPORT UInt64Type : public PrimitiveType { PRIMITIVE_DECL(UInt64Type, uint64_t, UINT64, 8, "uint64"); }; -struct Int64Type : public PrimitiveType { +struct ARROW_EXPORT Int64Type : public PrimitiveType { PRIMITIVE_DECL(Int64Type, int64_t, INT64, 8, "int64"); }; -struct FloatType : public PrimitiveType { +struct ARROW_EXPORT FloatType : public PrimitiveType { PRIMITIVE_DECL(FloatType, float, FLOAT, 4, "float"); }; -struct DoubleType : public PrimitiveType { +struct ARROW_EXPORT DoubleType : public PrimitiveType { PRIMITIVE_DECL(DoubleType, double, DOUBLE, 8, "double"); }; -struct ListType : public DataType { +struct ARROW_EXPORT ListType : public DataType { // List can contain any other logical value type explicit ListType(const std::shared_ptr& value_type) : ListType(value_type, Type::LIST) {} @@ -260,7 +261,7 @@ struct ListType : public DataType { }; // BinaryType type is reprsents lists of 1-byte values. -struct BinaryType : public ListType { +struct ARROW_EXPORT BinaryType : public ListType { BinaryType() : BinaryType(Type::BINARY) {} static char const* name() { return "binary"; } std::string ToString() const override; @@ -272,7 +273,7 @@ struct BinaryType : public ListType { }; // UTF encoded strings -struct StringType : public BinaryType { +struct ARROW_EXPORT StringType : public BinaryType { StringType() : BinaryType(Type::STRING) {} static char const* name() { return "string"; } @@ -283,7 +284,7 @@ struct StringType : public BinaryType { explicit StringType(Type::type logical_type) : BinaryType(logical_type) {} }; -struct StructType : public DataType { +struct ARROW_EXPORT StructType : public DataType { explicit StructType(const std::vector>& fields) : DataType(Type::STRUCT) { children_ = fields; diff --git a/cpp/src/arrow/types/construct.h b/cpp/src/arrow/types/construct.h index d0370840ca1..afdadbe0790 100644 --- a/cpp/src/arrow/types/construct.h +++ b/cpp/src/arrow/types/construct.h @@ -21,6 +21,9 @@ #include #include #include + +#include "arrow/util/visibility.h" + namespace arrow { class Array; @@ -31,18 +34,18 @@ struct Field; class MemoryPool; class Status; -Status MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, +Status ARROW_EXPORT MakeBuilder(MemoryPool* pool, const std::shared_ptr& type, std::shared_ptr* out); // Create new arrays for logical types that are backed by primitive arrays. -Status MakePrimitiveArray(const std::shared_ptr& type, int32_t length, - const std::shared_ptr& data, int32_t null_count, +Status ARROW_EXPORT MakePrimitiveArray(const std::shared_ptr& type, + int32_t length, const std::shared_ptr& data, int32_t null_count, const std::shared_ptr& null_bitmap, std::shared_ptr* out); // Create new list arrays for logical types that are backed by ListArrays (e.g. list of // primitives and strings) // TODO(emkornfield) split up string vs list? -Status MakeListArray(const std::shared_ptr& type, int32_t length, +Status ARROW_EXPORT MakeListArray(const std::shared_ptr& type, int32_t length, const std::shared_ptr& offests, const std::shared_ptr& values, int32_t null_count, const std::shared_ptr& null_bitmap, std::shared_ptr* out); diff --git a/cpp/src/arrow/types/decimal.h b/cpp/src/arrow/types/decimal.h index 598df3ef70d..6c497c597d9 100644 --- a/cpp/src/arrow/types/decimal.h +++ b/cpp/src/arrow/types/decimal.h @@ -21,10 +21,11 @@ #include #include "arrow/type.h" +#include "arrow/util/visibility.h" namespace arrow { -struct DecimalType : public DataType { +struct ARROW_EXPORT DecimalType : public DataType { explicit DecimalType(int precision_, int scale_) : DataType(Type::DECIMAL), precision(precision_), scale(scale_) {} int precision; diff --git a/cpp/src/arrow/types/list.h b/cpp/src/arrow/types/list.h index 2f6f85d66ca..f3894510d09 100644 --- a/cpp/src/arrow/types/list.h +++ b/cpp/src/arrow/types/list.h @@ -31,12 +31,13 @@ #include "arrow/util/buffer.h" #include "arrow/util/logging.h" #include "arrow/util/status.h" +#include "arrow/util/visibility.h" namespace arrow { class MemoryPool; -class ListArray : public Array { +class ARROW_EXPORT ListArray : public Array { public: ListArray(const TypePtr& type, int32_t length, std::shared_ptr offsets, const ArrayPtr& values, int32_t null_count = 0, @@ -96,7 +97,7 @@ class ListArray : public Array { // represent multiple different logical types. If no logical type is provided // at construction time, the class defaults to List where t is taken from the // value_builder/values that the object is constructed with. -class ListBuilder : public ArrayBuilder { +class ARROW_EXPORT ListBuilder : public ArrayBuilder { public: // Use this constructor to incrementally build the value array along with offsets and // null bitmap. @@ -116,6 +117,8 @@ class ListBuilder : public ArrayBuilder { offset_builder_(pool), values_(values) {} + virtual ~ListBuilder() {} + Status Init(int32_t elements) override { DCHECK_LT(elements, std::numeric_limits::max()); RETURN_NOT_OK(ArrayBuilder::Init(elements)); diff --git a/cpp/src/arrow/types/primitive.h b/cpp/src/arrow/types/primitive.h index f1ec417d510..18f954adc08 100644 --- a/cpp/src/arrow/types/primitive.h +++ b/cpp/src/arrow/types/primitive.h @@ -29,6 +29,7 @@ #include "arrow/util/bit-util.h" #include "arrow/util/buffer.h" #include "arrow/util/status.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -36,7 +37,7 @@ class MemoryPool; // Base class for fixed-size logical types. See MakePrimitiveArray // (types/construct.h) for constructing a specific subclass. -class PrimitiveArray : public Array { +class ARROW_EXPORT PrimitiveArray : public Array { public: virtual ~PrimitiveArray() {} @@ -53,7 +54,7 @@ class PrimitiveArray : public Array { }; #define NUMERIC_ARRAY_DECL(NAME, TypeClass, T) \ - class NAME : public PrimitiveArray { \ + class ARROW_EXPORT NAME : public PrimitiveArray { \ public: \ using value_type = T; \ \ @@ -102,7 +103,7 @@ NUMERIC_ARRAY_DECL(FloatArray, FloatType, float); NUMERIC_ARRAY_DECL(DoubleArray, DoubleType, double); template -class PrimitiveBuilder : public ArrayBuilder { +class ARROW_EXPORT PrimitiveBuilder : public ArrayBuilder { public: typedef typename Type::c_type value_type; @@ -149,7 +150,7 @@ class PrimitiveBuilder : public ArrayBuilder { }; template -class NumericBuilder : public PrimitiveBuilder { +class ARROW_EXPORT NumericBuilder : public PrimitiveBuilder { public: using typename PrimitiveBuilder::value_type; using PrimitiveBuilder::PrimitiveBuilder; @@ -262,7 +263,7 @@ typedef NumericBuilder Int64Builder; typedef NumericBuilder FloatBuilder; typedef NumericBuilder DoubleBuilder; -class BooleanArray : public PrimitiveArray { +class ARROW_EXPORT BooleanArray : public PrimitiveArray { public: BooleanArray(int32_t length, const std::shared_ptr& data, int32_t null_count = 0, const std::shared_ptr& null_bitmap = nullptr); @@ -288,7 +289,7 @@ struct type_traits { } }; -class BooleanBuilder : public PrimitiveBuilder { +class ARROW_EXPORT BooleanBuilder : public PrimitiveBuilder { public: explicit BooleanBuilder(MemoryPool* pool, const TypePtr& type) : PrimitiveBuilder(pool, type) {} diff --git a/cpp/src/arrow/types/string-test.cc b/cpp/src/arrow/types/string-test.cc index a141fc11321..6807b00e8ca 100644 --- a/cpp/src/arrow/types/string-test.cc +++ b/cpp/src/arrow/types/string-test.cc @@ -115,7 +115,7 @@ TEST_F(TestStringContainer, TestListFunctions) { int pos = 0; for (size_t i = 0; i < expected_.size(); ++i) { ASSERT_EQ(pos, strings_->value_offset(i)); - ASSERT_EQ(expected_[i].size(), strings_->value_length(i)); + ASSERT_EQ(static_cast(expected_[i].size()), strings_->value_length(i)); pos += expected_[i].size(); } } @@ -189,7 +189,7 @@ TEST_F(TestStringBuilder, TestScalarAppend) { ASSERT_FALSE(result_->IsNull(i)); result_->GetValue(i, &length); ASSERT_EQ(pos, result_->offset(i)); - ASSERT_EQ(strings[i % N].size(), length); + ASSERT_EQ(static_cast(strings[i % N].size()), length); ASSERT_EQ(strings[i % N], result_->GetString(i)); pos += length; @@ -267,7 +267,7 @@ TEST_F(TestBinaryContainer, TestListFunctions) { int pos = 0; for (size_t i = 0; i < expected_.size(); ++i) { ASSERT_EQ(pos, strings_->value_offset(i)); - ASSERT_EQ(expected_[i].size(), strings_->value_length(i)); + ASSERT_EQ(static_cast(expected_[i].size()), strings_->value_length(i)); pos += expected_[i].size(); } } @@ -339,7 +339,7 @@ TEST_F(TestBinaryBuilder, TestScalarAppend) { } else { ASSERT_FALSE(result_->IsNull(i)); const uint8_t* vals = result_->GetValue(i, &length); - ASSERT_EQ(strings[i % N].size(), length); + ASSERT_EQ(static_cast(strings[i % N].size()), length); ASSERT_EQ(0, std::memcmp(vals, strings[i % N].data(), length)); } } diff --git a/cpp/src/arrow/types/string.cc b/cpp/src/arrow/types/string.cc index da02c7d1d8a..2f0037024c7 100644 --- a/cpp/src/arrow/types/string.cc +++ b/cpp/src/arrow/types/string.cc @@ -61,6 +61,15 @@ Status StringArray::Validate() const { return BinaryArray::Validate(); } -TypePtr BinaryBuilder::value_type_ = TypePtr(new UInt8Type()); +// This used to be a static member variable of BinaryBuilder, but it can cause +// valgrind to report a (spurious?) memory leak when needed in other shared +// libraries. The problem came up while adding explicit visibility to libarrow +// and libarrow_parquet +static TypePtr kBinaryValueType = TypePtr(new UInt8Type()); + +BinaryBuilder::BinaryBuilder(MemoryPool* pool, const TypePtr& type) + : ListBuilder(pool, std::make_shared(pool, kBinaryValueType), type) { + byte_builder_ = static_cast(value_builder_.get()); +} } // namespace arrow diff --git a/cpp/src/arrow/types/string.h b/cpp/src/arrow/types/string.h index b3c00d298b3..bab0c58f617 100644 --- a/cpp/src/arrow/types/string.h +++ b/cpp/src/arrow/types/string.h @@ -28,13 +28,14 @@ #include "arrow/types/list.h" #include "arrow/types/primitive.h" #include "arrow/util/status.h" +#include "arrow/util/visibility.h" namespace arrow { class Buffer; class MemoryPool; -class BinaryArray : public ListArray { +class ARROW_EXPORT BinaryArray : public ListArray { public: BinaryArray(int32_t length, const std::shared_ptr& offsets, const ArrayPtr& values, int32_t null_count = 0, @@ -62,7 +63,7 @@ class BinaryArray : public ListArray { const uint8_t* raw_bytes_; }; -class StringArray : public BinaryArray { +class ARROW_EXPORT StringArray : public BinaryArray { public: StringArray(int32_t length, const std::shared_ptr& offsets, const ArrayPtr& values, int32_t null_count = 0, @@ -87,12 +88,10 @@ class StringArray : public BinaryArray { }; // BinaryBuilder : public ListBuilder -class BinaryBuilder : public ListBuilder { +class ARROW_EXPORT BinaryBuilder : public ListBuilder { public: - explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type) - : ListBuilder(pool, std::make_shared(pool, value_type_), type) { - byte_builder_ = static_cast(value_builder_.get()); - } + explicit BinaryBuilder(MemoryPool* pool, const TypePtr& type); + virtual ~BinaryBuilder() {} Status Append(const uint8_t* value, int32_t length) { RETURN_NOT_OK(ListBuilder::Append()); @@ -105,11 +104,10 @@ class BinaryBuilder : public ListBuilder { protected: UInt8Builder* byte_builder_; - static TypePtr value_type_; }; // String builder -class StringBuilder : public BinaryBuilder { +class ARROW_EXPORT StringBuilder : public BinaryBuilder { public: explicit StringBuilder(MemoryPool* pool, const TypePtr& type) : BinaryBuilder(pool, type) {} diff --git a/cpp/src/arrow/types/struct-test.cc b/cpp/src/arrow/types/struct-test.cc index d2bd2971d04..ccf5a52dc83 100644 --- a/cpp/src/arrow/types/struct-test.cc +++ b/cpp/src/arrow/types/struct-test.cc @@ -116,7 +116,7 @@ class TestStructBuilder : public TestBuilder { ASSERT_OK(MakeBuilder(pool_, type_, &tmp)); builder_ = std::dynamic_pointer_cast(tmp); - ASSERT_EQ(2, builder_->field_builders().size()); + ASSERT_EQ(2, static_cast(builder_->field_builders().size())); } void Done() { result_ = std::dynamic_pointer_cast(builder_->Finish()); } @@ -132,7 +132,7 @@ class TestStructBuilder : public TestBuilder { TEST_F(TestStructBuilder, TestAppendNull) { ASSERT_OK(builder_->AppendNull()); ASSERT_OK(builder_->AppendNull()); - ASSERT_EQ(2, builder_->field_builders().size()); + ASSERT_EQ(2, static_cast(builder_->field_builders().size())); ListBuilder* list_vb = static_cast(builder_->field_builder(0).get()); ASSERT_OK(list_vb->AppendNull()); @@ -148,7 +148,7 @@ TEST_F(TestStructBuilder, TestAppendNull) { ASSERT_OK(result_->Validate()); - ASSERT_EQ(2, result_->fields().size()); + ASSERT_EQ(2, static_cast(result_->fields().size())); ASSERT_EQ(2, result_->length()); ASSERT_EQ(2, result_->field(0)->length()); ASSERT_EQ(2, result_->field(1)->length()); @@ -174,7 +174,7 @@ TEST_F(TestStructBuilder, TestBasics) { ListBuilder* list_vb = static_cast(builder_->field_builder(0).get()); Int8Builder* char_vb = static_cast(list_vb->value_builder().get()); Int32Builder* int_vb = static_cast(builder_->field_builder(1).get()); - ASSERT_EQ(2, builder_->field_builders().size()); + ASSERT_EQ(2, static_cast(builder_->field_builders().size())); EXPECT_OK(builder_->Resize(list_lengths.size())); EXPECT_OK(char_vb->Resize(list_values.size())); diff --git a/cpp/src/arrow/types/struct.h b/cpp/src/arrow/types/struct.h index 78afd29eb8d..63955eb31bb 100644 --- a/cpp/src/arrow/types/struct.h +++ b/cpp/src/arrow/types/struct.h @@ -25,10 +25,11 @@ #include "arrow/type.h" #include "arrow/types/list.h" #include "arrow/types/primitive.h" +#include "arrow/util/visibility.h" namespace arrow { -class StructArray : public Array { +class ARROW_EXPORT StructArray : public Array { public: StructArray(const TypePtr& type, int32_t length, std::vector& field_arrays, int32_t null_count = 0, std::shared_ptr null_bitmap = nullptr) @@ -64,7 +65,7 @@ class StructArray : public Array { // Append, Resize and Reserve methods are acting on StructBuilder. // Please make sure all these methods of all child-builders' are consistently // called to maintain data-structure consistency. -class StructBuilder : public ArrayBuilder { +class ARROW_EXPORT StructBuilder : public ArrayBuilder { public: StructBuilder(MemoryPool* pool, const std::shared_ptr& type, const std::vector>& field_builders) diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index d2a4b091fad..4e941fb5f5c 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -27,6 +27,7 @@ install(FILES macros.h memory-pool.h status.h + visibility.h DESTINATION include/arrow/util) ####################################### diff --git a/cpp/src/arrow/util/buffer.h b/cpp/src/arrow/util/buffer.h index f845d67761f..1aeebc69b4e 100644 --- a/cpp/src/arrow/util/buffer.h +++ b/cpp/src/arrow/util/buffer.h @@ -26,6 +26,7 @@ #include "arrow/util/bit-util.h" #include "arrow/util/macros.h" #include "arrow/util/status.h" +#include "arrow/util/visibility.h" namespace arrow { @@ -41,7 +42,7 @@ class Status; // Capacity is the number of bytes that where allocated for the buffer in // total. // The following invariant is always true: Size < Capacity -class Buffer : public std::enable_shared_from_this { +class ARROW_EXPORT Buffer : public std::enable_shared_from_this { public: Buffer(const uint8_t* data, int64_t size) : data_(data), size_(size), capacity_(size) {} virtual ~Buffer(); @@ -95,7 +96,7 @@ class Buffer : public std::enable_shared_from_this { }; // A Buffer whose contents can be mutated. May or may not own its data. -class MutableBuffer : public Buffer { +class ARROW_EXPORT MutableBuffer : public Buffer { public: MutableBuffer(uint8_t* data, int64_t size) : Buffer(data, size) { mutable_data_ = data; @@ -112,7 +113,7 @@ class MutableBuffer : public Buffer { uint8_t* mutable_data_; }; -class ResizableBuffer : public MutableBuffer { +class ARROW_EXPORT ResizableBuffer : public MutableBuffer { public: // Change buffer reported size to indicated size, allocating memory if // necessary. This will ensure that the capacity of the buffer is a multiple @@ -129,7 +130,7 @@ class ResizableBuffer : public MutableBuffer { }; // A Buffer whose lifetime is tied to a particular MemoryPool -class PoolBuffer : public ResizableBuffer { +class ARROW_EXPORT PoolBuffer : public ResizableBuffer { public: explicit PoolBuffer(MemoryPool* pool = nullptr); virtual ~PoolBuffer(); @@ -145,7 +146,8 @@ static constexpr int64_t MIN_BUFFER_CAPACITY = 1024; class BufferBuilder { public: - explicit BufferBuilder(MemoryPool* pool) : pool_(pool), capacity_(0), size_(0) {} + explicit BufferBuilder(MemoryPool* pool) + : pool_(pool), data_(nullptr), capacity_(0), size_(0) {} // Resizes the buffer to the nearest multiple of 64 bytes per Layout.md Status Resize(int32_t elements) { diff --git a/cpp/src/arrow/util/memory-pool-test.cc b/cpp/src/arrow/util/memory-pool-test.cc index 4ab9736c2b4..8e7dfd60baa 100644 --- a/cpp/src/arrow/util/memory-pool-test.cc +++ b/cpp/src/arrow/util/memory-pool-test.cc @@ -31,7 +31,7 @@ TEST(DefaultMemoryPool, MemoryTracking) { uint8_t* data; ASSERT_OK(pool->Allocate(100, &data)); - EXPECT_EQ(0, reinterpret_cast(data) % 64); + EXPECT_EQ(static_cast(0), reinterpret_cast(data) % 64); ASSERT_EQ(100, pool->bytes_allocated()); pool->Free(data, 100); diff --git a/cpp/src/arrow/util/memory-pool.h b/cpp/src/arrow/util/memory-pool.h index 824c7248e2e..4c1d699addd 100644 --- a/cpp/src/arrow/util/memory-pool.h +++ b/cpp/src/arrow/util/memory-pool.h @@ -20,11 +20,13 @@ #include +#include "arrow/util/visibility.h" + namespace arrow { class Status; -class MemoryPool { +class ARROW_EXPORT MemoryPool { public: virtual ~MemoryPool(); @@ -34,7 +36,7 @@ class MemoryPool { virtual int64_t bytes_allocated() const = 0; }; -MemoryPool* default_memory_pool(); +ARROW_EXPORT MemoryPool* default_memory_pool(); } // namespace arrow diff --git a/cpp/src/arrow/util/status.cc b/cpp/src/arrow/util/status.cc index d194ed5572f..8dd07d0d064 100644 --- a/cpp/src/arrow/util/status.cc +++ b/cpp/src/arrow/util/status.cc @@ -58,6 +58,9 @@ std::string Status::CodeAsString() const { case StatusCode::NotImplemented: type = "NotImplemented"; break; + default: + type = "Unknown"; + break; } return std::string(type); } diff --git a/cpp/src/arrow/util/status.h b/cpp/src/arrow/util/status.h index d1a74250008..6ba2035bcd3 100644 --- a/cpp/src/arrow/util/status.h +++ b/cpp/src/arrow/util/status.h @@ -19,6 +19,8 @@ #include #include +#include "arrow/util/visibility.h" + // Return the given status if it is not OK. #define ARROW_RETURN_NOT_OK(s) \ do { \ @@ -82,7 +84,7 @@ enum class StatusCode : char { NotImplemented = 10, }; -class Status { +class ARROW_EXPORT Status { public: // Create a success status. Status() : state_(NULL) {} diff --git a/cpp/src/arrow/util/visibility.h b/cpp/src/arrow/util/visibility.h new file mode 100644 index 00000000000..b197c198297 --- /dev/null +++ b/cpp/src/arrow/util/visibility.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef ARROW_UTIL_VISIBILITY_H +#define ARROW_UTIL_VISIBILITY_H + +#if defined(_WIN32) || defined(__CYGWIN__) +#define ARROW_EXPORT __declspec(dllexport) +#else // Not Windows +#ifndef ARROW_EXPORT +#define ARROW_EXPORT __attribute__((visibility("default"))) +#endif +#ifndef ARROW_NO_EXPORT +#define ARROW_NO_EXPORT __attribute__((visibility("hidden"))) +#endif +#endif // Non-Windows + +#endif // ARROW_UTIL_VISIBILITY_H diff --git a/python/conda.recipe/build.sh b/python/conda.recipe/build.sh index a164c1af518..f32710073c7 100644 --- a/python/conda.recipe/build.sh +++ b/python/conda.recipe/build.sh @@ -19,13 +19,14 @@ if [ "$(uname)" == "Darwin" ]; then export MACOSX_DEPLOYMENT_TARGET=10.7 fi -echo Setting the compiler... -if [ `uname` == Linux ]; then - EXTRA_CMAKE_ARGS=-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++ -elif [ `uname` == Darwin ]; then - EXTRA_CMAKE_ARGS= -fi +# echo Setting the compiler... +# if [ `uname` == Linux ]; then +# EXTRA_CMAKE_ARGS=-DCMAKE_SHARED_LINKER_FLAGS=-static-libstdc++ +# elif [ `uname` == Darwin ]; then +# EXTRA_CMAKE_ARGS= +# fi cd .. -$PYTHON setup.py build_ext --extra-cmake-args=$EXTRA_CMAKE_ARGS || exit 1 +# $PYTHON setup.py build_ext --extra-cmake-args=$EXTRA_CMAKE_ARGS || exit 1 +$PYTHON setup.py build_ext || exit 1 $PYTHON setup.py install || exit 1 diff --git a/python/src/pyarrow/adapters/builtin.h b/python/src/pyarrow/adapters/builtin.h index 88869c20480..4e997e31dd6 100644 --- a/python/src/pyarrow/adapters/builtin.h +++ b/python/src/pyarrow/adapters/builtin.h @@ -28,6 +28,7 @@ #include #include "pyarrow/common.h" +#include "pyarrow/visibility.h" namespace arrow { class Array; } @@ -35,6 +36,7 @@ namespace pyarrow { class Status; +PYARROW_EXPORT Status ConvertPySequence(PyObject* obj, std::shared_ptr* out); } // namespace pyarrow diff --git a/python/src/pyarrow/adapters/pandas.h b/python/src/pyarrow/adapters/pandas.h index 17922349de6..c3377685bcc 100644 --- a/python/src/pyarrow/adapters/pandas.h +++ b/python/src/pyarrow/adapters/pandas.h @@ -25,6 +25,8 @@ #include +#include "pyarrow/visibility.h" + namespace arrow { class Array; @@ -36,12 +38,15 @@ namespace pyarrow { class Status; +PYARROW_EXPORT Status ArrowToPandas(const std::shared_ptr& col, PyObject* py_ref, PyObject** out); +PYARROW_EXPORT Status PandasMaskedToArrow(arrow::MemoryPool* pool, PyObject* ao, PyObject* mo, std::shared_ptr* out); +PYARROW_EXPORT Status PandasToArrow(arrow::MemoryPool* pool, PyObject* ao, std::shared_ptr* out); diff --git a/python/src/pyarrow/common.h b/python/src/pyarrow/common.h index 0211e8948f2..fb0ba3e4822 100644 --- a/python/src/pyarrow/common.h +++ b/python/src/pyarrow/common.h @@ -22,6 +22,8 @@ #include "arrow/util/buffer.h" +#include "pyarrow/visibility.h" + namespace arrow { class MemoryPool; } namespace pyarrow { @@ -94,9 +96,9 @@ struct PyObjectStringify { return Status::UnknownError(message); \ } -arrow::MemoryPool* GetMemoryPool(); +PYARROW_EXPORT arrow::MemoryPool* GetMemoryPool(); -class NumPyBuffer : public arrow::Buffer { +class PYARROW_EXPORT NumPyBuffer : public arrow::Buffer { public: NumPyBuffer(PyArrayObject* arr) : Buffer(nullptr, 0) { diff --git a/python/src/pyarrow/config.h b/python/src/pyarrow/config.h index 48ae715d842..82936b1a5f3 100644 --- a/python/src/pyarrow/config.h +++ b/python/src/pyarrow/config.h @@ -21,6 +21,7 @@ #include #include "pyarrow/numpy_interop.h" +#include "pyarrow/visibility.h" #if PY_MAJOR_VERSION >= 3 #define PyString_Check PyUnicode_Check @@ -28,10 +29,13 @@ namespace pyarrow { +PYARROW_EXPORT extern PyObject* numpy_nan; +PYARROW_EXPORT void pyarrow_init(); +PYARROW_EXPORT void pyarrow_set_numpy_nan(PyObject* obj); } // namespace pyarrow diff --git a/python/src/pyarrow/helpers.h b/python/src/pyarrow/helpers.h index ec42bb31d3b..fa9c713b0c2 100644 --- a/python/src/pyarrow/helpers.h +++ b/python/src/pyarrow/helpers.h @@ -21,6 +21,8 @@ #include #include +#include "pyarrow/visibility.h" + namespace pyarrow { using arrow::DataType; @@ -40,6 +42,7 @@ extern const std::shared_ptr FLOAT; extern const std::shared_ptr DOUBLE; extern const std::shared_ptr STRING; +PYARROW_EXPORT std::shared_ptr GetPrimitiveType(Type::type type); } // namespace pyarrow diff --git a/python/src/pyarrow/status.h b/python/src/pyarrow/status.h index cb8c8add210..67cd66c58ee 100644 --- a/python/src/pyarrow/status.h +++ b/python/src/pyarrow/status.h @@ -17,6 +17,8 @@ #include #include +#include "pyarrow/visibility.h" + namespace pyarrow { #define PY_RETURN_NOT_OK(s) do { \ @@ -38,7 +40,7 @@ enum class StatusCode: char { UnknownError = 10 }; -class Status { +class PYARROW_EXPORT Status { public: // Create a success status. Status() : state_(NULL) { } diff --git a/python/src/pyarrow/visibility.h b/python/src/pyarrow/visibility.h new file mode 100644 index 00000000000..9f0c13b4b20 --- /dev/null +++ b/python/src/pyarrow/visibility.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef PYARROW_VISIBILITY_H +#define PYARROW_VISIBILITY_H + +#if defined(_WIN32) || defined(__CYGWIN__) +#define PYARROW_EXPORT __declspec(dllexport) +#else // Not Windows +#ifndef PYARROW_EXPORT +#define PYARROW_EXPORT __attribute__((visibility("default"))) +#endif +#ifndef PYARROW_NO_EXPORT +#define PYARROW_NO_EXPORT __attribute__((visibility("hidden"))) +#endif +#endif // Non-Windows + +#endif // PYARROW_VISIBILITY_H From 02538271cfc4166e9920912507ef9be1d0bc42f5 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Sun, 10 Jul 2016 13:01:23 -0700 Subject: [PATCH 2/2] Remove -Wno-unused-local-typedef --- cpp/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d1a9e3a44d1..a39a7521231 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -183,7 +183,6 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang") # http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html # http://petereisentraut.blogspot.com/2011/09/ccache-and-clang-part-2.html set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedef") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CLANG_OPTIONS}") endif()