From 5c19005fb710d2ef67fae5c258c2783faf660ad3 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 10 Jul 2020 16:31:34 -0400 Subject: [PATCH 01/31] first cut simplifying python build --- .gitignore | 1 + cmake/Pybind.txt.in | 16 - cmake/modules/FindNumPy.cmake | 41 -- cmake/modules/FindPyArrow.cmake | 82 ---- cmake/modules/FindPybind.cmake | 40 -- cmake/modules/FindPythonHeaders.cmake | 268 ------------- cpp/perspective/CMakeLists.txt | 542 ++++---------------------- python/perspective/setup.py | 241 +++++++----- scripts/build_python.js | 21 +- scripts/clean.js | 2 +- 10 files changed, 258 insertions(+), 996 deletions(-) delete mode 100644 cmake/Pybind.txt.in delete mode 100644 cmake/modules/FindNumPy.cmake delete mode 100644 cmake/modules/FindPyArrow.cmake delete mode 100644 cmake/modules/FindPybind.cmake delete mode 100644 cmake/modules/FindPythonHeaders.cmake diff --git a/.gitignore b/.gitignore index cf1e63a851..8799afc976 100644 --- a/.gitignore +++ b/.gitignore @@ -133,6 +133,7 @@ packages/*/cjs cppbuild docsbuild .coverage +cpp/perspective/third # docs generated docs/_build diff --git a/cmake/Pybind.txt.in b/cmake/Pybind.txt.in deleted file mode 100644 index 440ed2796f..0000000000 --- a/cmake/Pybind.txt.in +++ /dev/null @@ -1,16 +0,0 @@ -cmake_minimum_required(VERSION 3.7.2) - -project(pybind11-download NONE) - -include(ExternalProject) -ExternalProject_Add(pybind11 - GIT_REPOSITORY https://github.com/pybind/pybind11.git - GIT_TAG master - SOURCE_DIR "${CMAKE_BINARY_DIR}/pybind11-src" - BINARY_DIR "${CMAKE_BINARY_DIR}/pybind11-build" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - CMAKE_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" -) diff --git a/cmake/modules/FindNumPy.cmake b/cmake/modules/FindNumPy.cmake deleted file mode 100644 index 5997fcfd7b..0000000000 --- a/cmake/modules/FindNumPy.cmake +++ /dev/null @@ -1,41 +0,0 @@ -# Find the Python NumPy package -# PYTHON_NUMPY_INCLUDE_DIR -# PYTHON_NUMPY_FOUND -# will be set by this script - -cmake_minimum_required(VERSION 2.6) - -if(NOT Python_EXECUTABLE) - if(NumPy_FIND_QUIETLY) - find_package( PythonInterp REQUIRED QUIET) - else() - find_package( PythonInterp REQUIRED ) - set(__numpy_out 1) - endif() -endif() - -if (Python_EXECUTABLE) - # Find out the include path - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function;import numpy;print(numpy.get_include(), end='')" - OUTPUT_VARIABLE __numpy_path) - # And the version - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function;import numpy;print(numpy.__version__, end='')" - OUTPUT_VARIABLE __numpy_version) -elseif(__numpy_out) - message(STATUS "Python executable not found.") -endif(Python_EXECUTABLE) - -find_path(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h - HINTS "${__numpy_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH) - -if(PYTHON_NUMPY_INCLUDE_DIR) - set(PYTHON_NUMPY_FOUND 1 CACHE INTERNAL "Python numpy found") -endif(PYTHON_NUMPY_INCLUDE_DIR) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(NumPy REQUIRED_VARS PYTHON_NUMPY_INCLUDE_DIR - VERSION_VAR __numpy_version) diff --git a/cmake/modules/FindPyArrow.cmake b/cmake/modules/FindPyArrow.cmake deleted file mode 100644 index fe2785578e..0000000000 --- a/cmake/modules/FindPyArrow.cmake +++ /dev/null @@ -1,82 +0,0 @@ -# Find the Python PyArrow package -# PYTHON_PYARROW_INCLUDE_DIR -# PYTHON_PYARROW_FOUND -# PYTHON_PYARROW_LIBRARY_DIR -# PYTHON_PYARROW_LIBRARIES -# will be set by this script - -cmake_minimum_required(VERSION 2.6) - -if(NOT Python_EXECUTABLE) - if(PyArrow_FIND_QUIETLY) - find_package( PythonInterp 3.7 REQUIRED ) - else() - find_package( PythonInterp 3.7 REQUIRED ) - set(__numpy_out 1) - endif() -endif() - -if (Python_EXECUTABLE) - # Find out the include path - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import pyarrow; print(pyarrow.get_include(), end='')\nexcept:pass" - OUTPUT_VARIABLE __pyarrow_path) - # And the lib dirs - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import pyarrow; print(pyarrow.get_library_dirs()[0], end='')\nexcept:pass" - OUTPUT_VARIABLE __pyarrow_library_dirs) - - # And the lib dirs - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import pyarrow; print(' '.join(pyarrow.get_libraries()), end='')\nexcept:pass" - OUTPUT_VARIABLE __pyarrow_libraries) - - # And the version - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import pyarrow; print(pyarrow.__version__, end='')\nexcept:pass" - OUTPUT_VARIABLE __pyarrow_version) -elseif(__pyarrow_out) - message(STATUS "Python executable not found.") -endif(Python_EXECUTABLE) - -find_path(PYTHON_PYARROW_INCLUDE_DIR arrow/python/api.h - HINTS "${__pyarrow_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH) - -set(PYTHON_PYARROW_LIBRARY_DIR ${__pyarrow_library_dirs}) - -# Figure out the major version for the .so/.dylibs -message(${__pyarrow_version}) -string(REPLACE "." ";" PYARROW_VERSION_LIST ${__pyarrow_version}) -message(${PYARROW_VERSION_LIST}) -list(GET PYARROW_VERSION_LIST 0 PYARROW_VERSION_MAJOR) -list(GET PYARROW_VERSION_LIST 1 PYARROW_VERSION_MINOR) -list(GET PYARROW_VERSION_LIST 2 PYARROW_VERSION_PATCH) - -if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") - # windows its just "arrow.dll" - set(PYTHON_PYARROW_PYTHON_SHARED_LIBRARY "arrow_python") - set(PYTHON_PYARROW_ARROW_SHARED_LIBRARY "arrow") - set(PYTHON_PYARROW_LIBRARIES ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY}) -elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") - # Link against pre-built libarrow on MacOS - set(PYTHON_PYARROW_PYTHON_SHARED_LIBRARY ${PYTHON_PYARROW_LIBRARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow_python.${PYARROW_VERSION_MINOR}.dylib) - set(PYTHON_PYARROW_ARROW_SHARED_LIBRARY ${PYTHON_PYARROW_LIBRARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow.${PYARROW_VERSION_MINOR}.dylib) - set(PYTHON_PYARROW_LIBRARIES ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY}) -else() - # linux - set(PYTHON_PYARROW_PYTHON_SHARED_LIBRARY ${CMAKE_SHARED_LIBRARY_PREFIX}arrow_python${CMAKE_SHARED_LIBRARY_SUFFIX}.${PYARROW_VERSION_MINOR}) - set(PYTHON_PYARROW_ARROW_SHARED_LIBRARY ${CMAKE_SHARED_LIBRARY_PREFIX}arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PYARROW_VERSION_MINOR}) - set(PYTHON_PYARROW_LIBRARIES ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY}) -endif() - -if(PYTHON_PYARROW_INCLUDE_DIR AND PYTHON_PYARROW_LIBRARIES) - set(PYTHON_PYARROW_FOUND 1 CACHE INTERNAL "Python pyarrow found") -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(PyArrow REQUIRED_VARS PYTHON_PYARROW_INCLUDE_DIR PYTHON_PYARROW_LIBRARIES PYTHON_PYARROW_LIBRARY_DIR - VERSION_VAR __pyarrow_version) diff --git a/cmake/modules/FindPybind.cmake b/cmake/modules/FindPybind.cmake deleted file mode 100644 index cb31165cd0..0000000000 --- a/cmake/modules/FindPybind.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# Find the Python PyArrow package -# PYTHON_PYBIND_INCLUDE_DIR -# PYTHON_PYBIND_FOUND -# will be set by this script - -cmake_minimum_required(VERSION 2.6) - -if(NOT Python_EXECUTABLE) - if(PyArrow_FIND_QUIETLY) - find_package( PythonInterp 3.7 REQUIRED ) - else() - find_package( PythonInterp 3.7 REQUIRED ) - set(__numpy_out 1) - endif() -endif() - -if (Python_EXECUTABLE) - # Find out the include path - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import pybind11; print(pybind11.get_include(), end='')\nexcept:pass" - OUTPUT_VARIABLE __pybind_path) - - execute_process( - COMMAND "${Python_EXECUTABLE}" -c - "from __future__ import print_function\ntry: import pybind11; print('.'.join(int(x) for x in pybind11.version_info), end='')\nexcept:pass" - OUTPUT_VARIABLE __pybind_version) -elseif(__pybind_out) - message(STATUS "Python executable not found.") -endif(Python_EXECUTABLE) - -find_path(PYTHON_PYBIND_INCLUDE_DIR pybind11/pybind11.h - HINTS "${__pybind_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH) - -if(PYTHON_PYBIND_INCLUDE_DIR) - set(PYTHON_PYBIND_FOUND 1 CACHE INTERNAL "Python pybind11 found") -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(PyBind REQUIRED_VARS PYTHON_PYBIND_INCLUDE_DIR VERSION_VAR __pybind_version) diff --git a/cmake/modules/FindPythonHeaders.cmake b/cmake/modules/FindPythonHeaders.cmake deleted file mode 100644 index 12b9c9a418..0000000000 --- a/cmake/modules/FindPythonHeaders.cmake +++ /dev/null @@ -1,268 +0,0 @@ -# Distributed under the OSI-approved BSD 3-Clause License. See accompanying -# file Copyright.txt or https://cmake.org/licensing for details. - -#[=======================================================================[.rst: -FindPythonHeaders --------------- - -.. deprecated:: 3.12 - - Use :module:`FindPython3`, :module:`FindPython2` or :module:`FindPython` instead. - -Find python libraries - -This module finds if Python is installed and determines where the -include files and libraries are. It also determines what the name of -the library is. This code sets the following variables: - -:: - - PYTHONLIBS_FOUND - have the Python libs been found - PYTHON_LIBRARIES - path to the python library - PYTHON_INCLUDE_PATH - path to where Python.h is found (deprecated) - PYTHON_INCLUDE_DIRS - path to where Python.h is found - PYTHON_DEBUG_LIBRARIES - path to the debug library (deprecated) - PYTHONLIBS_VERSION_STRING - version of the Python libs found (since CMake 2.8.8) - - - -The Python_ADDITIONAL_VERSIONS variable can be used to specify a list -of version numbers that should be taken into account when searching -for Python. You need to set this variable before calling -find_package(PythonLibs). - -If you'd like to specify the installation of Python to use, you should -modify the following cache variables: - -:: - - PYTHON_LIBRARY - path to the python library - PYTHON_INCLUDE_DIR - path to where Python.h is found - -If calling both ``find_package(PythonInterp)`` and -``find_package(PythonLibs)``, call ``find_package(PythonInterp)`` first to -get the currently active Python version by default with a consistent version -of PYTHON_LIBRARIES. -#]=======================================================================] - -# Use the executable's path as a hint -set(_Python_LIBRARY_PATH_HINT) -if(IS_ABSOLUTE "${Python_EXECUTABLE}") - if(WIN32) - get_filename_component(_Python_PREFIX "${Python_EXECUTABLE}" PATH) - if(_Python_PREFIX) - set(_Python_LIBRARY_PATH_HINT ${_Python_PREFIX}/libs) - endif() - unset(_Python_PREFIX) - else() - get_filename_component(_Python_PREFIX "${Python_EXECUTABLE}" PATH) - get_filename_component(_Python_PREFIX "${_Python_PREFIX}" PATH) - if(_Python_PREFIX) - set(_Python_LIBRARY_PATH_HINT ${_Python_PREFIX}/lib) - endif() - unset(_Python_PREFIX) - endif() -endif() - -include(CMakeFindFrameworks) -# Search for the python framework on Apple. -CMAKE_FIND_FRAMEWORKS(Python) - -# Save CMAKE_FIND_FRAMEWORK -if(DEFINED CMAKE_FIND_FRAMEWORK) - set(_PythonLibs_CMAKE_FIND_FRAMEWORK ${CMAKE_FIND_FRAMEWORK}) -else() - unset(_PythonLibs_CMAKE_FIND_FRAMEWORK) -endif() -# To avoid picking up the system Python.h pre-maturely. -set(CMAKE_FIND_FRAMEWORK LAST) - -set(_PYTHON1_VERSIONS 1.6 1.5) -set(_PYTHON2_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0) -set(_PYTHON3_VERSIONS 3.8 3.7 3.6 3.5 3.4 3.3 3.2 3.1 3.0) - -if(PythonLibs_FIND_VERSION) - if(PythonLibs_FIND_VERSION_COUNT GREATER 1) - set(_PYTHON_FIND_MAJ_MIN "${PythonLibs_FIND_VERSION_MAJOR}.${PythonLibs_FIND_VERSION_MINOR}") - unset(_PYTHON_FIND_OTHER_VERSIONS) - if(PythonLibs_FIND_VERSION_EXACT) - if(_PYTHON_FIND_MAJ_MIN STREQUAL PythonLibs_FIND_VERSION) - set(_PYTHON_FIND_OTHER_VERSIONS "${PythonLibs_FIND_VERSION}") - else() - set(_PYTHON_FIND_OTHER_VERSIONS "${PythonLibs_FIND_VERSION}" "${_PYTHON_FIND_MAJ_MIN}") - endif() - else() - foreach(_PYTHON_V ${_PYTHON${PythonLibs_FIND_VERSION_MAJOR}_VERSIONS}) - if(NOT _PYTHON_V VERSION_LESS _PYTHON_FIND_MAJ_MIN) - list(APPEND _PYTHON_FIND_OTHER_VERSIONS ${_PYTHON_V}) - endif() - endforeach() - endif() - unset(_PYTHON_FIND_MAJ_MIN) - else() - set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON${PythonLibs_FIND_VERSION_MAJOR}_VERSIONS}) - endif() -else() - set(_PYTHON_FIND_OTHER_VERSIONS ${_PYTHON3_VERSIONS} ${_PYTHON2_VERSIONS} ${_PYTHON1_VERSIONS}) -endif() - -# Set up the versions we know about, in the order we will search. Always add -# the user supplied additional versions to the front. -# If FindPythonInterp has already found the major and minor version, -# insert that version between the user supplied versions and the stock -# version list. -set(_Python_VERSIONS ${Python_ADDITIONAL_VERSIONS}) -if(DEFINED PYTHON_VERSION_MAJOR AND DEFINED PYTHON_VERSION_MINOR) - list(APPEND _Python_VERSIONS ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) -endif() -list(APPEND _Python_VERSIONS ${_PYTHON_FIND_OTHER_VERSIONS}) - -unset(_PYTHON_FIND_OTHER_VERSIONS) -unset(_PYTHON1_VERSIONS) -unset(_PYTHON2_VERSIONS) -unset(_PYTHON3_VERSIONS) - -# Python distribution: define which architectures can be used -if (CMAKE_SIZEOF_VOID_P) - # In this case, search only for 64bit or 32bit - math (EXPR _PYTHON_ARCH "${CMAKE_SIZEOF_VOID_P} * 8") - set (_PYTHON_ARCH2 _PYTHON_PREFIX_ARCH}) -else() - if (Python_EXECUTABLE) - # determine interpreter architecture - execute_process (COMMAND "${Python_EXECUTABLE}" -c "import sys; print(sys.maxsize > 2**32)" - RESULT_VARIABLE _PYTHON_RESULT - OUTPUT_VARIABLE _PYTHON_IS64BIT - ERROR_VARIABLE _PYTHON_IS64BIT) - if (NOT _PYTHON_RESULT) - if (_PYTHON_IS64BIT) - set (_PYTHON_ARCH 64) - set (_PYTHON_ARCH2 64) - else() - set (_PYTHON_ARCH 32) - set (_PYTHON_ARCH2 32) - endif() - endif() - else() - # architecture unknown, search for both 64bit and 32bit - set (_PYTHON_ARCH 64) - set (_PYTHON_ARCH2 32) - endif() -endif() - -foreach(_CURRENT_VERSION ${_Python_VERSIONS}) - string(REPLACE "." "" _CURRENT_VERSION_NO_DOTS ${_CURRENT_VERSION}) - if(WIN32) - find_library(PYTHON_DEBUG_LIBRARY - NAMES python${_CURRENT_VERSION_NO_DOTS}_d python - NAMES_PER_DIR - HINTS ${_Python_LIBRARY_PATH_HINT} - PATHS - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs/Debug - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs/Debug - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs/Debug - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs/Debug - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs/Debug - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs/Debug - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs - ) - endif() - - set(PYTHON_FRAMEWORK_LIBRARIES) - if(Python_FRAMEWORKS AND NOT PYTHON_LIBRARY) - foreach(dir ${Python_FRAMEWORKS}) - list(APPEND PYTHON_FRAMEWORK_LIBRARIES - ${dir}/Versions/${_CURRENT_VERSION}/lib) - endforeach() - endif() - # Use the library's install prefix as a hint - set(_Python_INCLUDE_PATH_HINT) - # PYTHON_LIBRARY may contain a list because of SelectLibraryConfigurations - # which may have been run previously. If it is the case, the list can be: - # optimized;;debug; - foreach(lib ${PYTHON_LIBRARY} ${PYTHON_DEBUG_LIBRARY}) - if(IS_ABSOLUTE "${lib}") - get_filename_component(_Python_PREFIX "${lib}" PATH) - get_filename_component(_Python_PREFIX "${_Python_PREFIX}" PATH) - if(_Python_PREFIX) - list(APPEND _Python_INCLUDE_PATH_HINT ${_Python_PREFIX}/include) - endif() - unset(_Python_PREFIX) - endif() - endforeach() - - # Add framework directories to the search paths - set(PYTHON_FRAMEWORK_INCLUDES) - if(Python_FRAMEWORKS AND NOT PYTHON_INCLUDE_DIR) - foreach(dir ${Python_FRAMEWORKS}) - list(APPEND PYTHON_FRAMEWORK_INCLUDES - ${dir}/Versions/${_CURRENT_VERSION}/include) - endforeach() - endif() - - find_path(PYTHON_INCLUDE_DIR - NAMES Python.h - HINTS - ${_Python_INCLUDE_PATH_HINT} - PATHS - ${PYTHON_FRAMEWORK_INCLUDES} - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/include - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/include - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/include - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/include - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/include - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/include - PATH_SUFFIXES - python${_CURRENT_VERSION}mu - python${_CURRENT_VERSION}m - python${_CURRENT_VERSION}u - python${_CURRENT_VERSION} - ) - - # For backward compatibility, set PYTHON_INCLUDE_PATH. - set(PYTHON_INCLUDE_PATH "${PYTHON_INCLUDE_DIR}") - - if(PYTHON_INCLUDE_DIR AND EXISTS "${PYTHON_INCLUDE_DIR}/patchlevel.h") - file(STRINGS "${PYTHON_INCLUDE_DIR}/patchlevel.h" python_version_str - REGEX "^#define[ \t]+PY_VERSION[ \t]+\"[^\"]+\"") - string(REGEX REPLACE "^#define[ \t]+PY_VERSION[ \t]+\"([^\"]+)\".*" "\\1" - PYTHONLIBS_VERSION_STRING "${python_version_str}") - unset(python_version_str) - endif() - - if(PYTHON_INCLUDE_DIR) - break() - endif() -endforeach() - -unset(_Python_INCLUDE_PATH_HINT) -unset(_Python_LIBRARY_PATH_HINT) - -mark_as_advanced( - PYTHON_INCLUDE_DIR -) - -# We use PYTHON_INCLUDE_DIR, PYTHON_LIBRARY and PYTHON_DEBUG_LIBRARY for the -# cache entries because they are meant to specify the location of a single -# library. We now set the variables listed by the documentation for this -# module. -set(Python_INCLUDE_DIRS "${PYTHON_INCLUDE_DIR}") -unset(PYTHON_FOUND) - -# Restore CMAKE_FIND_FRAMEWORK -if(DEFINED _PythonLibs_CMAKE_FIND_FRAMEWORK) - set(CMAKE_FIND_FRAMEWORK ${_PythonLibs_CMAKE_FIND_FRAMEWORK}) - unset(_PythonLibs_CMAKE_FIND_FRAMEWORK) -else() - unset(CMAKE_FIND_FRAMEWORK) -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonHeaders - REQUIRED_VARS Python_INCLUDE_DIRS - VERSION_VAR PYTHONLIBS_VERSION_STRING) \ No newline at end of file diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index ee7298389f..ad5fb793f5 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -3,7 +3,6 @@ project(psp) set(CMAKE_BUILD_TYPE "Release") - set(CMAKE_CXX_STANDARD 14) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -92,22 +91,11 @@ find_package(Color) option(CMAKE_BUILD_TYPE "Release/Debug build" RELEASE) option(PSP_WASM_BUILD "Build the WebAssembly Project" ON) -option(PSP_CPP_BUILD "Build the C++ Project" OFF) -option(PSP_PYTHON_BUILD "Build the Python Bindings" OFF) -option(PSP_CPP_BUILD_STRICT "Build the C++ with strict warnings" OFF) -option(PSP_BUILD_DOCS "Build the Perspective documentation" OFF) if (NOT DEFINED PSP_WASM_BUILD) set(PSP_WASM_BUILD ON) - set(PSP_CPP_BUILD OFF) - set(PSP_PYTHON_BUILD OFF) -endif() - -if (PSP_WASM_BUILD AND PSP_CPP_BUILD) - message(FATAL_ERROR "${Red}CPP and Emscripten builds must be done separately${ColorReset}") endif() - if(DEFINED ENV{PSP_DEBUG}) set(CMAKE_BUILD_TYPE DEBUG) else() @@ -116,60 +104,12 @@ else() endif() endif() -if(DEFINED ENV{PSP_MANYLINUX}) - set(MANYLINUX ON) -else() - set(MANYLINUX OFF) -endif() - -if (NOT DEFINED PSP_CPP_BUILD) - set(PSP_CPP_BUILD ON) -endif() - -if (NOT DEFINED PSP_PYTHON_BUILD) - set(PSP_PYTHON_BUILD OFF) -elseif(PSP_PYTHON_BUILD) - # PSP_PYTHON_BUILD will always run PSP_CPP_BUILD - set(PSP_CPP_BUILD ON) - if(NOT DEFINED PSP_PYTHON_VERSION) - set(PSP_PYTHON_VERSION 3.7) - endif() -endif() - -if (NOT DEFINED PSP_CPP_BUILD_STRICT) - set(PSP_CPP_BUILD_STRICT OFF) -endif() - set(BUILD_MESSAGE "") -if(PSP_WASM_BUILD) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building WASM binding${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping WASM binding${ColorReset}") -endif() +set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building WASM binding${ColorReset}") if(NOT DEFINED PSP_CPP_SRC) set(PSP_CPP_SRC "${CMAKE_SOURCE_DIR}") endif() -if(PSP_CPP_BUILD) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ binding${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping C++ binding${ColorReset}") -endif() - -if (PSP_PYTHON_BUILD) - if(NOT DEFINED PSP_PYTHON_SRC) - set(PSP_PYTHON_SRC "${CMAKE_SOURCE_DIR}/../../python/perspective/perspective") - endif() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building Python ${Red}${PSP_PYTHON_VERSION}${Cyan} binding${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping Python binding${ColorReset}") -endif() - -if (PSP_CPP_BUILD AND NOT PSP_CPP_BUILD_STRICT) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Building C++ without strict warnings${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ with strict warnings${ColorReset}") -endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LOWER ) if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) @@ -179,12 +119,6 @@ else() set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building RELEASE${ColorReset}") endif() -if(PSP_BUILD_DOCS) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building Perspective Documentation${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Skipping Perspective Documentation${ColorReset}") -endif() - message(WARNING "${BUILD_MESSAGE}") ####################### @@ -195,248 +129,88 @@ if(NOT WIN32) set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") endif() -if (PSP_WASM_BUILD) - #################### - # EMSCRIPTEN BUILD # - #################### - execute_process(COMMAND which emcc OUTPUT_VARIABLE EMCC) - execute_process(COMMAND which em++ OUTPUT_VARIABLE EMPP) - string(STRIP ${EMCC} EMCC) - string(STRIP ${EMPP} EMPP) - set(CMAKE_C_COMPILER ${EMCC}) - set(CMAKE_CXX_COMPILER ${EMPP}) - set(CMAKE_TOOLCHAIN_FILE "$ENV{EMSCRIPTEN_ROOT}/cmake/Modules/Platform/Emscripten.cmake") - set(CMAKE_AR emar) - set(CMAKE_RANLIB emranlib) - set(CMAKE_EXECUTABLE_SUFFIX ".js") - list(APPEND CMAKE_PREFIX_PATH /usr/local) - - # Assumes that Boost includes will be in this folder. - include_directories("/usr/local/include" SYSTEM) - - # Include this docker-only directory. - include_directories("/boost_includes") - - # Build Rapidjson as it is used in the minimal arrow to be built later. - psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in") - - set(EXTENDED_FLAGS " \ - --bind \ - --source-map-base ./build/ \ - --memory-init-file 0 \ - -Wall \ - -s NO_EXIT_RUNTIME=1 \ - -s NO_FILESYSTEM=1 \ - -s ALLOW_MEMORY_GROWTH=1 \ - -s MODULARIZE=1 \ - -s EXPORT_NAME=\"load_perspective\" \ - -s EXPORT_ES6=1 \ - -s MAXIMUM_MEMORY=-1 \ - -s USE_ES6_IMPORT_META=0 \ - -s EXPORTED_FUNCTIONS=\"['_main']\" \ - ") - - if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(OPT_FLAGS " \ - -O0 \ - -g4 \ - --profiling \ - -s WARN_UNALIGNED=1 \ - -Wcast-align \ - -Wover-aligned \ - -s DISABLE_EXCEPTION_CATCHING=0 \ - -s ASSERTIONS=2 \ - -s DEMANGLE_SUPPORT=1 \ - ") - else() - set(OPT_FLAGS " \ - -O3 \ - -g0 \ - -flto \ - --closure 1 \ - -s AGGRESSIVE_VARIABLE_ELIMINATION=1 \ - ") - endif() - - set(ASYNC_MODE_FLAGS "-s -s BINARYEN_ASYNC_COMPILATION=1 -s WASM=1") -elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) - if(WIN32) - if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(OPT_FLAGS " \ - /DEBUG \ - /Z7 \ - /Zi - ") - else() - set(OPT_FLAGS " \ - /NDEBUG \ - /O2 \ - ") - endif() - else() - if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(OPT_FLAGS " \ - -O0 \ - -g3 \ - ") - else() - set(OPT_FLAGS " \ - -O3 \ - -g0 \ - ") - endif() - endif() - set(SYNC_MODE_FLAGS "") - set(ASYNC_MODE_FLAGS "") - set(ASMJS_MODE_FLAGS "") - - if (WIN32) - # BOOST_ROOT has been removed on Windows VMs in Azure: - # - # - https://github.com/actions/virtual-environments/issues/687 - # - https://github.com/actions/virtual-environments/issues/319 - # - # `BOOST_ROOT` must be set in the environment, and policy CMP0074 - # must be set to `NEW` to allow BOOST_ROOT to be defined by env var - cmake_policy(SET CMP0074 NEW) - - if(DEFINED ENV{BOOST_ROOT}) - set(Boost_NO_BOOST_CMAKE TRUE) - - message(WARNING "${Cyan}BOOST_ROOT: $ENV{BOOST_ROOT} ${ColorReset}") - message(WARNING "${Cyan}BOOST_INCLUDEDIR: $ENV{BOOST_INCLUDEDIR} ${ColorReset}") - message(WARNING "${Cyan}BOOST_LIBRARYDIR: $ENV{BOOST_LIBRARYDIR} ${ColorReset}") - endif() - endif() - - find_package(Boost REQUIRED) - if(NOT Boost_FOUND) - message(FATAL_ERROR "${Red}Boost could not be located${ColorReset}") - else() - message(WARNING "${Cyan}Found Boost: `Boost_INCLUDE_DIRS`: ${Boost_INCLUDE_DIRS}, `Boost_LIBRARY_DIRS` - ${Boost_LIBRARY_DIRS} ${ColorReset}") - include_directories( ${Boost_INCLUDE_DIRS} ) - endif() - - find_package(TBB) - if(NOT TBB_FOUND) - message("${Red}TBB could not be located - building TBB from external source ${ColorReset}") - psp_build_dep("tbb" "${PSP_CMAKE_MODULE_PATH}/TBB.txt.in") - else() - message("${Cyan}Found TBB: TBB_INCLUDE_DIRS - ${TBB_INCLUDE_DIRS}, TBB_LIBRARY_DIRS - ${TBB_LIBRARY_DIRS} ${ColorReset}") - include_directories( ${TBB_INCLUDE_DIRS} ) - endif() - - if(WIN32) - foreach(warning 4244 4251 4267 4275 4290 4786 4305 4996) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd${warning}") - endforeach(warning) - # When linking against Boost on Azure, bcrypt fails to link: - # - https://github.com/microsoft/vcpkg/issues/4481 - # - https://github.com/boostorg/uuid/issues/68 - #add_definitions("-DBOOST_UUID_FORCE_AUTO_LINK") - else() - include_directories("/usr/local/include") - endif() - - if(PSP_PYTHON_BUILD) - ######################### - # PYTHON BINDINGS BUILD # - ######################### - include_directories("${PSP_PYTHON_SRC}/perspective/include") - - # Set CMP0094 to NEW - find the first version that matches constraints, - # instead of the latest version installed. - cmake_policy(SET CMP0094 NEW) - - if(MANYLINUX) - # Manylinux docker images have no shared libraries - # The instead use a statically built python. - # Cmake's default FindPython can't find the python headers - # without also finding (or failing to find) the python libraries - # so we use a custom FindPythonHeaders that is the same as the - # default, but ignores when the python libraries can't be found. - message("${Red}Manylinux build has no python shared libraries${ColorReset}") - find_package( Python ${PSP_PYTHON_VERSION} REQUIRED COMPONENTS Interpreter ) - find_package( PythonHeaders ${PSP_PYTHON_VERSION} REQUIRED) - else() - message("${Cyan}Use python shared libraries${ColorReset}") - find_package( Python ${PSP_PYTHON_VERSION} REQUIRED COMPONENTS Interpreter Development) - link_directories( ${Python_LIBRARY_DIRS} ) - endif() - message("${Cyan}Using Python_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}, Python_LIBRARIES: ${Python_LIBRARIES} ${ColorReset}") - include_directories( ${Python_INCLUDE_DIRS} ) - - if(MACOS) - # on mac, use the vanilla pybind11 finder - find_package(pybind11) - if(pybind11_FOUND) - # Found PyBind installed by Homebrew - set(PYTHON_PYBIND_FOUND pybind11_FOUND) - else() - # Check if pip installed PyBind is available - find_package(Pybind) - if(PYTHON_PYBIND_FOUND) - # Need to add extra flags due to pybind weirness - # https://github.com/pybind/pybind11/blob/7830e8509f2adc97ce9ee32bf99cd4b82089cc4c/tools/pybind11Tools.cmake#L103 - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") - endif() - endif() - - else() - # On Linux/Docker, look for pip installed PyBind only - find_package(Pybind) - endif() - - if(NOT PYTHON_PYBIND_FOUND) - message("${Red}PyBind11 could not be located - building from external source${ColorReset}") - psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in") - else() - message("${Cyan}Found PyBind11 in ${PYTHON_PYBIND_INCLUDE_DIR}${ColorReset}") - include_directories( ${PYTHON_PYBIND_INCLUDE_DIR} ) - endif() - - find_package(NumPy) - if(NOT PYTHON_NUMPY_FOUND) - message(FATAL_ERROR "${Red}Numpy could not be located${ColorReset}") - else() - message(WARNING "${Cyan}Numpy found: ${PYTHON_NUMPY_INCLUDE_DIR}${ColorReset}") - include_directories( ${PYTHON_NUMPY_INCLUDE_DIR}) - endif() - - find_package(PyArrow REQUIRED) +#################### +# EMSCRIPTEN BUILD # +#################### +execute_process(COMMAND which emcc OUTPUT_VARIABLE EMCC) +execute_process(COMMAND which em++ OUTPUT_VARIABLE EMPP) +string(STRIP ${EMCC} EMCC) +string(STRIP ${EMPP} EMPP) +set(CMAKE_C_COMPILER ${EMCC}) +set(CMAKE_CXX_COMPILER ${EMPP}) +set(CMAKE_TOOLCHAIN_FILE "$ENV{EMSCRIPTEN_ROOT}/cmake/Modules/Platform/Emscripten.cmake") +set(CMAKE_AR emar) +set(CMAKE_RANLIB emranlib) +set(CMAKE_EXECUTABLE_SUFFIX ".js") +list(APPEND CMAKE_PREFIX_PATH /usr/local) + +# Assumes that Boost includes will be in this folder. +include_directories("/usr/local/include" SYSTEM) + +# Include this docker-only directory. +include_directories("/boost_includes") + +# Build Rapidjson as it is used in the minimal arrow to be built later. +psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in") + +set(EXTENDED_FLAGS " \ + --bind \ + --source-map-base ./build/ \ + --memory-init-file 0 \ + -Wall \ + -s NO_EXIT_RUNTIME=1 \ + -s NO_FILESYSTEM=1 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s MODULARIZE=1 \ + -s EXPORT_NAME=\"load_perspective\" \ + -s EXPORT_ES6=1 \ + -s MAXIMUM_MEMORY=-1 \ + -s USE_ES6_IMPORT_META=0 \ + -s EXPORTED_FUNCTIONS=\"['_main']\" \ + ") - if(NOT PYTHON_PYARROW_FOUND) - message(FATAL_ERROR "${Red}PyArrow could not be located${ColorReset}") - else() - message(WARNING "${Cyan}PyArrow found: PYTHON_PYARROW_INCLUDE_DIR - ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}") - message(WARNING "${Cyan}Using pre-built ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY} from: ${PYTHON_PYARROW_LIBRARY_DIR}${ColorReset}") - include_directories(${PYTHON_PYARROW_INCLUDE_DIR}) - link_directories(${PYTHON_PYARROW_LIBRARY_DIR}) - endif() - ##################### - endif() +if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) + set(OPT_FLAGS " \ + -O0 \ + -g4 \ + --profiling \ + -s WARN_UNALIGNED=1 \ + -Wcast-align \ + -Wover-aligned \ + -s DISABLE_EXCEPTION_CATCHING=0 \ + -s ASSERTIONS=2 \ + -s DEMANGLE_SUPPORT=1 \ + ") +else() + set(OPT_FLAGS " \ + -O3 \ + -g0 \ + -flto \ + --closure 1 \ + -s AGGRESSIVE_VARIABLE_ELIMINATION=1 \ + ") endif() +set(ASYNC_MODE_FLAGS "-s -s BINARYEN_ASYNC_COMPILATION=1 -s WASM=1") + # Build header-only dependencies from external source psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in") psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in") psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in") -# For WASM/CPP build, build minimal arrow from source -if (NOT PSP_PYTHON_BUILD) - # build arrow + dependencies from source for Emscripten and C++ - message("${Cyan}Building minimal Apache Arrow${ColorReset}") +# For WASM build, build minimal arrow from source +# build arrow + dependencies from source for Emscripten and C++ +message("${Cyan}Building minimal Apache Arrow${ColorReset}") - psp_build_dep("double-conversion" "${PSP_CMAKE_MODULE_PATH}/double-conversion.txt.in") - psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in") +psp_build_dep("double-conversion" "${PSP_CMAKE_MODULE_PATH}/double-conversion.txt.in") +psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in") - find_package(Flatbuffers) - if(NOT FLATBUFFERS_FOUND) - message(FATAL_ERROR"${Red}Flatbuffers could not be located${ColorReset}") - else() - message("${Cyan}Found Flatbuffers in ${FLATBUFFERS_INCLUDE_DIR}${ColorReset}") - include_directories( ${FLATBUFFERS_INCLUDE_DIR} ) - endif() +find_package(Flatbuffers) +if(NOT FLATBUFFERS_FOUND) + message(FATAL_ERROR"${Red}Flatbuffers could not be located${ColorReset}") +else() + message("${Cyan}Found Flatbuffers in ${FLATBUFFERS_INCLUDE_DIR}${ColorReset}") + include_directories( ${FLATBUFFERS_INCLUDE_DIR} ) endif() ##################### @@ -538,22 +312,6 @@ set (SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/vocab.cpp ) -set(PYTHON_SOURCE_FILES ${SOURCE_FILES} - ${PSP_PYTHON_SRC}/src/column.cpp - ) - -set (PYTHON_BINDING_SOURCE_FILES - ${PSP_PYTHON_SRC}/src/accessor.cpp - ${PSP_PYTHON_SRC}/src/computed.cpp - ${PSP_PYTHON_SRC}/src/context.cpp - ${PSP_PYTHON_SRC}/src/fill.cpp - ${PSP_PYTHON_SRC}/src/numpy.cpp - ${PSP_PYTHON_SRC}/src/python.cpp - ${PSP_PYTHON_SRC}/src/serialization.cpp - ${PSP_PYTHON_SRC}/src/table.cpp - ${PSP_PYTHON_SRC}/src/utils.cpp - ${PSP_PYTHON_SRC}/src/view.cpp - ) if (WIN32) set(CMAKE_CXX_FLAGS " /EHsc /MP") @@ -561,146 +319,14 @@ else() set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS}") endif() -if (PSP_WASM_BUILD) - add_library(psp ${SOURCE_FILES}) - target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1) - set_target_properties(psp PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") - target_link_libraries(psp arrow) - - add_executable(perspective.async src/cpp/emscripten.cpp) - target_link_libraries(perspective.async psp "${ASYNC_MODE_FLAGS}") - target_compile_definitions(perspective.async PRIVATE PSP_ENABLE_WASM=1) - set_target_properties(perspective.async PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") - set_target_properties(perspective.async PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./build/") - set_target_properties(perspective.async PROPERTIES OUTPUT_NAME "psp.async") -elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) - if(NOT WIN32) - set(CMAKE_SHARED_LIBRARY_SUFFIX .so) - # Look for the binary using @loader_path (relative to binary location) instead of @rpath - # and include arrow in @rpath so it can be found by libbinding/libpsp - set(CMAKE_MACOSX_RPATH TRUE) - set(CMAKE_SKIP_BUILD_RPATH FALSE) - set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - set(CMAKE_INSTALL_NAME_DIR "@rpath/") - - # module_origin_path is the location of the binary - if(MACOS) - set(module_origin_path "@loader_path/") - else() - set(module_origin_path "\$ORIGIN") - endif() - else() - set(CMAKE_SHARED_LIBRARY_PREFIX lib) - endif() - - if(PSP_PYTHON_BUILD) - ######################## - # Python extra targets # - ######################## - add_library(psp SHARED ${PYTHON_SOURCE_FILES}) - add_library(binding SHARED ${PYTHON_BINDING_SOURCE_FILES}) - - include_directories(${PSP_PYTHON_SRC}/include) - - target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1) - target_compile_definitions(binding PRIVATE PSP_ENABLE_PYTHON=1) - - if (WIN32) - target_compile_definitions(binding PRIVATE WIN32=1) - target_compile_definitions(binding PRIVATE _WIN32=1) - - # .dll not importable - set_property(TARGET binding PROPERTY SUFFIX .pyd) - else() - target_compile_options(binding PRIVATE -Wdeprecated-declarations) - # Add a relative path to search for PyArrow - when Perspective is - # installed from a wheel, PyArrow may not be in the same directory - # as the PyArrow which was used to build the wheel. Assuming that - # both Pyarrow and Perspective are installed in `site-packages`, - # the relative search path should be able to pick up pyarrow. - set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR}) - set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR}) - endif() - - target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES}) - target_link_libraries(binding ${PYTHON_PYARROW_LIBRARIES}) - - if(WIN32) - # Don't link - else() - target_link_libraries(psp ${PYTHON_LIBRARIES}) - target_link_libraries(binding ${PYTHON_LIBRARIES}) - endif() - - target_link_libraries(psp tbb) - target_link_libraries(binding tbb) - - target_link_libraries(binding psp) - - # The compiled libraries will be put in CMAKE_LIBRARY_OUTPUT_DIRECTORY by default. In the - # setup.py file, we designate this to be in the build/lib. directory. However, - # since we want to be able to test perspective in-source, we also copy the libraries into - # the source folder. These two commands do that. - add_custom_command(TARGET psp POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) - add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) - - if(WIN32) - # inline arrow dlls - file(GLOB ARROW_DLLS "${PYTHON_PYARROW_LIBRARY_DIR}/*.dll") - - add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_DLLS} ${PSP_PYTHON_SRC}/table/) - if(NOT TBB_FOUND) - add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) - endif() - endif() - ######################## - else() - add_library(psp SHARED ${SOURCE_FILES}) - - # Link perspective against custom-built minimal arrow - target_link_libraries(psp arrow) - endif() - - if(PSP_CPP_BUILD_STRICT AND NOT WIN32) - target_compile_options(psp PRIVATE -Wall -Werror) - target_compile_options(psp PRIVATE $<$:-fPIC -O0>) - if(PSP_PYTHON_BUILD) - target_compile_options(binding PRIVATE $<$:-fPIC -O0>) - endif() - elseif(WIN32) - target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1) - target_compile_definitions(psp PRIVATE WIN32=1) - target_compile_definitions(psp PRIVATE _WIN32=1) - endif() -endif() - -######## -# Docs # -######## -if(PSP_BUILD_DOCS) - add_custom_target(doxygen) - add_custom_command(TARGET doxygen - COMMAND doxygen doxygen.conf - WORKING_DIRECTORY ../../docs - COMMENT "Build doxygen xml files used by sphinx/breathe." - ) - - add_custom_target(docs-html ALL) - add_custom_command(TARGET docs-html - COMMAND sphinx-build -b html . build/html - WORKING_DIRECTORY ../../docs - COMMENT "Build html documentation" - ) - add_dependencies(docs-html doxygen) - - # add_custom_target(docs-markdown ALL) - # add_custom_command(TARGET docs-html - # COMMAND sphinx-build -M markdown . build/ - # WORKING_DIRECTORY ../../docs - # COMMENT "Build markdown documentation" - # ) - # add_dependencies(docs-html doxygen) - -endif() -########## +add_library(psp ${SOURCE_FILES}) +target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1) +set_target_properties(psp PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") +target_link_libraries(psp arrow) + +add_executable(perspective.async src/cpp/emscripten.cpp) +target_link_libraries(perspective.async psp "${ASYNC_MODE_FLAGS}") +target_compile_definitions(perspective.async PRIVATE PSP_ENABLE_WASM=1) +set_target_properties(perspective.async PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") +set_target_properties(perspective.async PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./build/") +set_target_properties(perspective.async PROPERTIES OUTPUT_NAME "psp.async") \ No newline at end of file diff --git a/python/perspective/setup.py b/python/perspective/setup.py index d0bbda195e..23ad68b7ad 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -33,6 +33,22 @@ import multiprocessing CPU_COUNT = multiprocessing.cpu_count() +try: + import numpy + numpy_includes = numpy.get_include() +except ImportError: + print('Must install numpy prior to installing perspective!') + raise + +try: + import pyarrow + pyarrow_includes = pyarrow.get_include() + pyarrow_library_dirs = pyarrow.get_library_dirs() + pyarrow_libraries = pyarrow.get_libraries() +except ImportError: + print('Must install pyarrow prior to installing perspective!') + raise + here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: @@ -84,91 +100,142 @@ def get_version(file, name='__version__'): version = get_version(os.path.join(here, 'perspective', 'core', '_version.py')) -class PSPExtension(Extension): - def __init__(self, name, sourcedir='dist'): - Extension.__init__(self, name, sources=[]) - self.sourcedir = os.path.abspath(sourcedir) - +sources = [ + 'dist/src/cpp/aggregate.cpp', + 'dist/src/cpp/aggspec.cpp', + 'dist/src/cpp/arg_sort.cpp', + 'dist/src/cpp/arrow_loader.cpp', + 'dist/src/cpp/arrow_writer.cpp', + 'dist/src/cpp/base.cpp', + 'dist/src/cpp/base_impl_linux.cpp', + 'dist/src/cpp/base_impl_osx.cpp', + 'dist/src/cpp/base_impl_wasm.cpp', + 'dist/src/cpp/base_impl_win.cpp', + 'dist/src/cpp/binding.cpp', + 'dist/src/cpp/build_filter.cpp', + 'dist/src/cpp/column.cpp', + 'dist/src/cpp/comparators.cpp', + 'dist/src/cpp/compat.cpp', + 'dist/src/cpp/compat_impl_linux.cpp', + 'dist/src/cpp/compat_impl_osx.cpp', + 'dist/src/cpp/compat_impl_win.cpp', + 'dist/src/cpp/computed.cpp', + 'dist/src/cpp/computed_column_map.cpp', + 'dist/src/cpp/computed_function.cpp', + 'dist/src/cpp/config.cpp', + 'dist/src/cpp/context_base.cpp', + 'dist/src/cpp/context_grouped_pkey.cpp', + 'dist/src/cpp/context_handle.cpp', + 'dist/src/cpp/context_one.cpp', + 'dist/src/cpp/context_two.cpp', + 'dist/src/cpp/context_zero.cpp', + 'dist/src/cpp/custom_column.cpp', + 'dist/src/cpp/data.cpp', + 'dist/src/cpp/data_slice.cpp', + 'dist/src/cpp/data_table.cpp', + 'dist/src/cpp/date.cpp', + 'dist/src/cpp/dense_nodes.cpp', + 'dist/src/cpp/dense_tree_context.cpp', + 'dist/src/cpp/dense_tree.cpp', + 'dist/src/cpp/dependency.cpp', + 'dist/src/cpp/extract_aggregate.cpp', + 'dist/src/cpp/filter.cpp', + 'dist/src/cpp/flat_traversal.cpp', + 'dist/src/cpp/get_data_extents.cpp', + 'dist/src/cpp/gnode.cpp', + 'dist/src/cpp/gnode_state.cpp', + 'dist/src/cpp/histogram.cpp', + 'dist/src/cpp/logtime.cpp', + 'dist/src/cpp/mask.cpp', + 'dist/src/cpp/min_max.cpp', + 'dist/src/cpp/multi_sort.cpp', + 'dist/src/cpp/none.cpp', + 'dist/src/cpp/path.cpp', + 'dist/src/cpp/pivot.cpp', + 'dist/src/cpp/pool.cpp', + 'dist/src/cpp/port.cpp', + 'dist/src/cpp/process_state.cpp', + 'dist/src/cpp/raii.cpp', + 'dist/src/cpp/raii_impl_linux.cpp', + 'dist/src/cpp/raii_impl_osx.cpp', + 'dist/src/cpp/raii_impl_win.cpp', + 'dist/src/cpp/range.cpp', + 'dist/src/cpp/rlookup.cpp', + 'dist/src/cpp/scalar.cpp', + 'dist/src/cpp/schema_column.cpp', + 'dist/src/cpp/schema.cpp', + 'dist/src/cpp/slice.cpp', + 'dist/src/cpp/sort_specification.cpp', + 'dist/src/cpp/sparse_tree.cpp', + 'dist/src/cpp/sparse_tree_node.cpp', + 'dist/src/cpp/step_delta.cpp', + 'dist/src/cpp/storage.cpp', + 'dist/src/cpp/storage_impl_linux.cpp', + 'dist/src/cpp/storage_impl_osx.cpp', + 'dist/src/cpp/storage_impl_win.cpp', + 'dist/src/cpp/sym_table.cpp', + 'dist/src/cpp/table.cpp', + 'dist/src/cpp/time.cpp', + 'dist/src/cpp/traversal.cpp', + 'dist/src/cpp/traversal_nodes.cpp', + 'dist/src/cpp/tree_context_common.cpp', + 'dist/src/cpp/utils.cpp', + 'dist/src/cpp/update_task.cpp', + 'dist/src/cpp/view.cpp', + 'dist/src/cpp/view_config.cpp', + 'dist/src/cpp/vocab.cpp', + + # python-specific files + 'perspective/src/column.cpp', +] -class PSPBuild(build_ext): - def run(self): - self.run_cmake() - - def run_cmake(self): - self.cmake_cmd = which('cmake') - try: - out = subprocess.check_output([self.cmake_cmd, '--version']) - except OSError: - raise RuntimeError( - "CMake must be installed to build the following extensions: " + - ", ".join(e.name for e in self.extensions)) - - if platform.system() == "Windows": - cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', - out.decode()).group(1)) - if cmake_version < '3.1.0': - raise RuntimeError("CMake >= 3.1.0 is required on Windows") - - for ext in self.extensions: - self.build_extension_cmake(ext) - - def build_extension_cmake(self, ext): - extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) - cfg = 'Debug' if self.debug else 'Release' - - PYTHON_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor) - - cmake_args = [ - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + os.path.abspath(os.path.join(extdir, 'perspective', 'table')).replace('\\', '/'), - '-DCMAKE_BUILD_TYPE=' + cfg, - '-DPSP_CPP_BUILD=1', - '-DPSP_WASM_BUILD=0', - '-DPSP_PYTHON_BUILD=1', - '-DPSP_PYTHON_VERSION={}'.format(PYTHON_VERSION), - '-DPython_ADDITIONAL_VERSIONS={}'.format(PYTHON_VERSION), - '-DPython_FIND_VERSION={}'.format(PYTHON_VERSION), - '-DPython_EXECUTABLE={}'.format(sys.executable).replace('\\', '/'), - '-DPython_ROOT_DIR={}'.format(sysconfig.PREFIX).replace('\\', '/'), - '-DPython_ROOT={}'.format(sysconfig.PREFIX).replace('\\', '/'), - '-DPSP_CMAKE_MODULE_PATH={folder}'.format(folder=os.path.join(ext.sourcedir, 'cmake')).replace('\\', '/'), - '-DPSP_CPP_SRC={folder}'.format(folder=ext.sourcedir).replace('\\', '/'), - '-DPSP_PYTHON_SRC={folder}'.format(folder=os.path.join(ext.sourcedir, "..", 'perspective').replace('\\', '/')) - ] - - build_args = ['--config', cfg] - - if platform.system() == "Windows": - import distutils.msvccompiler as dm - msvc = {'12': 'Visual Studio 12 2013', - '14': 'Visual Studio 14 2015', - '14.1': 'Visual Studio 15 2017'}.get(dm.get_build_version(), 'Visual Studio 15 2017') - - cmake_args.extend([ - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format( - cfg.upper(), - extdir).replace('\\', '/'), - '-G', os.environ.get('PSP_GENERATOR', msvc)]) - - if sys.maxsize > 2**32: - # build 64 bit to match python - cmake_args += ['-A', 'x64'] - - build_args += ['--', '/m:{}'.format(CPU_COUNT), '/p:Configuration={}'.format(cfg)] - else: - cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] - build_args += ['--', '-j2' if os.environ.get('DOCKER', '') else '-j{}'.format(CPU_COUNT)] - - env = os.environ.copy() - env['PSP_ENABLE_PYTHON'] = '1' - env['OSX_DEPLOYMENT_TARGET'] = '10.9' - env["PYTHONPATH"] = os.path.sep.join((os.environ.get('PYTHONPATH', ''), os.path.pathsep.join((os.path.join(os.path.dirname(os.__file__), 'site-packages'), os.path.dirname(os.__file__))))) - - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - subprocess.check_call([self.cmake_cmd, os.path.abspath(ext.sourcedir)] + cmake_args, cwd=self.build_temp, env=env, stderr=subprocess.STDOUT) - subprocess.check_call([self.cmake_cmd, '--build', '.'] + build_args, cwd=self.build_temp, env=env, stderr=subprocess.STDOUT) - print() # Add an empty line for cleaner output +binding_sources = [ + 'perspective/src/accessor.cpp', + 'perspective/src/computed.cpp', + 'perspective/src/context.cpp', + 'perspective/src/fill.cpp', + 'perspective/src/numpy.cpp', + 'perspective/src/python.cpp', + 'perspective/src/serialization.cpp', + 'perspective/src/table.cpp', + 'perspective/src/utils.cpp', + 'perspective/src/view.cpp', +] +extensions = [ + Extension('perspective.table.libpsp', + define_macros=[('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))], + include_dirs=[ + 'perspective/include/', + 'dist/src/include/', + 'dist/third/date/include/', + 'dist/third/hopscotch/include/', + 'dist/third/ordered-map/include/', + 'dist/third/pybind11/include/', + numpy_includes, + pyarrow_includes, + ], + libraries=pyarrow_libraries, + library_dirs=pyarrow_library_dirs, + extra_compile_args=['-std=c++1y'] if os.name != 'nt' else ['/std:c++14'], + sources=sources), + Extension('perspective.table.libbinding', + define_macros=[('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))], + include_dirs=[ + 'perspective/include/', + 'dist/src/include/', + 'dist/third/date/include/', + 'dist/third/hopscotch/include/', + 'dist/third/ordered-map/include/', + 'dist/third/pybind11/include/', + numpy_includes, + pyarrow_includes, + ], + libraries=pyarrow_libraries, + library_dirs=pyarrow_library_dirs, + extra_compile_args=['-std=c++1y'] if os.name != 'nt' else ['/std:c++14'], + sources=binding_sources) +] class PSPCheckSDist(sdist): def run(self): @@ -176,10 +243,10 @@ def run(self): super(PSPCheckSDist, self).run() def run_check(self): - for file in ('CMakeLists.txt', 'cmake', 'src'): + for file in ('src', 'third'): path = os.path.abspath(os.path.join(here, 'dist', file)) if not os.path.exists(path): - raise Exception("Path is missing! {}\nMust run `yarn build_python` before building sdist so cmake files are installed".format(path)) + raise Exception("Path is missing! {}\nMust run `yarn build_python` before building sdist so c++ files are installed".format(path)) setup( @@ -210,6 +277,6 @@ def run_check(self): extras_require={ 'dev': requires_dev, }, - ext_modules=[PSPExtension('perspective')], - cmdclass=dict(build_ext=PSPBuild, sdist=PSPCheckSDist) + ext_modules=extensions, + cmdclass=dict(sdist=PSPCheckSDist) ) diff --git a/scripts/build_python.js b/scripts/build_python.js index b4262845e8..21b038f254 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -9,6 +9,7 @@ const {execute, execute_throw, docker, clean, resolve, getarg, bash, python_image} = require("./script_utils.js"); const fs = require("fs-extra"); +const rimraf = require("rimraf"); const IS_PY2 = getarg("--python2"); @@ -39,17 +40,31 @@ try { try { const dist = resolve`${__dirname}/../python/perspective/dist`; + const third = resolve`${__dirname}/../cpp/perspective/third`; const cpp = resolve`${__dirname}/../cpp/perspective`; const lic = resolve`${__dirname}/../LICENSE`; - const cmake = resolve`${__dirname}/../cmake`; - const dcmake = resolve`${dist}/cmake`; const dlic = resolve`${dist}/LICENSE`; const obj = resolve`${dist}/obj`; + // clone third party deps + if (!fs.existsSync(third)) { + console.log("Cloning third party dependencies"); + fs.mkdirpSync(third); + execute`git clone https://github.com/HowardHinnant/date.git ${third}/date`; + execute`git clone https://github.com/Tessil/hopscotch-map.git ${third}/hopscotch`; + execute`git clone https://github.com/Tessil/ordered-map.git ${third}/ordered-map`; + execute`git clone https://github.com/pybind/pybind11.git ${third}/pybind11`; + + rimraf.sync(`${third}/date/.git`); + rimraf.sync(`${third}/hopscotch/.git`); + rimraf.sync(`${third}/ordered-map/.git`); + rimraf.sync(`${third}/pybind11/.git`); + console.log("Cloning third party dependencies...done!"); + } + fs.mkdirpSync(dist); fs.copySync(cpp, dist, {preserveTimestamps: true}); fs.copySync(lic, dlic, {preserveTimestamps: true}); - fs.copySync(cmake, dcmake, {preserveTimestamps: true}); clean(obj); let cmd; diff --git a/scripts/clean.js b/scripts/clean.js index b2f8653deb..014e78941b 100644 --- a/scripts/clean.js +++ b/scripts/clean.js @@ -30,7 +30,7 @@ try { clean`cpp/perspective/obj`; } if (process.env.PSP_PROJECT === "python") { - clean("cpp/perspective/obj", "cpp/perspective/cppbuild", "python/perspective/dist", "python/perspective/build", "python/perspective/perspective_python.egg-info"); + clean("cpp/perspective/obj", "cpp/perspective/cppbuild", "cpp/perspective/third", "python/perspective/dist", "python/perspective/build", "python/perspective/perspective_python.egg-info"); return; } if (!IS_SCREENSHOTS && (!process.env.PACKAGE || minimatch("perspective", process.env.PACKAGE))) { From 1d7a2fc3827514d16436c671e99bb05473238196 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 10 Jul 2020 16:37:16 -0400 Subject: [PATCH 02/31] build inplace and choco install tbb --- azure-pipelines.yml | 8 ++++++-- scripts/build_python.js | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f69f7a4ba4..a013ad93fa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -163,6 +163,10 @@ jobs: inputs: versionSpec: '12.x' + - script: | + choco install tbb + displayName: "System deps" + - script: | which python > python.txt set /p PYTHON= Date: Fri, 10 Jul 2020 17:38:16 -0400 Subject: [PATCH 03/31] working on rpath stuff --- python/perspective/pyproject.toml | 2 +- python/perspective/setup.py | 257 +++++++++++++++--------------- scripts/build_python.js | 1 + 3 files changed, 128 insertions(+), 132 deletions(-) diff --git a/python/perspective/pyproject.toml b/python/perspective/pyproject.toml index 4114696901..a201cf948b 100644 --- a/python/perspective/pyproject.toml +++ b/python/perspective/pyproject.toml @@ -1,3 +1,3 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel"] +requires = ["setuptools", "wheel", "numpy", "pyarrow>=0.15.1"] diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 23ad68b7ad..ab4bde9749 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -7,31 +7,13 @@ # from __future__ import print_function from setuptools import setup, find_packages, Extension -from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist -from distutils.version import LooseVersion -from distutils import sysconfig from codecs import open import io -import logging import os import os.path -import re import platform import sys -import subprocess -from shutil import rmtree -try: - from shutil import which - CPU_COUNT = os.cpu_count() -except ImportError: - # Python2 - try: - from backports.shutil_which import which - except ImportError: - which = lambda x: x # just rely on path - import multiprocessing - CPU_COUNT = multiprocessing.cpu_count() try: import numpy @@ -100,143 +82,156 @@ def get_version(file, name='__version__'): version = get_version(os.path.join(here, 'perspective', 'core', '_version.py')) -sources = [ - 'dist/src/cpp/aggregate.cpp', - 'dist/src/cpp/aggspec.cpp', - 'dist/src/cpp/arg_sort.cpp', - 'dist/src/cpp/arrow_loader.cpp', - 'dist/src/cpp/arrow_writer.cpp', - 'dist/src/cpp/base.cpp', - 'dist/src/cpp/base_impl_linux.cpp', - 'dist/src/cpp/base_impl_osx.cpp', - 'dist/src/cpp/base_impl_wasm.cpp', - 'dist/src/cpp/base_impl_win.cpp', - 'dist/src/cpp/binding.cpp', - 'dist/src/cpp/build_filter.cpp', - 'dist/src/cpp/column.cpp', - 'dist/src/cpp/comparators.cpp', - 'dist/src/cpp/compat.cpp', - 'dist/src/cpp/compat_impl_linux.cpp', - 'dist/src/cpp/compat_impl_osx.cpp', - 'dist/src/cpp/compat_impl_win.cpp', - 'dist/src/cpp/computed.cpp', - 'dist/src/cpp/computed_column_map.cpp', - 'dist/src/cpp/computed_function.cpp', - 'dist/src/cpp/config.cpp', - 'dist/src/cpp/context_base.cpp', - 'dist/src/cpp/context_grouped_pkey.cpp', - 'dist/src/cpp/context_handle.cpp', - 'dist/src/cpp/context_one.cpp', - 'dist/src/cpp/context_two.cpp', - 'dist/src/cpp/context_zero.cpp', - 'dist/src/cpp/custom_column.cpp', - 'dist/src/cpp/data.cpp', - 'dist/src/cpp/data_slice.cpp', - 'dist/src/cpp/data_table.cpp', - 'dist/src/cpp/date.cpp', - 'dist/src/cpp/dense_nodes.cpp', - 'dist/src/cpp/dense_tree_context.cpp', - 'dist/src/cpp/dense_tree.cpp', - 'dist/src/cpp/dependency.cpp', - 'dist/src/cpp/extract_aggregate.cpp', - 'dist/src/cpp/filter.cpp', - 'dist/src/cpp/flat_traversal.cpp', - 'dist/src/cpp/get_data_extents.cpp', - 'dist/src/cpp/gnode.cpp', - 'dist/src/cpp/gnode_state.cpp', - 'dist/src/cpp/histogram.cpp', - 'dist/src/cpp/logtime.cpp', - 'dist/src/cpp/mask.cpp', - 'dist/src/cpp/min_max.cpp', - 'dist/src/cpp/multi_sort.cpp', - 'dist/src/cpp/none.cpp', - 'dist/src/cpp/path.cpp', - 'dist/src/cpp/pivot.cpp', - 'dist/src/cpp/pool.cpp', - 'dist/src/cpp/port.cpp', - 'dist/src/cpp/process_state.cpp', - 'dist/src/cpp/raii.cpp', - 'dist/src/cpp/raii_impl_linux.cpp', - 'dist/src/cpp/raii_impl_osx.cpp', - 'dist/src/cpp/raii_impl_win.cpp', - 'dist/src/cpp/range.cpp', - 'dist/src/cpp/rlookup.cpp', - 'dist/src/cpp/scalar.cpp', - 'dist/src/cpp/schema_column.cpp', - 'dist/src/cpp/schema.cpp', - 'dist/src/cpp/slice.cpp', - 'dist/src/cpp/sort_specification.cpp', - 'dist/src/cpp/sparse_tree.cpp', - 'dist/src/cpp/sparse_tree_node.cpp', - 'dist/src/cpp/step_delta.cpp', - 'dist/src/cpp/storage.cpp', - 'dist/src/cpp/storage_impl_linux.cpp', - 'dist/src/cpp/storage_impl_osx.cpp', - 'dist/src/cpp/storage_impl_win.cpp', - 'dist/src/cpp/sym_table.cpp', - 'dist/src/cpp/table.cpp', - 'dist/src/cpp/time.cpp', - 'dist/src/cpp/traversal.cpp', - 'dist/src/cpp/traversal_nodes.cpp', - 'dist/src/cpp/tree_context_common.cpp', - 'dist/src/cpp/utils.cpp', - 'dist/src/cpp/update_task.cpp', - 'dist/src/cpp/view.cpp', - 'dist/src/cpp/view_config.cpp', - 'dist/src/cpp/vocab.cpp', +sources = [ + 'dist/src/cpp/aggregate.cpp', + 'dist/src/cpp/aggspec.cpp', + 'dist/src/cpp/arg_sort.cpp', + 'dist/src/cpp/arrow_loader.cpp', + 'dist/src/cpp/arrow_writer.cpp', + 'dist/src/cpp/base.cpp', + 'dist/src/cpp/base_impl_linux.cpp', + 'dist/src/cpp/base_impl_osx.cpp', + 'dist/src/cpp/base_impl_wasm.cpp', + 'dist/src/cpp/base_impl_win.cpp', + 'dist/src/cpp/binding.cpp', + 'dist/src/cpp/build_filter.cpp', + 'dist/src/cpp/column.cpp', + 'dist/src/cpp/comparators.cpp', + 'dist/src/cpp/compat.cpp', + 'dist/src/cpp/compat_impl_linux.cpp', + 'dist/src/cpp/compat_impl_osx.cpp', + 'dist/src/cpp/compat_impl_win.cpp', + 'dist/src/cpp/computed.cpp', + 'dist/src/cpp/computed_column_map.cpp', + 'dist/src/cpp/computed_function.cpp', + 'dist/src/cpp/config.cpp', + 'dist/src/cpp/context_base.cpp', + 'dist/src/cpp/context_grouped_pkey.cpp', + 'dist/src/cpp/context_handle.cpp', + 'dist/src/cpp/context_one.cpp', + 'dist/src/cpp/context_two.cpp', + 'dist/src/cpp/context_zero.cpp', + 'dist/src/cpp/custom_column.cpp', + 'dist/src/cpp/data.cpp', + 'dist/src/cpp/data_slice.cpp', + 'dist/src/cpp/data_table.cpp', + 'dist/src/cpp/date.cpp', + 'dist/src/cpp/dense_nodes.cpp', + 'dist/src/cpp/dense_tree_context.cpp', + 'dist/src/cpp/dense_tree.cpp', + 'dist/src/cpp/dependency.cpp', + 'dist/src/cpp/extract_aggregate.cpp', + 'dist/src/cpp/filter.cpp', + 'dist/src/cpp/flat_traversal.cpp', + 'dist/src/cpp/get_data_extents.cpp', + 'dist/src/cpp/gnode.cpp', + 'dist/src/cpp/gnode_state.cpp', + 'dist/src/cpp/histogram.cpp', + 'dist/src/cpp/logtime.cpp', + 'dist/src/cpp/mask.cpp', + 'dist/src/cpp/min_max.cpp', + 'dist/src/cpp/multi_sort.cpp', + 'dist/src/cpp/none.cpp', + 'dist/src/cpp/path.cpp', + 'dist/src/cpp/pivot.cpp', + 'dist/src/cpp/pool.cpp', + 'dist/src/cpp/port.cpp', + 'dist/src/cpp/process_state.cpp', + 'dist/src/cpp/raii.cpp', + 'dist/src/cpp/raii_impl_linux.cpp', + 'dist/src/cpp/raii_impl_osx.cpp', + 'dist/src/cpp/raii_impl_win.cpp', + 'dist/src/cpp/range.cpp', + 'dist/src/cpp/rlookup.cpp', + 'dist/src/cpp/scalar.cpp', + 'dist/src/cpp/schema_column.cpp', + 'dist/src/cpp/schema.cpp', + 'dist/src/cpp/slice.cpp', + 'dist/src/cpp/sort_specification.cpp', + 'dist/src/cpp/sparse_tree.cpp', + 'dist/src/cpp/sparse_tree_node.cpp', + 'dist/src/cpp/step_delta.cpp', + 'dist/src/cpp/storage.cpp', + 'dist/src/cpp/storage_impl_linux.cpp', + 'dist/src/cpp/storage_impl_osx.cpp', + 'dist/src/cpp/storage_impl_win.cpp', + 'dist/src/cpp/sym_table.cpp', + 'dist/src/cpp/table.cpp', + 'dist/src/cpp/time.cpp', + 'dist/src/cpp/traversal.cpp', + 'dist/src/cpp/traversal_nodes.cpp', + 'dist/src/cpp/tree_context_common.cpp', + 'dist/src/cpp/utils.cpp', + 'dist/src/cpp/update_task.cpp', + 'dist/src/cpp/view.cpp', + 'dist/src/cpp/view_config.cpp', + 'dist/src/cpp/vocab.cpp', # python-specific files - 'perspective/src/column.cpp', + 'perspective/src/column.cpp', ] binding_sources = [ - 'perspective/src/accessor.cpp', - 'perspective/src/computed.cpp', - 'perspective/src/context.cpp', - 'perspective/src/fill.cpp', - 'perspective/src/numpy.cpp', - 'perspective/src/python.cpp', - 'perspective/src/serialization.cpp', - 'perspective/src/table.cpp', - 'perspective/src/utils.cpp', - 'perspective/src/view.cpp', + 'perspective/src/accessor.cpp', + 'perspective/src/computed.cpp', + 'perspective/src/context.cpp', + 'perspective/src/fill.cpp', + 'perspective/src/numpy.cpp', + 'perspective/src/python.cpp', + 'perspective/src/serialization.cpp', + 'perspective/src/table.cpp', + 'perspective/src/utils.cpp', + 'perspective/src/view.cpp', ] + +extra_link_args = [] + +if platform.system() == 'Darwin': + extra_link_args.append('-Wl,-rpath,' + ';'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) +else: + extra_link_args.append('-Wl,-rpath,' + ';'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) + extensions = [ Extension('perspective.table.libpsp', define_macros=[('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))], include_dirs=[ - 'perspective/include/', - 'dist/src/include/', - 'dist/third/date/include/', - 'dist/third/hopscotch/include/', - 'dist/third/ordered-map/include/', - 'dist/third/pybind11/include/', - numpy_includes, - pyarrow_includes, - ], + 'perspective/include/', + 'dist/src/include/', + 'dist/third/date/include/', + 'dist/third/hopscotch/include/', + 'dist/third/ordered-map/include/', + 'dist/third/pybind11/include/', + numpy_includes, + pyarrow_includes, + ], libraries=pyarrow_libraries, library_dirs=pyarrow_library_dirs, + runtime_library_dirs=pyarrow_library_dirs, extra_compile_args=['-std=c++1y'] if os.name != 'nt' else ['/std:c++14'], + extra_link_args=extra_link_args, sources=sources), Extension('perspective.table.libbinding', define_macros=[('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))], include_dirs=[ - 'perspective/include/', - 'dist/src/include/', - 'dist/third/date/include/', - 'dist/third/hopscotch/include/', - 'dist/third/ordered-map/include/', - 'dist/third/pybind11/include/', - numpy_includes, - pyarrow_includes, - ], + 'perspective/include/', + 'dist/src/include/', + 'dist/third/date/include/', + 'dist/third/hopscotch/include/', + 'dist/third/ordered-map/include/', + 'dist/third/pybind11/include/', + numpy_includes, + pyarrow_includes, + ], libraries=pyarrow_libraries, library_dirs=pyarrow_library_dirs, + runtime_library_dirs=pyarrow_library_dirs, extra_compile_args=['-std=c++1y'] if os.name != 'nt' else ['/std:c++14'], + extra_link_args=extra_link_args, sources=binding_sources) ] + class PSPCheckSDist(sdist): def run(self): self.run_check() diff --git a/scripts/build_python.js b/scripts/build_python.js index cd123e0be9..2f36cd37f2 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -40,6 +40,7 @@ try { try { const dist = resolve`${__dirname}/../python/perspective/dist`; + const python = resolve`${__dirname}/../python/perspective/`; const third = resolve`${__dirname}/../cpp/perspective/third`; const cpp = resolve`${__dirname}/../cpp/perspective`; const lic = resolve`${__dirname}/../LICENSE`; From 88cbcb840224cbf9d3e00a00074d649c46379a4b Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Thu, 16 Jul 2020 22:57:26 -0400 Subject: [PATCH 04/31] Add yes to choco --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a013ad93fa..09678d5762 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -164,7 +164,7 @@ jobs: versionSpec: '12.x' - script: | - choco install tbb + choco install tbb -y displayName: "System deps" - script: | From b8344af0dcc93c781e1993f75f1be7a082a563f7 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 12:17:19 -0400 Subject: [PATCH 05/31] attempt to allow empty checksums --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 09678d5762..c49c31394e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -164,7 +164,7 @@ jobs: versionSpec: '12.x' - script: | - choco install tbb -y + choco install tbb -y --allow-empty-checksums displayName: "System deps" - script: | From c4e1ee6428b4338cb8f342952bd3bb5dd2177024 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 13:28:34 -0400 Subject: [PATCH 06/31] use cflag now that no cmake --- azure-pipelines.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c49c31394e..fbdd1d1ded 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -188,13 +188,11 @@ jobs: - script: yarn build_python --ci $(python_flag) displayName: 'build' env: - # Set `BOOST_ROOT` manually, as `BOOST_ROOT` is removed in the VM: + # Pass boost include into python distutils: # https://github.com/actions/virtual-environments/issues/687 # 06/18/2020 - seems like boost got moved to `x86_64` inside # the boost folder, which broke builds for a bit. - BOOST_ROOT: "C:/hostedtoolcache/windows/Boost/1.69.0/x86_64/" - BOOST_INCLUDEDIR: "C:/hostedtoolcache/windows/Boost/1.69.0/x86_64/include" - BOOST_LIBRARYDIR: "C:/hostedtoolcache/windows/Boost/1.69.0/x86_64/libs" + CFLAGS: "-IC:/hostedtoolcache/windows/Boost/1.69.0/x86_64/include" - job: 'MacOS_Mojave' pool: From cdb90caf6624d782e7672cb79b48bd8cf91c8f7b Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 13:40:08 -0400 Subject: [PATCH 07/31] install tbb-devel in docker images --- docker/python/manylinux2010/Dockerfile | 3 +-- docker/python/manylinux2014/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile index 537c6e9395..2fa317cf45 100644 --- a/docker/python/manylinux2010/Dockerfile +++ b/docker/python/manylinux2010/Dockerfile @@ -7,7 +7,7 @@ # FROM quay.io/pypa/manylinux2010_x86_64 -RUN yum -y install rapidjson-devel sudo wget +RUN yum -y install rapidjson-devel sudo wget tbb-devel # Build new cmake RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q @@ -26,7 +26,6 @@ RUN rm -rf /opt/python/cp36-cp36m/bin/auditwheel RUN rm -rf /opt/python/cp37-cp37m/bin/auditwheel RUN rm -rf /opt/python/cp38-cp38/bin/auditwheel - # Copy assets RUN cp -arf /opt/python/cp27-cp27m/* /usr/local/ RUN cp -arf /opt/python/cp36-cp36m/* /usr/local/ diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile index a0e21d4587..e7c099d42c 100644 --- a/docker/python/manylinux2014/Dockerfile +++ b/docker/python/manylinux2014/Dockerfile @@ -7,7 +7,7 @@ # FROM quay.io/pypa/manylinux2014_x86_64 -RUN yum -y install rapidjson-devel sudo wget +RUN yum -y install rapidjson-devel sudo wget tbb-devel # Build new cmake RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q From 518a82cdbcf3bc7602c4e5f8fc0d1e66efdad557 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 13:45:10 -0400 Subject: [PATCH 08/31] set cxxflags on windows --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fbdd1d1ded..50653f2315 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -193,6 +193,7 @@ jobs: # 06/18/2020 - seems like boost got moved to `x86_64` inside # the boost folder, which broke builds for a bit. CFLAGS: "-IC:/hostedtoolcache/windows/Boost/1.69.0/x86_64/include" + CXXFLAGS: "-IC:/hostedtoolcache/windows/Boost/1.69.0/x86_64/include" - job: 'MacOS_Mojave' pool: From cc6902a7d12306a686dd0ae594f59e65444ac05e Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 14:44:19 -0400 Subject: [PATCH 09/31] vendor boost since we only use headers --- azure-pipelines.yml | 12 ++----- docker/python/manylinux2010/Dockerfile | 19 +--------- docker/python/manylinux2014/Dockerfile | 19 +--------- package.json | 1 + python/perspective/setup.py | 2 ++ scripts/build_python.js | 38 ++++++++++++++++---- yarn.lock | 49 ++++++++++++++++++++++++++ 7 files changed, 88 insertions(+), 52 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 50653f2315..60fa58c911 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -161,6 +161,7 @@ jobs: - task: NodeTool@0 inputs: + versionSpec: '12.x' - script: | @@ -187,13 +188,6 @@ jobs: - script: yarn build_python --ci $(python_flag) displayName: 'build' - env: - # Pass boost include into python distutils: - # https://github.com/actions/virtual-environments/issues/687 - # 06/18/2020 - seems like boost got moved to `x86_64` inside - # the boost folder, which broke builds for a bit. - CFLAGS: "-IC:/hostedtoolcache/windows/Boost/1.69.0/x86_64/include" - CXXFLAGS: "-IC:/hostedtoolcache/windows/Boost/1.69.0/x86_64/include" - job: 'MacOS_Mojave' pool: @@ -236,7 +230,7 @@ jobs: versionSpec: '12.x' - script: | - brew install boost tbb + brew install tbb displayName: "System deps" - script: | @@ -356,7 +350,7 @@ jobs: versionSpec: '12.x' - script: | - brew install boost tbb + brew install tbb displayName: "System deps" - script: | diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile index 2fa317cf45..2e38c93370 100644 --- a/docker/python/manylinux2010/Dockerfile +++ b/docker/python/manylinux2010/Dockerfile @@ -46,13 +46,7 @@ RUN python3.6 -m pip install --ignore-installed auditwheel RUN python3.7 -m pip install --ignore-installed auditwheel RUN python3.8 -m pip install --ignore-installed auditwheel -# install boost -RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 -RUN tar xfz boost_1_71_0.tar.gz -# https://github.com/boostorg/build/issues/468 -RUN cd boost_1_71_0 && ./bootstrap.sh -RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install - +# install pinned dependencies RUN python2.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' git+https://chromium.googlesource.com/external/gyp RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' @@ -72,14 +66,3 @@ RUN yum install -y nodejs RUN npm install --global yarn RUN yarn --version - -# install flatbuffers -RUN mkdir -p /usr/local \ - && cd /usr/local \ - && git clone https://github.com/google/flatbuffers.git \ - && cd flatbuffers \ - && cmake -G "Unix Makefiles" \ - && make \ - && cp -r /usr/local/flatbuffers/include/flatbuffers /usr/local/include \ - && ln -s /usr/local/flatbuffers/flatc /usr/local/bin/flatc \ - && chmod +x /usr/local/flatbuffers/flatc diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile index e7c099d42c..46396f08f9 100644 --- a/docker/python/manylinux2014/Dockerfile +++ b/docker/python/manylinux2014/Dockerfile @@ -43,13 +43,7 @@ RUN python3.6 -m pip install --ignore-installed auditwheel RUN python3.7 -m pip install --ignore-installed auditwheel RUN python3.8 -m pip install --ignore-installed auditwheel -# install boost -RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 -RUN tar xfz boost_1_71_0.tar.gz -# https://github.com/boostorg/build/issues/468 -RUN cd boost_1_71_0 && ./bootstrap.sh -RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install - +# install pinned dependencies RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' @@ -67,14 +61,3 @@ RUN yum install -y nodejs RUN npm install --global yarn RUN yarn --version - -# install flatbuffers -RUN mkdir -p /usr/local \ - && cd /usr/local \ - && git clone https://github.com/google/flatbuffers.git \ - && cd flatbuffers \ - && cmake -G "Unix Makefiles" \ - && make \ - && cp -r /usr/local/flatbuffers/include/flatbuffers /usr/local/include \ - && ln -s /usr/local/flatbuffers/flatc /usr/local/bin/flatc \ - && chmod +x /usr/local/flatbuffers/flatc diff --git a/package.json b/package.json index b2e3354bfd..3a8f6d4ee5 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "sinon": "^7.3.1", "source-map-explorer": "^2.0.1", "style-loader": "^0.18.2", + "tar": "^6.0.2", "term-img": "^4.1.0", "ts-jest": "^25.1.0", "ts-loader": "^6.2.0", diff --git a/python/perspective/setup.py b/python/perspective/setup.py index ab4bde9749..26037f2f96 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -198,6 +198,7 @@ def get_version(file, name='__version__'): include_dirs=[ 'perspective/include/', 'dist/src/include/', + 'dist/third/boost/boost_1_71_0/', 'dist/third/date/include/', 'dist/third/hopscotch/include/', 'dist/third/ordered-map/include/', @@ -216,6 +217,7 @@ def get_version(file, name='__version__'): include_dirs=[ 'perspective/include/', 'dist/src/include/', + 'dist/third/boost/boost_1_71_0/', 'dist/third/date/include/', 'dist/third/hopscotch/include/', 'dist/third/ordered-map/include/', diff --git a/scripts/build_python.js b/scripts/build_python.js index 2f36cd37f2..6b8c04f269 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -10,6 +10,7 @@ const {execute, execute_throw, docker, clean, resolve, getarg, bash, python_image} = require("./script_utils.js"); const fs = require("fs-extra"); const rimraf = require("rimraf"); +const tar = require("tar"); const IS_PY2 = getarg("--python2"); @@ -40,7 +41,6 @@ try { try { const dist = resolve`${__dirname}/../python/perspective/dist`; - const python = resolve`${__dirname}/../python/perspective/`; const third = resolve`${__dirname}/../cpp/perspective/third`; const cpp = resolve`${__dirname}/../cpp/perspective`; const lic = resolve`${__dirname}/../LICENSE`; @@ -48,20 +48,44 @@ try { const obj = resolve`${dist}/obj`; // clone third party deps + console.log("Cloning third party dependencies"); if (!fs.existsSync(third)) { - console.log("Cloning third party dependencies"); fs.mkdirpSync(third); - execute`git clone https://github.com/HowardHinnant/date.git ${third}/date`; - execute`git clone https://github.com/Tessil/hopscotch-map.git ${third}/hopscotch`; - execute`git clone https://github.com/Tessil/ordered-map.git ${third}/ordered-map`; - execute`git clone https://github.com/pybind/pybind11.git ${third}/pybind11`; + } + if (!fs.existsSync(`${third}/boost`)) { + console.log("Cloning boost"); + fs.mkdirpSync(`${third}/boost`); + execute`wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -O ${third}/boost/boost.tgz`; + + tar.x({sync: true, file: `${third}/boost/boost.tgz`, cwd: `${third}/boost`}); + rimraf.sync(`${third}/boost/boost.tgz`); + } + + if (!fs.existsSync(`${third}/date`)) { + console.log("Cloning date"); + execute`git clone https://github.com/HowardHinnant/date.git ${third}/date`; rimraf.sync(`${third}/date/.git`); + } + + if (!fs.existsSync(`${third}/hopscotch`)) { + console.log("Cloning hopscotch"); + execute`git clone https://github.com/Tessil/hopscotch-map.git ${third}/hopscotch`; rimraf.sync(`${third}/hopscotch/.git`); + } + + if (!fs.existsSync(`${third}/ordered-map`)) { + console.log("Cloning ordered-map"); + execute`git clone https://github.com/Tessil/ordered-map.git ${third}/ordered-map`; rimraf.sync(`${third}/ordered-map/.git`); + } + + if (!fs.existsSync(`${third}/pybind11`)) { + console.log("Cloning pybind11"); + execute`git clone https://github.com/pybind/pybind11.git ${third}/pybind11`; rimraf.sync(`${third}/pybind11/.git`); - console.log("Cloning third party dependencies...done!"); } + console.log("Cloning third party dependencies...done!"); fs.mkdirpSync(dist); fs.copySync(cpp, dist, {preserveTimestamps: true}); diff --git a/yarn.lock b/yarn.lock index f0faf8460f..318ff060fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3587,6 +3587,11 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142" integrity sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw== +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + chroma-js@^1.3.4: version "1.4.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.4.1.tgz#eb2d9c4d1ff24616be84b35119f4d26f8205f134" @@ -6853,6 +6858,13 @@ fs-minipass@^1.2.5: dependencies: minipass "^2.6.0" +fs-minipass@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + fs-readdir-recursive@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz#e32fc030a2ccee44a6b5371308da54be0b397d27" @@ -10205,6 +10217,13 @@ minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0: safe-buffer "^5.1.2" yallist "^3.0.0" +minipass@^3.0.0: + version "3.1.3" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd" + integrity sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg== + dependencies: + yallist "^4.0.0" + minizlib@^1.2.1: version "1.3.3" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d" @@ -10212,6 +10231,14 @@ minizlib@^1.2.1: dependencies: minipass "^2.9.0" +minizlib@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.0.tgz#fd52c645301ef09a63a2c209697c294c6ce02cf3" + integrity sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + mississippi@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022" @@ -10248,6 +10275,11 @@ mkdirp@0.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: dependencies: minimist "0.0.8" +mkdirp@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + mobile-drag-drop@^2.3.0-rc.2: version "2.3.0-rc.2" resolved "https://registry.yarnpkg.com/mobile-drag-drop/-/mobile-drag-drop-2.3.0-rc.2.tgz#00d6e85e04512a620fd5357366e8786bd29aa7aa" @@ -14134,6 +14166,18 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.3" +tar@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.0.2.tgz#5df17813468a6264ff14f766886c622b84ae2f39" + integrity sha512-Glo3jkRtPcvpDlAs/0+hozav78yoXKFr+c4wgw62NNMO3oo4AaJdCo21Uu7lcwr55h39W2XD1LMERc64wtbItg== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.0" + mkdirp "^1.0.3" + yallist "^4.0.0" + tcp-port-used@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tcp-port-used/-/tcp-port-used-1.0.1.tgz#46061078e2d38c73979a2c2c12b5a674e6689d70" @@ -15515,6 +15559,11 @@ yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yamljs@^0.2.1: version "0.2.10" resolved "https://registry.yarnpkg.com/yamljs/-/yamljs-0.2.10.tgz#481cc7c25ca73af59f591f0c96e3ce56c757a40f" From 4e854d9c891dfe1c3b2ba167fff795d3a2e6040f Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 15:04:34 -0400 Subject: [PATCH 10/31] use curl instead of wget --- python/perspective/setup.py | 4 ++-- scripts/build_python.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 26037f2f96..390e22f511 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -188,9 +188,9 @@ def get_version(file, name='__version__'): extra_link_args = [] if platform.system() == 'Darwin': - extra_link_args.append('-Wl,-rpath,' + ';'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,' + ';'.join(('@loader_path/../../pyarrow/', pyarrow_library_dirs[0]))) else: - extra_link_args.append('-Wl,-rpath,' + ';'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,' + ';'.join(('$ORIGIN/../../pyarrow/', pyarrow_library_dirs[0]))) extensions = [ Extension('perspective.table.libpsp', diff --git a/scripts/build_python.js b/scripts/build_python.js index 6b8c04f269..2a9dbbdca8 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -56,7 +56,9 @@ try { if (!fs.existsSync(`${third}/boost`)) { console.log("Cloning boost"); fs.mkdirpSync(`${third}/boost`); - execute`wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -O ${third}/boost/boost.tgz`; + + const file = fs.createWriteStream(`${third}/boost/boost.tgz`); + execute`curl -L https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -o ${third}/boost/boost.tgz`; tar.x({sync: true, file: `${third}/boost/boost.tgz`, cwd: `${third}/boost`}); rimraf.sync(`${third}/boost/boost.tgz`); From c15c9432ed331a42e263bf45b9047bf851f72c65 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 15:27:51 -0400 Subject: [PATCH 11/31] tweak download/extract messages --- python/perspective/setup.py | 4 ++-- scripts/build_python.js | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 390e22f511..a24d457596 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -188,9 +188,9 @@ def get_version(file, name='__version__'): extra_link_args = [] if platform.system() == 'Darwin': - extra_link_args.append('-Wl,-rpath,' + ';'.join(('@loader_path/../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) else: - extra_link_args.append('-Wl,-rpath,' + ';'.join(('$ORIGIN/../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,' + ';'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) extensions = [ Extension('perspective.table.libpsp', diff --git a/scripts/build_python.js b/scripts/build_python.js index 2a9dbbdca8..d9b906baa4 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -54,13 +54,16 @@ try { } if (!fs.existsSync(`${third}/boost`)) { - console.log("Cloning boost"); + console.log("Downloading boost"); fs.mkdirpSync(`${third}/boost`); const file = fs.createWriteStream(`${third}/boost/boost.tgz`); execute`curl -L https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -o ${third}/boost/boost.tgz`; + console.log("Downloading boost...done!"); + console.log("Extracting boost"); tar.x({sync: true, file: `${third}/boost/boost.tgz`, cwd: `${third}/boost`}); + console.log("Extracting boost...done!"); rimraf.sync(`${third}/boost/boost.tgz`); } @@ -68,24 +71,28 @@ try { console.log("Cloning date"); execute`git clone https://github.com/HowardHinnant/date.git ${third}/date`; rimraf.sync(`${third}/date/.git`); + console.log("Cloning date...done!"); } if (!fs.existsSync(`${third}/hopscotch`)) { console.log("Cloning hopscotch"); execute`git clone https://github.com/Tessil/hopscotch-map.git ${third}/hopscotch`; rimraf.sync(`${third}/hopscotch/.git`); + console.log("Cloning hopscotch...done!"); } if (!fs.existsSync(`${third}/ordered-map`)) { console.log("Cloning ordered-map"); execute`git clone https://github.com/Tessil/ordered-map.git ${third}/ordered-map`; rimraf.sync(`${third}/ordered-map/.git`); + console.log("Cloning ordered-map...done!"); } if (!fs.existsSync(`${third}/pybind11`)) { console.log("Cloning pybind11"); execute`git clone https://github.com/pybind/pybind11.git ${third}/pybind11`; rimraf.sync(`${third}/pybind11/.git`); + console.log("Cloning pybind11...done!"); } console.log("Cloning third party dependencies...done!"); From e522250b2fabac50b92314a134413e81a08ba2a9 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 15:40:06 -0400 Subject: [PATCH 12/31] verbose --- python/perspective/setup.py | 4 ++-- scripts/build_python.js | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index a24d457596..c2f1d61bf2 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -188,9 +188,9 @@ def get_version(file, name='__version__'): extra_link_args = [] if platform.system() == 'Darwin': - extra_link_args.append('-Wl,-rpath,'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,' + '-rpath,'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) else: - extra_link_args.append('-Wl,-rpath,' + ';'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,' + '-rpath,'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) extensions = [ Extension('perspective.table.libpsp', diff --git a/scripts/build_python.js b/scripts/build_python.js index d9b906baa4..a68ccb4c28 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -63,8 +63,8 @@ try { console.log("Extracting boost"); tar.x({sync: true, file: `${third}/boost/boost.tgz`, cwd: `${third}/boost`}); - console.log("Extracting boost...done!"); rimraf.sync(`${third}/boost/boost.tgz`); + console.log("Extracting boost...done!"); } if (!fs.existsSync(`${third}/date`)) { @@ -97,8 +97,13 @@ try { console.log("Cloning third party dependencies...done!"); fs.mkdirpSync(dist); + console.log("Copying C++ to python dist"); fs.copySync(cpp, dist, {preserveTimestamps: true}); + console.log("Copying C++ to python dist...done!"); + + console.log("Copying LICENSE to python dist"); fs.copySync(lic, dlic, {preserveTimestamps: true}); + console.log("Copying LICENSE to python dist...done!"); clean(obj); let cmd; From 4cc782d26d67df510a050bf6367527b8e24819df Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 18:18:13 -0400 Subject: [PATCH 13/31] fix path weirdness on windows --- python/perspective/setup.py | 27 ++++----------------------- scripts/build_python.js | 9 +++++---- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index c2f1d61bf2..0bfcd2059a 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -188,30 +188,11 @@ def get_version(file, name='__version__'): extra_link_args = [] if platform.system() == 'Darwin': - extra_link_args.append('-Wl,' + '-rpath,'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) else: - extra_link_args.append('-Wl,' + '-rpath,'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) extensions = [ - Extension('perspective.table.libpsp', - define_macros=[('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))], - include_dirs=[ - 'perspective/include/', - 'dist/src/include/', - 'dist/third/boost/boost_1_71_0/', - 'dist/third/date/include/', - 'dist/third/hopscotch/include/', - 'dist/third/ordered-map/include/', - 'dist/third/pybind11/include/', - numpy_includes, - pyarrow_includes, - ], - libraries=pyarrow_libraries, - library_dirs=pyarrow_library_dirs, - runtime_library_dirs=pyarrow_library_dirs, - extra_compile_args=['-std=c++1y'] if os.name != 'nt' else ['/std:c++14'], - extra_link_args=extra_link_args, - sources=sources), Extension('perspective.table.libbinding', define_macros=[('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))], include_dirs=[ @@ -225,12 +206,12 @@ def get_version(file, name='__version__'): numpy_includes, pyarrow_includes, ], - libraries=pyarrow_libraries, + libraries=pyarrow_libraries + ['tbb'], library_dirs=pyarrow_library_dirs, runtime_library_dirs=pyarrow_library_dirs, extra_compile_args=['-std=c++1y'] if os.name != 'nt' else ['/std:c++14'], extra_link_args=extra_link_args, - sources=binding_sources) + sources=sources + binding_sources) ] diff --git a/scripts/build_python.js b/scripts/build_python.js index a68ccb4c28..c34b007e1d 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -57,13 +57,14 @@ try { console.log("Downloading boost"); fs.mkdirpSync(`${third}/boost`); - const file = fs.createWriteStream(`${third}/boost/boost.tgz`); - execute`curl -L https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -o ${third}/boost/boost.tgz`; + const tarball = resolve`${third}/boost/boost.tgz`; + const boostout = resolve`${third}/boost`; + execute`curl -L https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -o ${tarball}`; console.log("Downloading boost...done!"); console.log("Extracting boost"); - tar.x({sync: true, file: `${third}/boost/boost.tgz`, cwd: `${third}/boost`}); - rimraf.sync(`${third}/boost/boost.tgz`); + tar.x({sync: true, file: `${tarball}`, cwd: `${boostout}`}); + rimraf.sync(`${tarball}`); console.log("Extracting boost...done!"); } From 3b86f88136ce42503f405b5466334b8022861902 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 19:13:07 -0400 Subject: [PATCH 14/31] remove shutil-which dep on py2, add pathlib dep --- .../perspective/tests/client/test_client.py | 27 ++++++++----------- python/perspective/setup.py | 2 +- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/python/perspective/perspective/tests/client/test_client.py b/python/perspective/perspective/tests/client/test_client.py index ebad2f7d3c..a9202175fe 100644 --- a/python/perspective/perspective/tests/client/test_client.py +++ b/python/perspective/perspective/tests/client/test_client.py @@ -7,27 +7,28 @@ # import six +import sys import os import numpy as np import pandas as pd +import pathlib from datetime import date, datetime from functools import partial from types import MethodType -if os.name == 'nt': - BINDING = 'libbinding.pyd' - PSP = 'libpsp.dll' -else: - BINDING = 'libbinding.so' - PSP = 'libpsp.so' - # rename libbinding.so and libpsp.so temporarily to ensure that client mode # works automatically when the C++ build fails. lib_path = os.path.join(os.path.dirname(__file__), "..", "..", "table") -binding = os.path.join(lib_path, BINDING) -psp = os.path.join(lib_path, PSP) + +if os.name == 'nt': + BINDING = 'libbinding*.pyd' +else: + BINDING = 'libbinding*.so' + + + +binding = os.path.join(str(list(pathlib.Path(lib_path).glob(BINDING))[0])) new_binding = os.path.join(lib_path, "notlibbinding.so") -new_psp = os.path.join(lib_path, "notlibpsp.so") def mock_post(self, msg, msg_id=None, assert_msg=None): @@ -37,20 +38,14 @@ def mock_post(self, msg, msg_id=None, assert_msg=None): def setup_module(): os.rename(binding, new_binding) - os.rename(psp, new_psp) assert os.path.exists(new_binding) - assert os.path.exists(new_psp) assert not os.path.exists(binding) - assert not os.path.exists(psp) def teardown_module(): os.rename(new_binding, binding) - os.rename(new_psp, psp) assert os.path.exists(binding) - assert os.path.exists(psp) assert not os.path.exists(new_binding) - assert not os.path.exists(new_psp) class TestClient(object): diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 0bfcd2059a..1fd8fda835 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -48,7 +48,7 @@ ] if sys.version_info.major < 3: - requires.append("backports.shutil-which") + requires.append("pathlib") if (sys.version_info.major == 2 and sys.version_info.minor < 7) or \ (sys.version_info.major == 3 and sys.version_info.minor < 6): From 292afa33ff3d6e6e24a7af35e11b8dcd6ff0c8b0 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 19:19:15 -0400 Subject: [PATCH 15/31] attempt to solve boost issues on windows --- python/perspective/setup.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 1fd8fda835..fe316e09b4 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -192,9 +192,19 @@ def get_version(file, name='__version__'): else: extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) + +defines = [('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))] +extra_compiler_args = [] + +if os.name == 'nt': + defines.append(('BOOST_WINDOWS', '1')) + extra_compiler_args.append('-std=c++1y') +else: + extra_compiler_args.append('/std:c++14') + extensions = [ Extension('perspective.table.libbinding', - define_macros=[('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))], + define_macros=defines, include_dirs=[ 'perspective/include/', 'dist/src/include/', @@ -209,7 +219,7 @@ def get_version(file, name='__version__'): libraries=pyarrow_libraries + ['tbb'], library_dirs=pyarrow_library_dirs, runtime_library_dirs=pyarrow_library_dirs, - extra_compile_args=['-std=c++1y'] if os.name != 'nt' else ['/std:c++14'], + extra_compile_args=extra_compiler_args, extra_link_args=extra_link_args, sources=sources + binding_sources) ] From 55f5790793f8a20031e58cd45c7875d1353fa9dd Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 19:26:56 -0400 Subject: [PATCH 16/31] copypast --- python/perspective/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index fe316e09b4..47d0ad23ec 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -198,9 +198,9 @@ def get_version(file, name='__version__'): if os.name == 'nt': defines.append(('BOOST_WINDOWS', '1')) - extra_compiler_args.append('-std=c++1y') -else: extra_compiler_args.append('/std:c++14') +else: + extra_compiler_args.append('-std=c++1y') extensions = [ Extension('perspective.table.libbinding', From c0273da014a2e2c9f5bd50f6e784361a14ebe977 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 19:53:17 -0400 Subject: [PATCH 17/31] adding win32 flags --- python/perspective/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 47d0ad23ec..da87a0a0f3 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -197,7 +197,7 @@ def get_version(file, name='__version__'): extra_compiler_args = [] if os.name == 'nt': - defines.append(('BOOST_WINDOWS', '1')) + defines.append(('BOOST_WINDOWS', '1'), ('WIN32', '1'), ('_WIN32', '1')) extra_compiler_args.append('/std:c++14') else: extra_compiler_args.append('-std=c++1y') From 0dff414476bd4b5b7e4a6a3dc2b69491f0da8b24 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 19:57:14 -0400 Subject: [PATCH 18/31] pin pyproject to match --- python/perspective/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/perspective/pyproject.toml b/python/perspective/pyproject.toml index a201cf948b..9c938d599c 100644 --- a/python/perspective/pyproject.toml +++ b/python/perspective/pyproject.toml @@ -1,3 +1,3 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel", "numpy", "pyarrow>=0.15.1"] +requires = ["setuptools", "wheel", "numpy", "pyarrow==0.16.0"] From 33dd756bdf429a7f593de40987eaded3ba7f786f Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 20:52:44 -0400 Subject: [PATCH 19/31] copypasta --- python/perspective/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index da87a0a0f3..d0aa14d44f 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -197,7 +197,7 @@ def get_version(file, name='__version__'): extra_compiler_args = [] if os.name == 'nt': - defines.append(('BOOST_WINDOWS', '1'), ('WIN32', '1'), ('_WIN32', '1')) + defines.extend([('BOOST_WINDOWS', '1'), ('WIN32', '1'), ('_WIN32', '1')]) extra_compiler_args.append('/std:c++14') else: extra_compiler_args.append('-std=c++1y') From 4cedb8318192936125804c50c87cb1d9a5d96201 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 22:19:44 -0400 Subject: [PATCH 20/31] adding tbb to cflags --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 60fa58c911..4a2c69ce2d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -188,6 +188,8 @@ jobs: - script: yarn build_python --ci $(python_flag) displayName: 'build' + env: + CFLAGS: /IC:\ProgramData\chocolatey\lib\tbb\tools\include - job: 'MacOS_Mojave' pool: From 5063bb173b4705187da153821b4506506d709c49 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Fri, 17 Jul 2020 23:19:36 -0400 Subject: [PATCH 21/31] Add path to tbb --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4a2c69ce2d..32d4d1ed3c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -189,7 +189,7 @@ jobs: - script: yarn build_python --ci $(python_flag) displayName: 'build' env: - CFLAGS: /IC:\ProgramData\chocolatey\lib\tbb\tools\include + CFLAGS: /IC:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\include\ - job: 'MacOS_Mojave' pool: From c71acb3fd65e2b54c08068bd31f0e39d76dcf1e5 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sat, 18 Jul 2020 12:44:08 -0400 Subject: [PATCH 22/31] working on windows :-) --- azure-pipelines.yml | 1 + docs/md/development.md | 34 ++++++-- .../include/perspective/python/numpy.h | 4 - python/perspective/pyproject.toml | 2 +- python/perspective/setup.py | 61 ++++++++++--- scripts/build_python.js | 86 +++++++++++++++---- 6 files changed, 147 insertions(+), 41 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 32d4d1ed3c..7471a3c83c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -190,6 +190,7 @@ jobs: displayName: 'build' env: CFLAGS: /IC:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\include\ + LDFLAGS: /LIBPATH:C:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\lib\intel64\vc11\ - job: 'MacOS_Mojave' pool: diff --git a/docs/md/development.md b/docs/md/development.md index 0c449e138c..088e5333ba 100644 --- a/docs/md/development.md +++ b/docs/md/development.md @@ -61,8 +61,8 @@ built from source: - Boost (version 1.67) - CMake (version 3.15.4 or higher) -- TBB -- [Flatbuffers](https://google.github.io/flatbuffers/flatbuffers_guide_building.html) +- TBB (when building python) +- [Flatbuffers](https://google.github.io/flatbuffers/flatbuffers_guide_building.html) (when building JS) ### `Perspective.js` @@ -98,16 +98,31 @@ To build the Python library, navigate to the python source folder (`python/perspective`) and install the dependencies using `pip`: ```bash -python3 -m pip install -r requirements.txt -python3 -m pip install -r requirements-dev.txt +python -m pip install numpy "pyarrow>=0.15.1,<0.17" ``` Make sure that TBB is installed, as it is a system dependency: +OS X ```bash brew install TBB ``` +Debian +```bash +apt-get install libtbb-devel +``` + +RHEL +```bash +yum install tbb-devel +``` + +Windows +```bash +choco install tbb +``` + `perspective-python` supports Python 3.7 and upwards, as well as Python 2.7.17. To build the Python 2 version of the library, use the `--python2` flag: @@ -131,13 +146,18 @@ You'll also need the system dependencies noted earlier - installing them through Homebrew is the easiest way, especially as flatbuffers can be installed through `brew` without building from source: +Javascript ```bash -brew install boost@1.67 -brew install tbb brew install cmake +brew install boost@1.67 brew install flatbuffers ``` +Python +```bash +brew install tbb +``` + If building the Python 2 version of the library, make sure your version of Python 2 is the latest version (`2.7.17`) supplied by Homebrew, and not the earlier version that ships with MacOS. To install Python 2 using Homebrew: @@ -148,7 +168,7 @@ brew install python2 ### Windows 10 specific instructions -You need to use bash in order to build Perspective packages. To successfully +You need to use bash in order to build Perspective JS packages. To successfully build on Windows 10, enable [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) (WSL) and install the linux distribution of your choice. diff --git a/python/perspective/perspective/include/perspective/python/numpy.h b/python/perspective/perspective/include/perspective/python/numpy.h index 034fd8bba5..6851e4b5fe 100644 --- a/python/perspective/perspective/include/perspective/python/numpy.h +++ b/python/perspective/perspective/include/perspective/python/numpy.h @@ -18,12 +18,8 @@ #include #ifdef WIN32 -#ifndef PERSPECTIVE_EXPORTS #define PERSPECTIVE_BINDING_EXPORT __declspec(dllexport) #else -#define PERSPECTIVE_BINDING_EXPORT __declspec(dllimport) -#endif -#else #define PERSPECTIVE_BINDING_EXPORT __attribute__((visibility("default"))) #endif diff --git a/python/perspective/pyproject.toml b/python/perspective/pyproject.toml index 9c938d599c..c07cca8d99 100644 --- a/python/perspective/pyproject.toml +++ b/python/perspective/pyproject.toml @@ -1,3 +1,3 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel", "numpy", "pyarrow==0.16.0"] +requires = ["setuptools", "wheel", "numpy", "pyarrow>=0.15.1,<0.17"] diff --git a/python/perspective/setup.py b/python/perspective/setup.py index d0aa14d44f..41884cb8b5 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -15,13 +15,37 @@ import platform import sys +# ******************************************** # +# Get number of cores for numpy parallel build # +# ******************************************** # +try: + from shutil import which + CPU_COUNT = os.cpu_count() +except ImportError: + # Python2 + import multiprocessing + CPU_COUNT = multiprocessing.cpu_count() + +# *************************************** # +# Numpy build path and compiler toolchain # +# *************************************** # try: import numpy numpy_includes = numpy.get_include() + + # enable numpy faster compiler + from numpy.distutils.ccompiler import CCompiler_compile + import distutils.ccompiler + distutils.ccompiler.CCompiler.compile = CCompiler_compile + os.environ['NPY_NUM_BUILD_JOBS'] = str(CPU_COUNT) + except ImportError: print('Must install numpy prior to installing perspective!') raise +# ****************** # +# PyArrow build path # +# ****************** # try: import pyarrow pyarrow_includes = pyarrow.get_include() @@ -36,12 +60,15 @@ with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() +# ******************************** # +# Requires and platform validation # +# ******************************** # requires = [ 'ipywidgets>=7.5.1', 'future>=0.16.0', 'numpy>=1.13.1', 'pandas>=0.22.0', - 'pyarrow==0.16.0', + 'pyarrow>=0.15.1,<0.17', 'python-dateutil>=2.8.0', 'six>=1.11.0', 'traitlets>=4.3.2', @@ -82,6 +109,9 @@ def get_version(file, name='__version__'): version = get_version(os.path.join(here, 'perspective', 'core', '_version.py')) +# *********** # +# C++ sources # +# *********** # sources = [ 'dist/src/cpp/aggregate.cpp', 'dist/src/cpp/aggspec.cpp', @@ -184,8 +214,10 @@ def get_version(file, name='__version__'): 'perspective/src/view.cpp', ] - -extra_link_args = [] +# **************************** # +# Compiler and extension setup # +# **************************** # +extra_link_args = os.environ.get('LDFLAGS', '').split() if platform.system() == 'Darwin': extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) @@ -194,13 +226,18 @@ def get_version(file, name='__version__'): defines = [('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))] -extra_compiler_args = [] +extra_compiler_args = os.environ.get('CFLAGS', '').split() + os.environ.get('CXXFLAGS', '').split() +library_dirs = pyarrow_library_dirs +libraries = pyarrow_libraries + ['tbb'] if os.name == 'nt': - defines.extend([('BOOST_WINDOWS', '1'), ('WIN32', '1'), ('_WIN32', '1')]) + defines.extend([('BOOST_WINDOWS', '1'), ('WIN32', '1'), ('_WIN32', '1'), ('PERSPECTIVE_EXPORTS', '1')]) extra_compiler_args.append('/std:c++14') + extra_compiler_args.append('/MP') + runtime_library_dirs = [] else: extra_compiler_args.append('-std=c++1y') + runtime_library_dirs = pyarrow_library_dirs extensions = [ Extension('perspective.table.libbinding', @@ -216,15 +253,17 @@ def get_version(file, name='__version__'): numpy_includes, pyarrow_includes, ], - libraries=pyarrow_libraries + ['tbb'], - library_dirs=pyarrow_library_dirs, - runtime_library_dirs=pyarrow_library_dirs, + libraries=libraries, + library_dirs=library_dirs, + runtime_library_dirs=runtime_library_dirs, extra_compile_args=extra_compiler_args, extra_link_args=extra_link_args, sources=sources + binding_sources) ] - +# **************** # +# SDist validation # +# **************** # class PSPCheckSDist(sdist): def run(self): self.run_check() @@ -237,6 +276,9 @@ def run_check(self): raise Exception("Path is missing! {}\nMust run `yarn build_python` before building sdist so c++ files are installed".format(path)) +# ***** # +# setup # +# ***** # setup( name='perspective-python', version=version, @@ -256,7 +298,6 @@ def run_check(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', ], - keywords='analytics tools plotting', packages=find_packages(), include_package_data=True, diff --git a/scripts/build_python.js b/scripts/build_python.js index c34b007e1d..37c663086a 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -41,8 +41,22 @@ try { try { const dist = resolve`${__dirname}/../python/perspective/dist`; + const dist_src = resolve`${__dirname}/../python/perspective/dist/src`; + const dist_third = resolve`${__dirname}/../python/perspective/dist/third`; + const dist_third_boost = resolve`${__dirname}/../python/perspective/dist/third/boost`; + const dist_third_date = resolve`${__dirname}/../python/perspective/dist/third/date`; + const dist_third_hopscotch = resolve`${__dirname}/../python/perspective/dist/third/hopscotch`; + const dist_third_ordered_map = resolve`${__dirname}/../python/perspective/dist/third/ordered-map`; + const dist_third_pybind11 = resolve`${__dirname}/../python/perspective/dist/third/pybind11`; + const third = resolve`${__dirname}/../cpp/perspective/third`; - const cpp = resolve`${__dirname}/../cpp/perspective`; + const third_boost = resolve`${__dirname}/../cpp/perspective/third/boost`; + const third_date = resolve`${__dirname}/../cpp/perspective/third/date`; + const third_hopscotch = resolve`${__dirname}/../cpp/perspective/third/hopscotch`; + const third_ordered_map = resolve`${__dirname}/../cpp/perspective/third/ordered-map`; + const third_pybind11 = resolve`${__dirname}/../cpp/perspective/third/pybind11`; + + const cpp_src = resolve`${__dirname}/../cpp/perspective/src`; const lic = resolve`${__dirname}/../LICENSE`; const dlic = resolve`${dist}/LICENSE`; const obj = resolve`${dist}/obj`; @@ -53,55 +67,89 @@ try { fs.mkdirpSync(third); } - if (!fs.existsSync(`${third}/boost`)) { + if (!fs.existsSync(third_boost)) { console.log("Downloading boost"); - fs.mkdirpSync(`${third}/boost`); + fs.mkdirpSync(third_boost); - const tarball = resolve`${third}/boost/boost.tgz`; - const boostout = resolve`${third}/boost`; + const tarball = resolve`${third_boost}/boost.tgz`; execute`curl -L https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -o ${tarball}`; console.log("Downloading boost...done!"); console.log("Extracting boost"); - tar.x({sync: true, file: `${tarball}`, cwd: `${boostout}`}); + tar.x({sync: true, file: `${tarball}`, cwd: `${third_boost}`}); rimraf.sync(`${tarball}`); console.log("Extracting boost...done!"); } - if (!fs.existsSync(`${third}/date`)) { + if (!fs.existsSync(third_date)) { console.log("Cloning date"); - execute`git clone https://github.com/HowardHinnant/date.git ${third}/date`; - rimraf.sync(`${third}/date/.git`); + execute`git clone https://github.com/HowardHinnant/date.git ${third_date}`; + rimraf.sync(`${third_date}/.git`); console.log("Cloning date...done!"); } - if (!fs.existsSync(`${third}/hopscotch`)) { + if (!fs.existsSync(third_hopscotch)) { console.log("Cloning hopscotch"); - execute`git clone https://github.com/Tessil/hopscotch-map.git ${third}/hopscotch`; - rimraf.sync(`${third}/hopscotch/.git`); + execute`git clone https://github.com/Tessil/hopscotch-map.git ${third_hopscotch}`; + rimraf.sync(`${third_hopscotch}/.git`); console.log("Cloning hopscotch...done!"); } - if (!fs.existsSync(`${third}/ordered-map`)) { + if (!fs.existsSync(third_ordered_map)) { console.log("Cloning ordered-map"); - execute`git clone https://github.com/Tessil/ordered-map.git ${third}/ordered-map`; - rimraf.sync(`${third}/ordered-map/.git`); + execute`git clone https://github.com/Tessil/ordered-map.git ${third_ordered_map}`; + rimraf.sync(`${third_ordered_map}/.git`); console.log("Cloning ordered-map...done!"); } - if (!fs.existsSync(`${third}/pybind11`)) { + if (!fs.existsSync(third_pybind11)) { console.log("Cloning pybind11"); - execute`git clone https://github.com/pybind/pybind11.git ${third}/pybind11`; - rimraf.sync(`${third}/pybind11/.git`); + execute`git clone https://github.com/pybind/pybind11.git ${third_pybind11}`; + rimraf.sync(`${third_pybind11}/.git`); console.log("Cloning pybind11...done!"); } console.log("Cloning third party dependencies...done!"); fs.mkdirpSync(dist); + fs.mkdirpSync(dist_src); + fs.mkdirpSync(dist_third); + console.log("Copying C++ to python dist"); - fs.copySync(cpp, dist, {preserveTimestamps: true}); + fs.copySync(cpp_src, dist_src, {preserveTimestamps: true}); console.log("Copying C++ to python dist...done!"); + if (!fs.existsSync(dist_third_boost)) { + console.log("Copying boost to python dist"); + fs.copySync(third_boost, dist_third_boost, {preserveTimestamps: true}); + console.log("Copying boost to python dist...done!"); + } + + if (!fs.existsSync(dist_third_date)) { + console.log("Copying date to python dist"); + fs.copySync(third_date, dist_third_date, {preserveTimestamps: true}); + console.log("Copying date to python dist...done!"); + } + + if (!fs.existsSync(dist_third_hopscotch)) { + console.log("Copying hopscotch to python dist"); + fs.copySync(third_hopscotch, dist_third_hopscotch, {preserveTimestamps: true}); + console.log("Copying hopscotch to python dist...done!"); + } + + if (!fs.existsSync(dist_third_ordered_map)) { + console.log("Copying ordered-map to python dist"); + fs.copySync(third_ordered_map, dist_third_ordered_map, {preserveTimestamps: true}); + console.log("Copying ordered-map to python dist...done!"); + } + + if (!fs.existsSync(dist_third_pybind11)) { + console.log("Copying pybind11 to python dist"); + fs.copySync(third_pybind11, dist_third_pybind11, {preserveTimestamps: true}); + console.log("Copying pybind11 to python dist...done!"); + } + + + console.log("Copying LICENSE to python dist"); fs.copySync(lic, dlic, {preserveTimestamps: true}); console.log("Copying LICENSE to python dist...done!"); From b8796ccfb8a6277549e01095e67e0c29c2f9d0ab Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sat, 18 Jul 2020 13:12:57 -0400 Subject: [PATCH 23/31] build inplace for testing --- scripts/build_python.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_python.js b/scripts/build_python.js index 37c663086a..5ea68436c4 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -165,6 +165,7 @@ try { cmd = cmd + `${PYTHON} -m pip install -e .[dev] && \ + ${PYTHON} setup.py build_ext --inplace && \ ${PYTHON} -m flake8 perspective && echo OK && \ ${PYTHON} -m pytest -vvv --noconftest perspective/tests/client && \ ${PYTHON} -m pytest -vvv perspective \ From ff08268b19584c22d49aaf25d9a0b4d223b31fcd Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sat, 18 Jul 2020 21:12:14 -0400 Subject: [PATCH 24/31] reorder tests --- scripts/build_python.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_python.js b/scripts/build_python.js index 5ea68436c4..95bffadfc2 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -167,11 +167,11 @@ try { `${PYTHON} -m pip install -e .[dev] && \ ${PYTHON} setup.py build_ext --inplace && \ ${PYTHON} -m flake8 perspective && echo OK && \ - ${PYTHON} -m pytest -vvv --noconftest perspective/tests/client && \ ${PYTHON} -m pytest -vvv perspective \ --ignore=perspective/tests/client \ --junitxml=python_junit.xml --cov-report=xml --cov-branch \ - --cov=perspective`; + --cov=perspective && \ + ${PYTHON} -m pytest -vvv --noconftest perspective/tests/client`; if (IMAGE == "python") { cmd = cmd + From 1a3fc12ff4bbc5d423aaffd4b23c5af96a564f10 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sun, 19 Jul 2020 17:23:16 -0400 Subject: [PATCH 25/31] some more adjustments --- .gitignore | 1 + azure-pipelines.yml | 3 +- python/perspective/MANIFEST.in | 15 +------- .../perspective/tests/client/test_client.py | 1 + python/perspective/pyproject.toml | 2 +- python/perspective/setup.py | 38 +++++++++++++++++-- scripts/build_python.js | 18 ++++++++- 7 files changed, 56 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 8799afc976..8eae080222 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ flycheck_*.el *.so *.dylib *.dll +*.pyd # Fortran module files *.mod diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7471a3c83c..7e854a8c3d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -191,7 +191,8 @@ jobs: env: CFLAGS: /IC:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\include\ LDFLAGS: /LIBPATH:C:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\lib\intel64\vc11\ - + PATH: C:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\bin\intel64\vc11;%PATH% + - job: 'MacOS_Mojave' pool: vmImage: 'macos-10.14' diff --git a/python/perspective/MANIFEST.in b/python/perspective/MANIFEST.in index 75abbd6a97..fe339d0526 100644 --- a/python/perspective/MANIFEST.in +++ b/python/perspective/MANIFEST.in @@ -12,32 +12,19 @@ include perspective/table/*.so include perspective/table/*.dll include perspective/table/*.dylib - graft perspective/tests -# Documentation -graft docs -exclude docs/\#* - -# docs subdirs we want to skip -prune docs/build -prune docs/gh-pages -prune docs/dist - # don't ship benchmark prune bench - # Use package.json for versioning include package.json # C++ build -graft dist/cmake graft dist/src +graft dist/third graft perspective/src graft perspective/include -graft dist/test -include dist/CMakeLists.txt # Patterns to exclude from any directory global-exclude *~ diff --git a/python/perspective/perspective/tests/client/test_client.py b/python/perspective/perspective/tests/client/test_client.py index a9202175fe..2af2f9d726 100644 --- a/python/perspective/perspective/tests/client/test_client.py +++ b/python/perspective/perspective/tests/client/test_client.py @@ -9,6 +9,7 @@ import six import sys import os +import os.path import numpy as np import pandas as pd import pathlib diff --git a/python/perspective/pyproject.toml b/python/perspective/pyproject.toml index c07cca8d99..f067adf63f 100644 --- a/python/perspective/pyproject.toml +++ b/python/perspective/pyproject.toml @@ -1,3 +1,3 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel", "numpy", "pyarrow>=0.15.1,<0.17"] +requires = ["setuptools", "wheel", "numpy", "pyarrow>=0.15.1,<0.17", "pathlib ; python_version < '3'"] diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 41884cb8b5..deafe4de8b 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -7,6 +7,7 @@ # from __future__ import print_function from setuptools import setup, find_packages, Extension +from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist from codecs import open import io @@ -14,12 +15,12 @@ import os.path import platform import sys +import shutil # ******************************************** # # Get number of cores for numpy parallel build # # ******************************************** # try: - from shutil import which CPU_COUNT = os.cpu_count() except ImportError: # Python2 @@ -77,6 +78,10 @@ if sys.version_info.major < 3: requires.append("pathlib") +if os.name == 'nt' and not os.environ.get('PSP_SYSTEM_TBB', '') == '1': + # helper to avoid dll fun + requires.append('tbb==2019.0') + if (sys.version_info.major == 2 and sys.version_info.minor < 7) or \ (sys.version_info.major == 3 and sys.version_info.minor < 6): raise Exception("Requires Python 2.7/3.6 or later") @@ -232,12 +237,13 @@ def get_version(file, name='__version__'): if os.name == 'nt': defines.extend([('BOOST_WINDOWS', '1'), ('WIN32', '1'), ('_WIN32', '1'), ('PERSPECTIVE_EXPORTS', '1')]) - extra_compiler_args.append('/std:c++14') - extra_compiler_args.append('/MP') + extra_compiler_args.extend(['/std:c++14', '/MP']) runtime_library_dirs = [] + extra_objects = [] else: extra_compiler_args.append('-std=c++1y') runtime_library_dirs = pyarrow_library_dirs + extra_objects = [] extensions = [ Extension('perspective.table.libbinding', @@ -258,6 +264,7 @@ def get_version(file, name='__version__'): runtime_library_dirs=runtime_library_dirs, extra_compile_args=extra_compiler_args, extra_link_args=extra_link_args, + extra_objects=extra_objects, sources=sources + binding_sources) ] @@ -276,6 +283,29 @@ def run_check(self): raise Exception("Path is missing! {}\nMust run `yarn build_python` before building sdist so c++ files are installed".format(path)) +# *********************** # +# DLL utility for windows # +# *********************** # +if os.name == 'nt' and not os.environ.get('PSP_NO_DLL_COPY', '') == '1': + class PSPBuild(build_ext): + def run(self): + # Run the standard build + build_ext.run(self) + + print('copying dlls into build') + self._copy_dlls() + + + def _copy_dlls(self): + import pathlib + import os.path + + arrow_dlls = pathlib.Path(os.path.abspath(os.path.dirname(pyarrow.__file__))).glob("*.dll") + for file in arrow_dlls: + shutil.copy(str(file), os.path.join(os.path.dirname(self.get_ext_fullpath("perspective")), 'perspective', 'table').replace('\\', '/')) +else: + PSPBuild = build_ext + # ***** # # setup # # ***** # @@ -307,5 +337,5 @@ def run_check(self): 'dev': requires_dev, }, ext_modules=extensions, - cmdclass=dict(sdist=PSPCheckSDist) + cmdclass=dict(sdist=PSPCheckSDist, build_ext=PSPBuild) ) diff --git a/scripts/build_python.js b/scripts/build_python.js index 5ea68436c4..b07647a899 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -17,6 +17,8 @@ const IS_PY2 = getarg("--python2"); let PYTHON = IS_PY2 ? "python2" : getarg("--python38") ? "python3.8" : getarg("--python36") ? "python3.6" : "python3.7"; let IMAGE = "manylinux2010"; const IS_DOCKER = process.env.PSP_DOCKER; +const PSP_SYSTEM_TBB = process.env.PSP_SYSTEM_TBB; + if (IS_DOCKER) { // defaults to 2010 @@ -48,6 +50,7 @@ try { const dist_third_hopscotch = resolve`${__dirname}/../python/perspective/dist/third/hopscotch`; const dist_third_ordered_map = resolve`${__dirname}/../python/perspective/dist/third/ordered-map`; const dist_third_pybind11 = resolve`${__dirname}/../python/perspective/dist/third/pybind11`; + const dist_third_tbb = resolve`${__dirname}/../python/perspective/dist/third/tbb`; const third = resolve`${__dirname}/../cpp/perspective/third`; const third_boost = resolve`${__dirname}/../cpp/perspective/third/boost`; @@ -55,6 +58,7 @@ try { const third_hopscotch = resolve`${__dirname}/../cpp/perspective/third/hopscotch`; const third_ordered_map = resolve`${__dirname}/../cpp/perspective/third/ordered-map`; const third_pybind11 = resolve`${__dirname}/../cpp/perspective/third/pybind11`; + const third_tbb = resolve`${__dirname}/../cpp/perspective/third/tbb`; const cpp_src = resolve`${__dirname}/../cpp/perspective/src`; const lic = resolve`${__dirname}/../LICENSE`; @@ -108,6 +112,13 @@ try { rimraf.sync(`${third_pybind11}/.git`); console.log("Cloning pybind11...done!"); } + + if (!fs.existsSync(third_tbb)) { + console.log("Cloning tbb"); + execute`git clone https://github.com/wjakob/tbb.git ${third_tbb}`; + rimraf.sync(`${third_tbb}/.git`); + console.log("Cloning tbb...done!"); + } console.log("Cloning third party dependencies...done!"); fs.mkdirpSync(dist); @@ -147,8 +158,11 @@ try { fs.copySync(third_pybind11, dist_third_pybind11, {preserveTimestamps: true}); console.log("Copying pybind11 to python dist...done!"); } - - + if (!fs.existsSync(dist_third_tbb)) { + console.log("Copying tbb to python dist"); + fs.copySync(third_tbb, dist_third_tbb, {preserveTimestamps: true}); + console.log("Copying tbb to python dist...done!"); + } console.log("Copying LICENSE to python dist"); fs.copySync(lic, dlic, {preserveTimestamps: true}); From 6c5114b5cf62ce4465da6c54b46daec3b5d1d351 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sun, 19 Jul 2020 18:31:08 -0400 Subject: [PATCH 26/31] try to set path --- azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7e854a8c3d..e9416e00a3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -186,12 +186,13 @@ jobs: - script: yarn displayName: 'Install Deps' - - script: yarn build_python --ci $(python_flag) + - script: + set PATH=C:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\bin\intel64\vc11;%PATH% + yarn build_python --ci $(python_flag) displayName: 'build' env: CFLAGS: /IC:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\include\ LDFLAGS: /LIBPATH:C:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\lib\intel64\vc11\ - PATH: C:\ProgramData\chocolatey\lib\tbb\tools\tbb41_20130314oss\bin\intel64\vc11;%PATH% - job: 'MacOS_Mojave' pool: From a66d7b441c83a453b4d623517ff9491d25cc4e49 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sun, 19 Jul 2020 20:05:13 -0400 Subject: [PATCH 27/31] add debug ldd --- python/perspective/setup.py | 4 ++-- scripts/build_python.js | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index deafe4de8b..2bd6a2846a 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -20,9 +20,9 @@ # ******************************************** # # Get number of cores for numpy parallel build # # ******************************************** # -try: +if os.version_info.major == 3: CPU_COUNT = os.cpu_count() -except ImportError: +else: # Python2 import multiprocessing CPU_COUNT = multiprocessing.cpu_count() diff --git a/scripts/build_python.js b/scripts/build_python.js index 134bb74e52..634fdd2889 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -181,6 +181,7 @@ try { `${PYTHON} -m pip install -e .[dev] && \ ${PYTHON} setup.py build_ext --inplace && \ ${PYTHON} -m flake8 perspective && echo OK && \ + otool -L perspective/table/libbinding* && \ ${PYTHON} -m pytest -vvv perspective \ --ignore=perspective/tests/client \ --junitxml=python_junit.xml --cov-report=xml --cov-branch \ From 93c077b6060e8cd202cc81f7c79fbfbecc763e8c Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sun, 19 Jul 2020 20:06:05 -0400 Subject: [PATCH 28/31] pip install -v --- scripts/build_python.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_python.js b/scripts/build_python.js index 634fdd2889..99f84a5819 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -178,7 +178,7 @@ try { cmd = cmd + - `${PYTHON} -m pip install -e .[dev] && \ + `${PYTHON} -m pip install -v -e .[dev] && \ ${PYTHON} setup.py build_ext --inplace && \ ${PYTHON} -m flake8 perspective && echo OK && \ otool -L perspective/table/libbinding* && \ From 3675eec2e2c3050bf905b4242c125de0881e9890 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sun, 19 Jul 2020 20:14:26 -0400 Subject: [PATCH 29/31] silly error --- python/perspective/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 2bd6a2846a..b37c7076dd 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -20,7 +20,7 @@ # ******************************************** # # Get number of cores for numpy parallel build # # ******************************************** # -if os.version_info.major == 3: +if sys.version_info.major == 3: CPU_COUNT = os.cpu_count() else: # Python2 From b8a5a0c58161ca089947d0174587a8b3d342fd61 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sun, 19 Jul 2020 20:29:41 -0400 Subject: [PATCH 30/31] add another test --- scripts/build_python.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_python.js b/scripts/build_python.js index 99f84a5819..1fb78f758a 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -182,6 +182,7 @@ try { ${PYTHON} setup.py build_ext --inplace && \ ${PYTHON} -m flake8 perspective && echo OK && \ otool -L perspective/table/libbinding* && \ + ${PYTHON} -c "import perspective.table.libbinding" && \ ${PYTHON} -m pytest -vvv perspective \ --ignore=perspective/tests/client \ --junitxml=python_junit.xml --cov-report=xml --cov-branch \ From 6b7b6bbcbdad8b320ce5bf2118f6bd1ec85ad12a Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Sun, 19 Jul 2020 22:14:35 -0400 Subject: [PATCH 31/31] more debug --- python/perspective/setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index b37c7076dd..4caabd0547 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -32,6 +32,7 @@ # *************************************** # try: import numpy + print('using numpy from {}'.format(os.path.dirname(numpy.__file__))) numpy_includes = numpy.get_include() # enable numpy faster compiler @@ -49,6 +50,7 @@ # ****************** # try: import pyarrow + print('using pyarrow from {}'.format(os.path.dirname(pyarrow.__file__))) pyarrow_includes = pyarrow.get_include() pyarrow_library_dirs = pyarrow.get_library_dirs() pyarrow_libraries = pyarrow.get_libraries() @@ -225,9 +227,9 @@ def get_version(file, name='__version__'): extra_link_args = os.environ.get('LDFLAGS', '').split() if platform.system() == 'Darwin': - extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(('@loader_path//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(['@loader_path//../../pyarrow/'] + pyarrow_library_dirs)) else: - extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(('$ORIGIN//../../pyarrow/', pyarrow_library_dirs[0]))) + extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(['$ORIGIN//../../pyarrow/'] + pyarrow_library_dirs[0])) defines = [('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))]