diff --git a/.gitignore b/.gitignore index cf1e63a851..8eae080222 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ flycheck_*.el *.so *.dylib *.dll +*.pyd # Fortran module files *.mod @@ -133,6 +134,7 @@ packages/*/cjs cppbuild docsbuild .coverage +cpp/perspective/third # docs generated docs/_build diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f69f7a4ba4..e9416e00a3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -161,8 +161,13 @@ jobs: - task: NodeTool@0 inputs: + versionSpec: '12.x' + - script: | + choco install tbb -y --allow-empty-checksums + displayName: "System deps" + - script: | which python > python.txt set /p PYTHON= 2**32)" - RESULT_VARIABLE _PYTHON_RESULT - OUTPUT_VARIABLE _PYTHON_IS64BIT - ERROR_VARIABLE _PYTHON_IS64BIT) - if (NOT _PYTHON_RESULT) - if (_PYTHON_IS64BIT) - set (_PYTHON_ARCH 64) - set (_PYTHON_ARCH2 64) - else() - set (_PYTHON_ARCH 32) - set (_PYTHON_ARCH2 32) - endif() - endif() - else() - # architecture unknown, search for both 64bit and 32bit - set (_PYTHON_ARCH 64) - set (_PYTHON_ARCH2 32) - endif() -endif() - -foreach(_CURRENT_VERSION ${_Python_VERSIONS}) - string(REPLACE "." "" _CURRENT_VERSION_NO_DOTS ${_CURRENT_VERSION}) - if(WIN32) - find_library(PYTHON_DEBUG_LIBRARY - NAMES python${_CURRENT_VERSION_NO_DOTS}_d python - NAMES_PER_DIR - HINTS ${_Python_LIBRARY_PATH_HINT} - PATHS - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs/Debug - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs/Debug - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs/Debug - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs/Debug - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs/Debug - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs/Debug - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/libs - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/libs - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/libs - ) - endif() - - set(PYTHON_FRAMEWORK_LIBRARIES) - if(Python_FRAMEWORKS AND NOT PYTHON_LIBRARY) - foreach(dir ${Python_FRAMEWORKS}) - list(APPEND PYTHON_FRAMEWORK_LIBRARIES - ${dir}/Versions/${_CURRENT_VERSION}/lib) - endforeach() - endif() - # Use the library's install prefix as a hint - set(_Python_INCLUDE_PATH_HINT) - # PYTHON_LIBRARY may contain a list because of SelectLibraryConfigurations - # which may have been run previously. If it is the case, the list can be: - # optimized;;debug; - foreach(lib ${PYTHON_LIBRARY} ${PYTHON_DEBUG_LIBRARY}) - if(IS_ABSOLUTE "${lib}") - get_filename_component(_Python_PREFIX "${lib}" PATH) - get_filename_component(_Python_PREFIX "${_Python_PREFIX}" PATH) - if(_Python_PREFIX) - list(APPEND _Python_INCLUDE_PATH_HINT ${_Python_PREFIX}/include) - endif() - unset(_Python_PREFIX) - endif() - endforeach() - - # Add framework directories to the search paths - set(PYTHON_FRAMEWORK_INCLUDES) - if(Python_FRAMEWORKS AND NOT PYTHON_INCLUDE_DIR) - foreach(dir ${Python_FRAMEWORKS}) - list(APPEND PYTHON_FRAMEWORK_INCLUDES - ${dir}/Versions/${_CURRENT_VERSION}/include) - endforeach() - endif() - - find_path(PYTHON_INCLUDE_DIR - NAMES Python.h - HINTS - ${_Python_INCLUDE_PATH_HINT} - PATHS - ${PYTHON_FRAMEWORK_INCLUDES} - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/include - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/include - [HKEY_LOCAL_MACHINE\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/include - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}\\InstallPath]/include - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH}\\InstallPath]/include - [HKEY_CURRENT_USER\\SOFTWARE\\Python\\PythonCore\\${_CURRENT_VERSION}-${_PYTHON_ARCH2}\\InstallPath]/include - PATH_SUFFIXES - python${_CURRENT_VERSION}mu - python${_CURRENT_VERSION}m - python${_CURRENT_VERSION}u - python${_CURRENT_VERSION} - ) - - # For backward compatibility, set PYTHON_INCLUDE_PATH. - set(PYTHON_INCLUDE_PATH "${PYTHON_INCLUDE_DIR}") - - if(PYTHON_INCLUDE_DIR AND EXISTS "${PYTHON_INCLUDE_DIR}/patchlevel.h") - file(STRINGS "${PYTHON_INCLUDE_DIR}/patchlevel.h" python_version_str - REGEX "^#define[ \t]+PY_VERSION[ \t]+\"[^\"]+\"") - string(REGEX REPLACE "^#define[ \t]+PY_VERSION[ \t]+\"([^\"]+)\".*" "\\1" - PYTHONLIBS_VERSION_STRING "${python_version_str}") - unset(python_version_str) - endif() - - if(PYTHON_INCLUDE_DIR) - break() - endif() -endforeach() - -unset(_Python_INCLUDE_PATH_HINT) -unset(_Python_LIBRARY_PATH_HINT) - -mark_as_advanced( - PYTHON_INCLUDE_DIR -) - -# We use PYTHON_INCLUDE_DIR, PYTHON_LIBRARY and PYTHON_DEBUG_LIBRARY for the -# cache entries because they are meant to specify the location of a single -# library. We now set the variables listed by the documentation for this -# module. -set(Python_INCLUDE_DIRS "${PYTHON_INCLUDE_DIR}") -unset(PYTHON_FOUND) - -# Restore CMAKE_FIND_FRAMEWORK -if(DEFINED _PythonLibs_CMAKE_FIND_FRAMEWORK) - set(CMAKE_FIND_FRAMEWORK ${_PythonLibs_CMAKE_FIND_FRAMEWORK}) - unset(_PythonLibs_CMAKE_FIND_FRAMEWORK) -else() - unset(CMAKE_FIND_FRAMEWORK) -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(PythonHeaders - REQUIRED_VARS Python_INCLUDE_DIRS - VERSION_VAR PYTHONLIBS_VERSION_STRING) \ No newline at end of file diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index ee7298389f..ad5fb793f5 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -3,7 +3,6 @@ project(psp) set(CMAKE_BUILD_TYPE "Release") - set(CMAKE_CXX_STANDARD 14) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -92,22 +91,11 @@ find_package(Color) option(CMAKE_BUILD_TYPE "Release/Debug build" RELEASE) option(PSP_WASM_BUILD "Build the WebAssembly Project" ON) -option(PSP_CPP_BUILD "Build the C++ Project" OFF) -option(PSP_PYTHON_BUILD "Build the Python Bindings" OFF) -option(PSP_CPP_BUILD_STRICT "Build the C++ with strict warnings" OFF) -option(PSP_BUILD_DOCS "Build the Perspective documentation" OFF) if (NOT DEFINED PSP_WASM_BUILD) set(PSP_WASM_BUILD ON) - set(PSP_CPP_BUILD OFF) - set(PSP_PYTHON_BUILD OFF) -endif() - -if (PSP_WASM_BUILD AND PSP_CPP_BUILD) - message(FATAL_ERROR "${Red}CPP and Emscripten builds must be done separately${ColorReset}") endif() - if(DEFINED ENV{PSP_DEBUG}) set(CMAKE_BUILD_TYPE DEBUG) else() @@ -116,60 +104,12 @@ else() endif() endif() -if(DEFINED ENV{PSP_MANYLINUX}) - set(MANYLINUX ON) -else() - set(MANYLINUX OFF) -endif() - -if (NOT DEFINED PSP_CPP_BUILD) - set(PSP_CPP_BUILD ON) -endif() - -if (NOT DEFINED PSP_PYTHON_BUILD) - set(PSP_PYTHON_BUILD OFF) -elseif(PSP_PYTHON_BUILD) - # PSP_PYTHON_BUILD will always run PSP_CPP_BUILD - set(PSP_CPP_BUILD ON) - if(NOT DEFINED PSP_PYTHON_VERSION) - set(PSP_PYTHON_VERSION 3.7) - endif() -endif() - -if (NOT DEFINED PSP_CPP_BUILD_STRICT) - set(PSP_CPP_BUILD_STRICT OFF) -endif() - set(BUILD_MESSAGE "") -if(PSP_WASM_BUILD) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building WASM binding${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping WASM binding${ColorReset}") -endif() +set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building WASM binding${ColorReset}") if(NOT DEFINED PSP_CPP_SRC) set(PSP_CPP_SRC "${CMAKE_SOURCE_DIR}") endif() -if(PSP_CPP_BUILD) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ binding${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping C++ binding${ColorReset}") -endif() - -if (PSP_PYTHON_BUILD) - if(NOT DEFINED PSP_PYTHON_SRC) - set(PSP_PYTHON_SRC "${CMAKE_SOURCE_DIR}/../../python/perspective/perspective") - endif() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building Python ${Red}${PSP_PYTHON_VERSION}${Cyan} binding${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping Python binding${ColorReset}") -endif() - -if (PSP_CPP_BUILD AND NOT PSP_CPP_BUILD_STRICT) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Building C++ without strict warnings${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ with strict warnings${ColorReset}") -endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LOWER ) if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) @@ -179,12 +119,6 @@ else() set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building RELEASE${ColorReset}") endif() -if(PSP_BUILD_DOCS) - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building Perspective Documentation${ColorReset}") -else() - set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Skipping Perspective Documentation${ColorReset}") -endif() - message(WARNING "${BUILD_MESSAGE}") ####################### @@ -195,248 +129,88 @@ if(NOT WIN32) set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") endif() -if (PSP_WASM_BUILD) - #################### - # EMSCRIPTEN BUILD # - #################### - execute_process(COMMAND which emcc OUTPUT_VARIABLE EMCC) - execute_process(COMMAND which em++ OUTPUT_VARIABLE EMPP) - string(STRIP ${EMCC} EMCC) - string(STRIP ${EMPP} EMPP) - set(CMAKE_C_COMPILER ${EMCC}) - set(CMAKE_CXX_COMPILER ${EMPP}) - set(CMAKE_TOOLCHAIN_FILE "$ENV{EMSCRIPTEN_ROOT}/cmake/Modules/Platform/Emscripten.cmake") - set(CMAKE_AR emar) - set(CMAKE_RANLIB emranlib) - set(CMAKE_EXECUTABLE_SUFFIX ".js") - list(APPEND CMAKE_PREFIX_PATH /usr/local) - - # Assumes that Boost includes will be in this folder. - include_directories("/usr/local/include" SYSTEM) - - # Include this docker-only directory. - include_directories("/boost_includes") - - # Build Rapidjson as it is used in the minimal arrow to be built later. - psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in") - - set(EXTENDED_FLAGS " \ - --bind \ - --source-map-base ./build/ \ - --memory-init-file 0 \ - -Wall \ - -s NO_EXIT_RUNTIME=1 \ - -s NO_FILESYSTEM=1 \ - -s ALLOW_MEMORY_GROWTH=1 \ - -s MODULARIZE=1 \ - -s EXPORT_NAME=\"load_perspective\" \ - -s EXPORT_ES6=1 \ - -s MAXIMUM_MEMORY=-1 \ - -s USE_ES6_IMPORT_META=0 \ - -s EXPORTED_FUNCTIONS=\"['_main']\" \ - ") - - if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(OPT_FLAGS " \ - -O0 \ - -g4 \ - --profiling \ - -s WARN_UNALIGNED=1 \ - -Wcast-align \ - -Wover-aligned \ - -s DISABLE_EXCEPTION_CATCHING=0 \ - -s ASSERTIONS=2 \ - -s DEMANGLE_SUPPORT=1 \ - ") - else() - set(OPT_FLAGS " \ - -O3 \ - -g0 \ - -flto \ - --closure 1 \ - -s AGGRESSIVE_VARIABLE_ELIMINATION=1 \ - ") - endif() - - set(ASYNC_MODE_FLAGS "-s -s BINARYEN_ASYNC_COMPILATION=1 -s WASM=1") -elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) - if(WIN32) - if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(OPT_FLAGS " \ - /DEBUG \ - /Z7 \ - /Zi - ") - else() - set(OPT_FLAGS " \ - /NDEBUG \ - /O2 \ - ") - endif() - else() - if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(OPT_FLAGS " \ - -O0 \ - -g3 \ - ") - else() - set(OPT_FLAGS " \ - -O3 \ - -g0 \ - ") - endif() - endif() - set(SYNC_MODE_FLAGS "") - set(ASYNC_MODE_FLAGS "") - set(ASMJS_MODE_FLAGS "") - - if (WIN32) - # BOOST_ROOT has been removed on Windows VMs in Azure: - # - # - https://github.com/actions/virtual-environments/issues/687 - # - https://github.com/actions/virtual-environments/issues/319 - # - # `BOOST_ROOT` must be set in the environment, and policy CMP0074 - # must be set to `NEW` to allow BOOST_ROOT to be defined by env var - cmake_policy(SET CMP0074 NEW) - - if(DEFINED ENV{BOOST_ROOT}) - set(Boost_NO_BOOST_CMAKE TRUE) - - message(WARNING "${Cyan}BOOST_ROOT: $ENV{BOOST_ROOT} ${ColorReset}") - message(WARNING "${Cyan}BOOST_INCLUDEDIR: $ENV{BOOST_INCLUDEDIR} ${ColorReset}") - message(WARNING "${Cyan}BOOST_LIBRARYDIR: $ENV{BOOST_LIBRARYDIR} ${ColorReset}") - endif() - endif() - - find_package(Boost REQUIRED) - if(NOT Boost_FOUND) - message(FATAL_ERROR "${Red}Boost could not be located${ColorReset}") - else() - message(WARNING "${Cyan}Found Boost: `Boost_INCLUDE_DIRS`: ${Boost_INCLUDE_DIRS}, `Boost_LIBRARY_DIRS` - ${Boost_LIBRARY_DIRS} ${ColorReset}") - include_directories( ${Boost_INCLUDE_DIRS} ) - endif() - - find_package(TBB) - if(NOT TBB_FOUND) - message("${Red}TBB could not be located - building TBB from external source ${ColorReset}") - psp_build_dep("tbb" "${PSP_CMAKE_MODULE_PATH}/TBB.txt.in") - else() - message("${Cyan}Found TBB: TBB_INCLUDE_DIRS - ${TBB_INCLUDE_DIRS}, TBB_LIBRARY_DIRS - ${TBB_LIBRARY_DIRS} ${ColorReset}") - include_directories( ${TBB_INCLUDE_DIRS} ) - endif() - - if(WIN32) - foreach(warning 4244 4251 4267 4275 4290 4786 4305 4996) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd${warning}") - endforeach(warning) - # When linking against Boost on Azure, bcrypt fails to link: - # - https://github.com/microsoft/vcpkg/issues/4481 - # - https://github.com/boostorg/uuid/issues/68 - #add_definitions("-DBOOST_UUID_FORCE_AUTO_LINK") - else() - include_directories("/usr/local/include") - endif() - - if(PSP_PYTHON_BUILD) - ######################### - # PYTHON BINDINGS BUILD # - ######################### - include_directories("${PSP_PYTHON_SRC}/perspective/include") - - # Set CMP0094 to NEW - find the first version that matches constraints, - # instead of the latest version installed. - cmake_policy(SET CMP0094 NEW) - - if(MANYLINUX) - # Manylinux docker images have no shared libraries - # The instead use a statically built python. - # Cmake's default FindPython can't find the python headers - # without also finding (or failing to find) the python libraries - # so we use a custom FindPythonHeaders that is the same as the - # default, but ignores when the python libraries can't be found. - message("${Red}Manylinux build has no python shared libraries${ColorReset}") - find_package( Python ${PSP_PYTHON_VERSION} REQUIRED COMPONENTS Interpreter ) - find_package( PythonHeaders ${PSP_PYTHON_VERSION} REQUIRED) - else() - message("${Cyan}Use python shared libraries${ColorReset}") - find_package( Python ${PSP_PYTHON_VERSION} REQUIRED COMPONENTS Interpreter Development) - link_directories( ${Python_LIBRARY_DIRS} ) - endif() - message("${Cyan}Using Python_INCLUDE_DIRS: ${Python_INCLUDE_DIRS}, Python_LIBRARIES: ${Python_LIBRARIES} ${ColorReset}") - include_directories( ${Python_INCLUDE_DIRS} ) - - if(MACOS) - # on mac, use the vanilla pybind11 finder - find_package(pybind11) - if(pybind11_FOUND) - # Found PyBind installed by Homebrew - set(PYTHON_PYBIND_FOUND pybind11_FOUND) - else() - # Check if pip installed PyBind is available - find_package(Pybind) - if(PYTHON_PYBIND_FOUND) - # Need to add extra flags due to pybind weirness - # https://github.com/pybind/pybind11/blob/7830e8509f2adc97ce9ee32bf99cd4b82089cc4c/tools/pybind11Tools.cmake#L103 - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup") - endif() - endif() - - else() - # On Linux/Docker, look for pip installed PyBind only - find_package(Pybind) - endif() - - if(NOT PYTHON_PYBIND_FOUND) - message("${Red}PyBind11 could not be located - building from external source${ColorReset}") - psp_build_dep("pybind11" "${PSP_CMAKE_MODULE_PATH}/Pybind.txt.in") - else() - message("${Cyan}Found PyBind11 in ${PYTHON_PYBIND_INCLUDE_DIR}${ColorReset}") - include_directories( ${PYTHON_PYBIND_INCLUDE_DIR} ) - endif() - - find_package(NumPy) - if(NOT PYTHON_NUMPY_FOUND) - message(FATAL_ERROR "${Red}Numpy could not be located${ColorReset}") - else() - message(WARNING "${Cyan}Numpy found: ${PYTHON_NUMPY_INCLUDE_DIR}${ColorReset}") - include_directories( ${PYTHON_NUMPY_INCLUDE_DIR}) - endif() - - find_package(PyArrow REQUIRED) +#################### +# EMSCRIPTEN BUILD # +#################### +execute_process(COMMAND which emcc OUTPUT_VARIABLE EMCC) +execute_process(COMMAND which em++ OUTPUT_VARIABLE EMPP) +string(STRIP ${EMCC} EMCC) +string(STRIP ${EMPP} EMPP) +set(CMAKE_C_COMPILER ${EMCC}) +set(CMAKE_CXX_COMPILER ${EMPP}) +set(CMAKE_TOOLCHAIN_FILE "$ENV{EMSCRIPTEN_ROOT}/cmake/Modules/Platform/Emscripten.cmake") +set(CMAKE_AR emar) +set(CMAKE_RANLIB emranlib) +set(CMAKE_EXECUTABLE_SUFFIX ".js") +list(APPEND CMAKE_PREFIX_PATH /usr/local) + +# Assumes that Boost includes will be in this folder. +include_directories("/usr/local/include" SYSTEM) + +# Include this docker-only directory. +include_directories("/boost_includes") + +# Build Rapidjson as it is used in the minimal arrow to be built later. +psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in") + +set(EXTENDED_FLAGS " \ + --bind \ + --source-map-base ./build/ \ + --memory-init-file 0 \ + -Wall \ + -s NO_EXIT_RUNTIME=1 \ + -s NO_FILESYSTEM=1 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s MODULARIZE=1 \ + -s EXPORT_NAME=\"load_perspective\" \ + -s EXPORT_ES6=1 \ + -s MAXIMUM_MEMORY=-1 \ + -s USE_ES6_IMPORT_META=0 \ + -s EXPORTED_FUNCTIONS=\"['_main']\" \ + ") - if(NOT PYTHON_PYARROW_FOUND) - message(FATAL_ERROR "${Red}PyArrow could not be located${ColorReset}") - else() - message(WARNING "${Cyan}PyArrow found: PYTHON_PYARROW_INCLUDE_DIR - ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}") - message(WARNING "${Cyan}Using pre-built ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY} from: ${PYTHON_PYARROW_LIBRARY_DIR}${ColorReset}") - include_directories(${PYTHON_PYARROW_INCLUDE_DIR}) - link_directories(${PYTHON_PYARROW_LIBRARY_DIR}) - endif() - ##################### - endif() +if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) + set(OPT_FLAGS " \ + -O0 \ + -g4 \ + --profiling \ + -s WARN_UNALIGNED=1 \ + -Wcast-align \ + -Wover-aligned \ + -s DISABLE_EXCEPTION_CATCHING=0 \ + -s ASSERTIONS=2 \ + -s DEMANGLE_SUPPORT=1 \ + ") +else() + set(OPT_FLAGS " \ + -O3 \ + -g0 \ + -flto \ + --closure 1 \ + -s AGGRESSIVE_VARIABLE_ELIMINATION=1 \ + ") endif() +set(ASYNC_MODE_FLAGS "-s -s BINARYEN_ASYNC_COMPILATION=1 -s WASM=1") + # Build header-only dependencies from external source psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in") psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in") psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in") -# For WASM/CPP build, build minimal arrow from source -if (NOT PSP_PYTHON_BUILD) - # build arrow + dependencies from source for Emscripten and C++ - message("${Cyan}Building minimal Apache Arrow${ColorReset}") +# For WASM build, build minimal arrow from source +# build arrow + dependencies from source for Emscripten and C++ +message("${Cyan}Building minimal Apache Arrow${ColorReset}") - psp_build_dep("double-conversion" "${PSP_CMAKE_MODULE_PATH}/double-conversion.txt.in") - psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in") +psp_build_dep("double-conversion" "${PSP_CMAKE_MODULE_PATH}/double-conversion.txt.in") +psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in") - find_package(Flatbuffers) - if(NOT FLATBUFFERS_FOUND) - message(FATAL_ERROR"${Red}Flatbuffers could not be located${ColorReset}") - else() - message("${Cyan}Found Flatbuffers in ${FLATBUFFERS_INCLUDE_DIR}${ColorReset}") - include_directories( ${FLATBUFFERS_INCLUDE_DIR} ) - endif() +find_package(Flatbuffers) +if(NOT FLATBUFFERS_FOUND) + message(FATAL_ERROR"${Red}Flatbuffers could not be located${ColorReset}") +else() + message("${Cyan}Found Flatbuffers in ${FLATBUFFERS_INCLUDE_DIR}${ColorReset}") + include_directories( ${FLATBUFFERS_INCLUDE_DIR} ) endif() ##################### @@ -538,22 +312,6 @@ set (SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/vocab.cpp ) -set(PYTHON_SOURCE_FILES ${SOURCE_FILES} - ${PSP_PYTHON_SRC}/src/column.cpp - ) - -set (PYTHON_BINDING_SOURCE_FILES - ${PSP_PYTHON_SRC}/src/accessor.cpp - ${PSP_PYTHON_SRC}/src/computed.cpp - ${PSP_PYTHON_SRC}/src/context.cpp - ${PSP_PYTHON_SRC}/src/fill.cpp - ${PSP_PYTHON_SRC}/src/numpy.cpp - ${PSP_PYTHON_SRC}/src/python.cpp - ${PSP_PYTHON_SRC}/src/serialization.cpp - ${PSP_PYTHON_SRC}/src/table.cpp - ${PSP_PYTHON_SRC}/src/utils.cpp - ${PSP_PYTHON_SRC}/src/view.cpp - ) if (WIN32) set(CMAKE_CXX_FLAGS " /EHsc /MP") @@ -561,146 +319,14 @@ else() set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS}") endif() -if (PSP_WASM_BUILD) - add_library(psp ${SOURCE_FILES}) - target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1) - set_target_properties(psp PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") - target_link_libraries(psp arrow) - - add_executable(perspective.async src/cpp/emscripten.cpp) - target_link_libraries(perspective.async psp "${ASYNC_MODE_FLAGS}") - target_compile_definitions(perspective.async PRIVATE PSP_ENABLE_WASM=1) - set_target_properties(perspective.async PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") - set_target_properties(perspective.async PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./build/") - set_target_properties(perspective.async PROPERTIES OUTPUT_NAME "psp.async") -elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) - if(NOT WIN32) - set(CMAKE_SHARED_LIBRARY_SUFFIX .so) - # Look for the binary using @loader_path (relative to binary location) instead of @rpath - # and include arrow in @rpath so it can be found by libbinding/libpsp - set(CMAKE_MACOSX_RPATH TRUE) - set(CMAKE_SKIP_BUILD_RPATH FALSE) - set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - set(CMAKE_INSTALL_NAME_DIR "@rpath/") - - # module_origin_path is the location of the binary - if(MACOS) - set(module_origin_path "@loader_path/") - else() - set(module_origin_path "\$ORIGIN") - endif() - else() - set(CMAKE_SHARED_LIBRARY_PREFIX lib) - endif() - - if(PSP_PYTHON_BUILD) - ######################## - # Python extra targets # - ######################## - add_library(psp SHARED ${PYTHON_SOURCE_FILES}) - add_library(binding SHARED ${PYTHON_BINDING_SOURCE_FILES}) - - include_directories(${PSP_PYTHON_SRC}/include) - - target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1) - target_compile_definitions(binding PRIVATE PSP_ENABLE_PYTHON=1) - - if (WIN32) - target_compile_definitions(binding PRIVATE WIN32=1) - target_compile_definitions(binding PRIVATE _WIN32=1) - - # .dll not importable - set_property(TARGET binding PROPERTY SUFFIX .pyd) - else() - target_compile_options(binding PRIVATE -Wdeprecated-declarations) - # Add a relative path to search for PyArrow - when Perspective is - # installed from a wheel, PyArrow may not be in the same directory - # as the PyArrow which was used to build the wheel. Assuming that - # both Pyarrow and Perspective are installed in `site-packages`, - # the relative search path should be able to pick up pyarrow. - set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR}) - set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} "${module_origin_path}/../../pyarrow" ${PYTHON_PYARROW_LIBRARY_DIR}) - endif() - - target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES}) - target_link_libraries(binding ${PYTHON_PYARROW_LIBRARIES}) - - if(WIN32) - # Don't link - else() - target_link_libraries(psp ${PYTHON_LIBRARIES}) - target_link_libraries(binding ${PYTHON_LIBRARIES}) - endif() - - target_link_libraries(psp tbb) - target_link_libraries(binding tbb) - - target_link_libraries(binding psp) - - # The compiled libraries will be put in CMAKE_LIBRARY_OUTPUT_DIRECTORY by default. In the - # setup.py file, we designate this to be in the build/lib. directory. However, - # since we want to be able to test perspective in-source, we also copy the libraries into - # the source folder. These two commands do that. - add_custom_command(TARGET psp POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) - add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) - - if(WIN32) - # inline arrow dlls - file(GLOB ARROW_DLLS "${PYTHON_PYARROW_LIBRARY_DIR}/*.dll") - - add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_DLLS} ${PSP_PYTHON_SRC}/table/) - if(NOT TBB_FOUND) - add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) - endif() - endif() - ######################## - else() - add_library(psp SHARED ${SOURCE_FILES}) - - # Link perspective against custom-built minimal arrow - target_link_libraries(psp arrow) - endif() - - if(PSP_CPP_BUILD_STRICT AND NOT WIN32) - target_compile_options(psp PRIVATE -Wall -Werror) - target_compile_options(psp PRIVATE $<$:-fPIC -O0>) - if(PSP_PYTHON_BUILD) - target_compile_options(binding PRIVATE $<$:-fPIC -O0>) - endif() - elseif(WIN32) - target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1) - target_compile_definitions(psp PRIVATE WIN32=1) - target_compile_definitions(psp PRIVATE _WIN32=1) - endif() -endif() - -######## -# Docs # -######## -if(PSP_BUILD_DOCS) - add_custom_target(doxygen) - add_custom_command(TARGET doxygen - COMMAND doxygen doxygen.conf - WORKING_DIRECTORY ../../docs - COMMENT "Build doxygen xml files used by sphinx/breathe." - ) - - add_custom_target(docs-html ALL) - add_custom_command(TARGET docs-html - COMMAND sphinx-build -b html . build/html - WORKING_DIRECTORY ../../docs - COMMENT "Build html documentation" - ) - add_dependencies(docs-html doxygen) - - # add_custom_target(docs-markdown ALL) - # add_custom_command(TARGET docs-html - # COMMAND sphinx-build -M markdown . build/ - # WORKING_DIRECTORY ../../docs - # COMMENT "Build markdown documentation" - # ) - # add_dependencies(docs-html doxygen) - -endif() -########## +add_library(psp ${SOURCE_FILES}) +target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1) +set_target_properties(psp PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") +target_link_libraries(psp arrow) + +add_executable(perspective.async src/cpp/emscripten.cpp) +target_link_libraries(perspective.async psp "${ASYNC_MODE_FLAGS}") +target_compile_definitions(perspective.async PRIVATE PSP_ENABLE_WASM=1) +set_target_properties(perspective.async PROPERTIES COMPILE_FLAGS "${ASYNC_MODE_FLAGS}") +set_target_properties(perspective.async PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./build/") +set_target_properties(perspective.async PROPERTIES OUTPUT_NAME "psp.async") \ No newline at end of file diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile index 537c6e9395..2e38c93370 100644 --- a/docker/python/manylinux2010/Dockerfile +++ b/docker/python/manylinux2010/Dockerfile @@ -7,7 +7,7 @@ # FROM quay.io/pypa/manylinux2010_x86_64 -RUN yum -y install rapidjson-devel sudo wget +RUN yum -y install rapidjson-devel sudo wget tbb-devel # Build new cmake RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q @@ -26,7 +26,6 @@ RUN rm -rf /opt/python/cp36-cp36m/bin/auditwheel RUN rm -rf /opt/python/cp37-cp37m/bin/auditwheel RUN rm -rf /opt/python/cp38-cp38/bin/auditwheel - # Copy assets RUN cp -arf /opt/python/cp27-cp27m/* /usr/local/ RUN cp -arf /opt/python/cp36-cp36m/* /usr/local/ @@ -47,13 +46,7 @@ RUN python3.6 -m pip install --ignore-installed auditwheel RUN python3.7 -m pip install --ignore-installed auditwheel RUN python3.8 -m pip install --ignore-installed auditwheel -# install boost -RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 -RUN tar xfz boost_1_71_0.tar.gz -# https://github.com/boostorg/build/issues/468 -RUN cd boost_1_71_0 && ./bootstrap.sh -RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install - +# install pinned dependencies RUN python2.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' git+https://chromium.googlesource.com/external/gyp RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' @@ -73,14 +66,3 @@ RUN yum install -y nodejs RUN npm install --global yarn RUN yarn --version - -# install flatbuffers -RUN mkdir -p /usr/local \ - && cd /usr/local \ - && git clone https://github.com/google/flatbuffers.git \ - && cd flatbuffers \ - && cmake -G "Unix Makefiles" \ - && make \ - && cp -r /usr/local/flatbuffers/include/flatbuffers /usr/local/include \ - && ln -s /usr/local/flatbuffers/flatc /usr/local/bin/flatc \ - && chmod +x /usr/local/flatbuffers/flatc diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile index a0e21d4587..46396f08f9 100644 --- a/docker/python/manylinux2014/Dockerfile +++ b/docker/python/manylinux2014/Dockerfile @@ -7,7 +7,7 @@ # FROM quay.io/pypa/manylinux2014_x86_64 -RUN yum -y install rapidjson-devel sudo wget +RUN yum -y install rapidjson-devel sudo wget tbb-devel # Build new cmake RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q @@ -43,13 +43,7 @@ RUN python3.6 -m pip install --ignore-installed auditwheel RUN python3.7 -m pip install --ignore-installed auditwheel RUN python3.8 -m pip install --ignore-installed auditwheel -# install boost -RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 -RUN tar xfz boost_1_71_0.tar.gz -# https://github.com/boostorg/build/issues/468 -RUN cd boost_1_71_0 && ./bootstrap.sh -RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install - +# install pinned dependencies RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' @@ -67,14 +61,3 @@ RUN yum install -y nodejs RUN npm install --global yarn RUN yarn --version - -# install flatbuffers -RUN mkdir -p /usr/local \ - && cd /usr/local \ - && git clone https://github.com/google/flatbuffers.git \ - && cd flatbuffers \ - && cmake -G "Unix Makefiles" \ - && make \ - && cp -r /usr/local/flatbuffers/include/flatbuffers /usr/local/include \ - && ln -s /usr/local/flatbuffers/flatc /usr/local/bin/flatc \ - && chmod +x /usr/local/flatbuffers/flatc diff --git a/docs/md/development.md b/docs/md/development.md index 0c449e138c..088e5333ba 100644 --- a/docs/md/development.md +++ b/docs/md/development.md @@ -61,8 +61,8 @@ built from source: - Boost (version 1.67) - CMake (version 3.15.4 or higher) -- TBB -- [Flatbuffers](https://google.github.io/flatbuffers/flatbuffers_guide_building.html) +- TBB (when building python) +- [Flatbuffers](https://google.github.io/flatbuffers/flatbuffers_guide_building.html) (when building JS) ### `Perspective.js` @@ -98,16 +98,31 @@ To build the Python library, navigate to the python source folder (`python/perspective`) and install the dependencies using `pip`: ```bash -python3 -m pip install -r requirements.txt -python3 -m pip install -r requirements-dev.txt +python -m pip install numpy "pyarrow>=0.15.1,<0.17" ``` Make sure that TBB is installed, as it is a system dependency: +OS X ```bash brew install TBB ``` +Debian +```bash +apt-get install libtbb-devel +``` + +RHEL +```bash +yum install tbb-devel +``` + +Windows +```bash +choco install tbb +``` + `perspective-python` supports Python 3.7 and upwards, as well as Python 2.7.17. To build the Python 2 version of the library, use the `--python2` flag: @@ -131,13 +146,18 @@ You'll also need the system dependencies noted earlier - installing them through Homebrew is the easiest way, especially as flatbuffers can be installed through `brew` without building from source: +Javascript ```bash -brew install boost@1.67 -brew install tbb brew install cmake +brew install boost@1.67 brew install flatbuffers ``` +Python +```bash +brew install tbb +``` + If building the Python 2 version of the library, make sure your version of Python 2 is the latest version (`2.7.17`) supplied by Homebrew, and not the earlier version that ships with MacOS. To install Python 2 using Homebrew: @@ -148,7 +168,7 @@ brew install python2 ### Windows 10 specific instructions -You need to use bash in order to build Perspective packages. To successfully +You need to use bash in order to build Perspective JS packages. To successfully build on Windows 10, enable [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) (WSL) and install the linux distribution of your choice. diff --git a/package.json b/package.json index b2e3354bfd..3a8f6d4ee5 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "sinon": "^7.3.1", "source-map-explorer": "^2.0.1", "style-loader": "^0.18.2", + "tar": "^6.0.2", "term-img": "^4.1.0", "ts-jest": "^25.1.0", "ts-loader": "^6.2.0", diff --git a/python/perspective/MANIFEST.in b/python/perspective/MANIFEST.in index 75abbd6a97..fe339d0526 100644 --- a/python/perspective/MANIFEST.in +++ b/python/perspective/MANIFEST.in @@ -12,32 +12,19 @@ include perspective/table/*.so include perspective/table/*.dll include perspective/table/*.dylib - graft perspective/tests -# Documentation -graft docs -exclude docs/\#* - -# docs subdirs we want to skip -prune docs/build -prune docs/gh-pages -prune docs/dist - # don't ship benchmark prune bench - # Use package.json for versioning include package.json # C++ build -graft dist/cmake graft dist/src +graft dist/third graft perspective/src graft perspective/include -graft dist/test -include dist/CMakeLists.txt # Patterns to exclude from any directory global-exclude *~ diff --git a/python/perspective/perspective/include/perspective/python/numpy.h b/python/perspective/perspective/include/perspective/python/numpy.h index 034fd8bba5..6851e4b5fe 100644 --- a/python/perspective/perspective/include/perspective/python/numpy.h +++ b/python/perspective/perspective/include/perspective/python/numpy.h @@ -18,12 +18,8 @@ #include #ifdef WIN32 -#ifndef PERSPECTIVE_EXPORTS #define PERSPECTIVE_BINDING_EXPORT __declspec(dllexport) #else -#define PERSPECTIVE_BINDING_EXPORT __declspec(dllimport) -#endif -#else #define PERSPECTIVE_BINDING_EXPORT __attribute__((visibility("default"))) #endif diff --git a/python/perspective/perspective/tests/client/test_client.py b/python/perspective/perspective/tests/client/test_client.py index ebad2f7d3c..2af2f9d726 100644 --- a/python/perspective/perspective/tests/client/test_client.py +++ b/python/perspective/perspective/tests/client/test_client.py @@ -7,27 +7,29 @@ # import six +import sys import os +import os.path import numpy as np import pandas as pd +import pathlib from datetime import date, datetime from functools import partial from types import MethodType -if os.name == 'nt': - BINDING = 'libbinding.pyd' - PSP = 'libpsp.dll' -else: - BINDING = 'libbinding.so' - PSP = 'libpsp.so' - # rename libbinding.so and libpsp.so temporarily to ensure that client mode # works automatically when the C++ build fails. lib_path = os.path.join(os.path.dirname(__file__), "..", "..", "table") -binding = os.path.join(lib_path, BINDING) -psp = os.path.join(lib_path, PSP) + +if os.name == 'nt': + BINDING = 'libbinding*.pyd' +else: + BINDING = 'libbinding*.so' + + + +binding = os.path.join(str(list(pathlib.Path(lib_path).glob(BINDING))[0])) new_binding = os.path.join(lib_path, "notlibbinding.so") -new_psp = os.path.join(lib_path, "notlibpsp.so") def mock_post(self, msg, msg_id=None, assert_msg=None): @@ -37,20 +39,14 @@ def mock_post(self, msg, msg_id=None, assert_msg=None): def setup_module(): os.rename(binding, new_binding) - os.rename(psp, new_psp) assert os.path.exists(new_binding) - assert os.path.exists(new_psp) assert not os.path.exists(binding) - assert not os.path.exists(psp) def teardown_module(): os.rename(new_binding, binding) - os.rename(new_psp, psp) assert os.path.exists(binding) - assert os.path.exists(psp) assert not os.path.exists(new_binding) - assert not os.path.exists(new_psp) class TestClient(object): diff --git a/python/perspective/pyproject.toml b/python/perspective/pyproject.toml index 4114696901..f067adf63f 100644 --- a/python/perspective/pyproject.toml +++ b/python/perspective/pyproject.toml @@ -1,3 +1,3 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools", "wheel"] +requires = ["setuptools", "wheel", "numpy", "pyarrow>=0.15.1,<0.17", "pathlib ; python_version < '3'"] diff --git a/python/perspective/setup.py b/python/perspective/setup.py index d0bbda195e..4caabd0547 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -9,48 +9,80 @@ from setuptools import setup, find_packages, Extension from setuptools.command.build_ext import build_ext from setuptools.command.sdist import sdist -from distutils.version import LooseVersion -from distutils import sysconfig from codecs import open import io -import logging import os import os.path -import re import platform import sys -import subprocess -from shutil import rmtree -try: - from shutil import which +import shutil + +# ******************************************** # +# Get number of cores for numpy parallel build # +# ******************************************** # +if sys.version_info.major == 3: CPU_COUNT = os.cpu_count() -except ImportError: +else: # Python2 - try: - from backports.shutil_which import which - except ImportError: - which = lambda x: x # just rely on path import multiprocessing CPU_COUNT = multiprocessing.cpu_count() +# *************************************** # +# Numpy build path and compiler toolchain # +# *************************************** # +try: + import numpy + print('using numpy from {}'.format(os.path.dirname(numpy.__file__))) + numpy_includes = numpy.get_include() + + # enable numpy faster compiler + from numpy.distutils.ccompiler import CCompiler_compile + import distutils.ccompiler + distutils.ccompiler.CCompiler.compile = CCompiler_compile + os.environ['NPY_NUM_BUILD_JOBS'] = str(CPU_COUNT) + +except ImportError: + print('Must install numpy prior to installing perspective!') + raise + +# ****************** # +# PyArrow build path # +# ****************** # +try: + import pyarrow + print('using pyarrow from {}'.format(os.path.dirname(pyarrow.__file__))) + pyarrow_includes = pyarrow.get_include() + pyarrow_library_dirs = pyarrow.get_library_dirs() + pyarrow_libraries = pyarrow.get_libraries() +except ImportError: + print('Must install pyarrow prior to installing perspective!') + raise + here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() +# ******************************** # +# Requires and platform validation # +# ******************************** # requires = [ 'ipywidgets>=7.5.1', 'future>=0.16.0', 'numpy>=1.13.1', 'pandas>=0.22.0', - 'pyarrow==0.16.0', + 'pyarrow>=0.15.1,<0.17', 'python-dateutil>=2.8.0', 'six>=1.11.0', 'traitlets>=4.3.2', ] if sys.version_info.major < 3: - requires.append("backports.shutil-which") + requires.append("pathlib") + +if os.name == 'nt' and not os.environ.get('PSP_SYSTEM_TBB', '') == '1': + # helper to avoid dll fun + requires.append('tbb==2019.0') if (sys.version_info.major == 2 and sys.version_info.minor < 7) or \ (sys.version_info.major == 3 and sys.version_info.minor < 6): @@ -84,104 +116,201 @@ def get_version(file, name='__version__'): version = get_version(os.path.join(here, 'perspective', 'core', '_version.py')) -class PSPExtension(Extension): - def __init__(self, name, sourcedir='dist'): - Extension.__init__(self, name, sources=[]) - self.sourcedir = os.path.abspath(sourcedir) - +# *********** # +# C++ sources # +# *********** # +sources = [ + 'dist/src/cpp/aggregate.cpp', + 'dist/src/cpp/aggspec.cpp', + 'dist/src/cpp/arg_sort.cpp', + 'dist/src/cpp/arrow_loader.cpp', + 'dist/src/cpp/arrow_writer.cpp', + 'dist/src/cpp/base.cpp', + 'dist/src/cpp/base_impl_linux.cpp', + 'dist/src/cpp/base_impl_osx.cpp', + 'dist/src/cpp/base_impl_wasm.cpp', + 'dist/src/cpp/base_impl_win.cpp', + 'dist/src/cpp/binding.cpp', + 'dist/src/cpp/build_filter.cpp', + 'dist/src/cpp/column.cpp', + 'dist/src/cpp/comparators.cpp', + 'dist/src/cpp/compat.cpp', + 'dist/src/cpp/compat_impl_linux.cpp', + 'dist/src/cpp/compat_impl_osx.cpp', + 'dist/src/cpp/compat_impl_win.cpp', + 'dist/src/cpp/computed.cpp', + 'dist/src/cpp/computed_column_map.cpp', + 'dist/src/cpp/computed_function.cpp', + 'dist/src/cpp/config.cpp', + 'dist/src/cpp/context_base.cpp', + 'dist/src/cpp/context_grouped_pkey.cpp', + 'dist/src/cpp/context_handle.cpp', + 'dist/src/cpp/context_one.cpp', + 'dist/src/cpp/context_two.cpp', + 'dist/src/cpp/context_zero.cpp', + 'dist/src/cpp/custom_column.cpp', + 'dist/src/cpp/data.cpp', + 'dist/src/cpp/data_slice.cpp', + 'dist/src/cpp/data_table.cpp', + 'dist/src/cpp/date.cpp', + 'dist/src/cpp/dense_nodes.cpp', + 'dist/src/cpp/dense_tree_context.cpp', + 'dist/src/cpp/dense_tree.cpp', + 'dist/src/cpp/dependency.cpp', + 'dist/src/cpp/extract_aggregate.cpp', + 'dist/src/cpp/filter.cpp', + 'dist/src/cpp/flat_traversal.cpp', + 'dist/src/cpp/get_data_extents.cpp', + 'dist/src/cpp/gnode.cpp', + 'dist/src/cpp/gnode_state.cpp', + 'dist/src/cpp/histogram.cpp', + 'dist/src/cpp/logtime.cpp', + 'dist/src/cpp/mask.cpp', + 'dist/src/cpp/min_max.cpp', + 'dist/src/cpp/multi_sort.cpp', + 'dist/src/cpp/none.cpp', + 'dist/src/cpp/path.cpp', + 'dist/src/cpp/pivot.cpp', + 'dist/src/cpp/pool.cpp', + 'dist/src/cpp/port.cpp', + 'dist/src/cpp/process_state.cpp', + 'dist/src/cpp/raii.cpp', + 'dist/src/cpp/raii_impl_linux.cpp', + 'dist/src/cpp/raii_impl_osx.cpp', + 'dist/src/cpp/raii_impl_win.cpp', + 'dist/src/cpp/range.cpp', + 'dist/src/cpp/rlookup.cpp', + 'dist/src/cpp/scalar.cpp', + 'dist/src/cpp/schema_column.cpp', + 'dist/src/cpp/schema.cpp', + 'dist/src/cpp/slice.cpp', + 'dist/src/cpp/sort_specification.cpp', + 'dist/src/cpp/sparse_tree.cpp', + 'dist/src/cpp/sparse_tree_node.cpp', + 'dist/src/cpp/step_delta.cpp', + 'dist/src/cpp/storage.cpp', + 'dist/src/cpp/storage_impl_linux.cpp', + 'dist/src/cpp/storage_impl_osx.cpp', + 'dist/src/cpp/storage_impl_win.cpp', + 'dist/src/cpp/sym_table.cpp', + 'dist/src/cpp/table.cpp', + 'dist/src/cpp/time.cpp', + 'dist/src/cpp/traversal.cpp', + 'dist/src/cpp/traversal_nodes.cpp', + 'dist/src/cpp/tree_context_common.cpp', + 'dist/src/cpp/utils.cpp', + 'dist/src/cpp/update_task.cpp', + 'dist/src/cpp/view.cpp', + 'dist/src/cpp/view_config.cpp', + 'dist/src/cpp/vocab.cpp', + + # python-specific files + 'perspective/src/column.cpp', +] -class PSPBuild(build_ext): - def run(self): - self.run_cmake() - - def run_cmake(self): - self.cmake_cmd = which('cmake') - try: - out = subprocess.check_output([self.cmake_cmd, '--version']) - except OSError: - raise RuntimeError( - "CMake must be installed to build the following extensions: " + - ", ".join(e.name for e in self.extensions)) - - if platform.system() == "Windows": - cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', - out.decode()).group(1)) - if cmake_version < '3.1.0': - raise RuntimeError("CMake >= 3.1.0 is required on Windows") - - for ext in self.extensions: - self.build_extension_cmake(ext) - - def build_extension_cmake(self, ext): - extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) - cfg = 'Debug' if self.debug else 'Release' - - PYTHON_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor) - - cmake_args = [ - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + os.path.abspath(os.path.join(extdir, 'perspective', 'table')).replace('\\', '/'), - '-DCMAKE_BUILD_TYPE=' + cfg, - '-DPSP_CPP_BUILD=1', - '-DPSP_WASM_BUILD=0', - '-DPSP_PYTHON_BUILD=1', - '-DPSP_PYTHON_VERSION={}'.format(PYTHON_VERSION), - '-DPython_ADDITIONAL_VERSIONS={}'.format(PYTHON_VERSION), - '-DPython_FIND_VERSION={}'.format(PYTHON_VERSION), - '-DPython_EXECUTABLE={}'.format(sys.executable).replace('\\', '/'), - '-DPython_ROOT_DIR={}'.format(sysconfig.PREFIX).replace('\\', '/'), - '-DPython_ROOT={}'.format(sysconfig.PREFIX).replace('\\', '/'), - '-DPSP_CMAKE_MODULE_PATH={folder}'.format(folder=os.path.join(ext.sourcedir, 'cmake')).replace('\\', '/'), - '-DPSP_CPP_SRC={folder}'.format(folder=ext.sourcedir).replace('\\', '/'), - '-DPSP_PYTHON_SRC={folder}'.format(folder=os.path.join(ext.sourcedir, "..", 'perspective').replace('\\', '/')) - ] - - build_args = ['--config', cfg] - - if platform.system() == "Windows": - import distutils.msvccompiler as dm - msvc = {'12': 'Visual Studio 12 2013', - '14': 'Visual Studio 14 2015', - '14.1': 'Visual Studio 15 2017'}.get(dm.get_build_version(), 'Visual Studio 15 2017') - - cmake_args.extend([ - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format( - cfg.upper(), - extdir).replace('\\', '/'), - '-G', os.environ.get('PSP_GENERATOR', msvc)]) - - if sys.maxsize > 2**32: - # build 64 bit to match python - cmake_args += ['-A', 'x64'] - - build_args += ['--', '/m:{}'.format(CPU_COUNT), '/p:Configuration={}'.format(cfg)] - else: - cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] - build_args += ['--', '-j2' if os.environ.get('DOCKER', '') else '-j{}'.format(CPU_COUNT)] - - env = os.environ.copy() - env['PSP_ENABLE_PYTHON'] = '1' - env['OSX_DEPLOYMENT_TARGET'] = '10.9' - env["PYTHONPATH"] = os.path.sep.join((os.environ.get('PYTHONPATH', ''), os.path.pathsep.join((os.path.join(os.path.dirname(os.__file__), 'site-packages'), os.path.dirname(os.__file__))))) - - if not os.path.exists(self.build_temp): - os.makedirs(self.build_temp) - subprocess.check_call([self.cmake_cmd, os.path.abspath(ext.sourcedir)] + cmake_args, cwd=self.build_temp, env=env, stderr=subprocess.STDOUT) - subprocess.check_call([self.cmake_cmd, '--build', '.'] + build_args, cwd=self.build_temp, env=env, stderr=subprocess.STDOUT) - print() # Add an empty line for cleaner output +binding_sources = [ + 'perspective/src/accessor.cpp', + 'perspective/src/computed.cpp', + 'perspective/src/context.cpp', + 'perspective/src/fill.cpp', + 'perspective/src/numpy.cpp', + 'perspective/src/python.cpp', + 'perspective/src/serialization.cpp', + 'perspective/src/table.cpp', + 'perspective/src/utils.cpp', + 'perspective/src/view.cpp', +] +# **************************** # +# Compiler and extension setup # +# **************************** # +extra_link_args = os.environ.get('LDFLAGS', '').split() + +if platform.system() == 'Darwin': + extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(['@loader_path//../../pyarrow/'] + pyarrow_library_dirs)) +else: + extra_link_args.append('-Wl,-rpath,' + ',-rpath,'.join(['$ORIGIN//../../pyarrow/'] + pyarrow_library_dirs[0])) + + +defines = [('PSP_ENABLE_PYTHON', '1'), ('PSP_DEBUG', os.environ.get('PSP_DEBUG', '0'))] +extra_compiler_args = os.environ.get('CFLAGS', '').split() + os.environ.get('CXXFLAGS', '').split() +library_dirs = pyarrow_library_dirs +libraries = pyarrow_libraries + ['tbb'] + +if os.name == 'nt': + defines.extend([('BOOST_WINDOWS', '1'), ('WIN32', '1'), ('_WIN32', '1'), ('PERSPECTIVE_EXPORTS', '1')]) + extra_compiler_args.extend(['/std:c++14', '/MP']) + runtime_library_dirs = [] + extra_objects = [] +else: + extra_compiler_args.append('-std=c++1y') + runtime_library_dirs = pyarrow_library_dirs + extra_objects = [] + +extensions = [ + Extension('perspective.table.libbinding', + define_macros=defines, + include_dirs=[ + 'perspective/include/', + 'dist/src/include/', + 'dist/third/boost/boost_1_71_0/', + 'dist/third/date/include/', + 'dist/third/hopscotch/include/', + 'dist/third/ordered-map/include/', + 'dist/third/pybind11/include/', + numpy_includes, + pyarrow_includes, + ], + libraries=libraries, + library_dirs=library_dirs, + runtime_library_dirs=runtime_library_dirs, + extra_compile_args=extra_compiler_args, + extra_link_args=extra_link_args, + extra_objects=extra_objects, + sources=sources + binding_sources) +] +# **************** # +# SDist validation # +# **************** # class PSPCheckSDist(sdist): def run(self): self.run_check() super(PSPCheckSDist, self).run() def run_check(self): - for file in ('CMakeLists.txt', 'cmake', 'src'): + for file in ('src', 'third'): path = os.path.abspath(os.path.join(here, 'dist', file)) if not os.path.exists(path): - raise Exception("Path is missing! {}\nMust run `yarn build_python` before building sdist so cmake files are installed".format(path)) + raise Exception("Path is missing! {}\nMust run `yarn build_python` before building sdist so c++ files are installed".format(path)) + +# *********************** # +# DLL utility for windows # +# *********************** # +if os.name == 'nt' and not os.environ.get('PSP_NO_DLL_COPY', '') == '1': + class PSPBuild(build_ext): + def run(self): + # Run the standard build + build_ext.run(self) + print('copying dlls into build') + self._copy_dlls() + + + def _copy_dlls(self): + import pathlib + import os.path + + arrow_dlls = pathlib.Path(os.path.abspath(os.path.dirname(pyarrow.__file__))).glob("*.dll") + for file in arrow_dlls: + shutil.copy(str(file), os.path.join(os.path.dirname(self.get_ext_fullpath("perspective")), 'perspective', 'table').replace('\\', '/')) +else: + PSPBuild = build_ext + +# ***** # +# setup # +# ***** # setup( name='perspective-python', version=version, @@ -201,7 +330,6 @@ def run_check(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', ], - keywords='analytics tools plotting', packages=find_packages(), include_package_data=True, @@ -210,6 +338,6 @@ def run_check(self): extras_require={ 'dev': requires_dev, }, - ext_modules=[PSPExtension('perspective')], - cmdclass=dict(build_ext=PSPBuild, sdist=PSPCheckSDist) + ext_modules=extensions, + cmdclass=dict(sdist=PSPCheckSDist, build_ext=PSPBuild) ) diff --git a/scripts/build_python.js b/scripts/build_python.js index b4262845e8..1fb78f758a 100644 --- a/scripts/build_python.js +++ b/scripts/build_python.js @@ -9,12 +9,16 @@ const {execute, execute_throw, docker, clean, resolve, getarg, bash, python_image} = require("./script_utils.js"); const fs = require("fs-extra"); +const rimraf = require("rimraf"); +const tar = require("tar"); const IS_PY2 = getarg("--python2"); let PYTHON = IS_PY2 ? "python2" : getarg("--python38") ? "python3.8" : getarg("--python36") ? "python3.6" : "python3.7"; let IMAGE = "manylinux2010"; const IS_DOCKER = process.env.PSP_DOCKER; +const PSP_SYSTEM_TBB = process.env.PSP_SYSTEM_TBB; + if (IS_DOCKER) { // defaults to 2010 @@ -39,17 +43,130 @@ try { try { const dist = resolve`${__dirname}/../python/perspective/dist`; - const cpp = resolve`${__dirname}/../cpp/perspective`; + const dist_src = resolve`${__dirname}/../python/perspective/dist/src`; + const dist_third = resolve`${__dirname}/../python/perspective/dist/third`; + const dist_third_boost = resolve`${__dirname}/../python/perspective/dist/third/boost`; + const dist_third_date = resolve`${__dirname}/../python/perspective/dist/third/date`; + const dist_third_hopscotch = resolve`${__dirname}/../python/perspective/dist/third/hopscotch`; + const dist_third_ordered_map = resolve`${__dirname}/../python/perspective/dist/third/ordered-map`; + const dist_third_pybind11 = resolve`${__dirname}/../python/perspective/dist/third/pybind11`; + const dist_third_tbb = resolve`${__dirname}/../python/perspective/dist/third/tbb`; + + const third = resolve`${__dirname}/../cpp/perspective/third`; + const third_boost = resolve`${__dirname}/../cpp/perspective/third/boost`; + const third_date = resolve`${__dirname}/../cpp/perspective/third/date`; + const third_hopscotch = resolve`${__dirname}/../cpp/perspective/third/hopscotch`; + const third_ordered_map = resolve`${__dirname}/../cpp/perspective/third/ordered-map`; + const third_pybind11 = resolve`${__dirname}/../cpp/perspective/third/pybind11`; + const third_tbb = resolve`${__dirname}/../cpp/perspective/third/tbb`; + + const cpp_src = resolve`${__dirname}/../cpp/perspective/src`; const lic = resolve`${__dirname}/../LICENSE`; - const cmake = resolve`${__dirname}/../cmake`; - const dcmake = resolve`${dist}/cmake`; const dlic = resolve`${dist}/LICENSE`; const obj = resolve`${dist}/obj`; + // clone third party deps + console.log("Cloning third party dependencies"); + if (!fs.existsSync(third)) { + fs.mkdirpSync(third); + } + + if (!fs.existsSync(third_boost)) { + console.log("Downloading boost"); + fs.mkdirpSync(third_boost); + + const tarball = resolve`${third_boost}/boost.tgz`; + execute`curl -L https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz -o ${tarball}`; + console.log("Downloading boost...done!"); + + console.log("Extracting boost"); + tar.x({sync: true, file: `${tarball}`, cwd: `${third_boost}`}); + rimraf.sync(`${tarball}`); + console.log("Extracting boost...done!"); + } + + if (!fs.existsSync(third_date)) { + console.log("Cloning date"); + execute`git clone https://github.com/HowardHinnant/date.git ${third_date}`; + rimraf.sync(`${third_date}/.git`); + console.log("Cloning date...done!"); + } + + if (!fs.existsSync(third_hopscotch)) { + console.log("Cloning hopscotch"); + execute`git clone https://github.com/Tessil/hopscotch-map.git ${third_hopscotch}`; + rimraf.sync(`${third_hopscotch}/.git`); + console.log("Cloning hopscotch...done!"); + } + + if (!fs.existsSync(third_ordered_map)) { + console.log("Cloning ordered-map"); + execute`git clone https://github.com/Tessil/ordered-map.git ${third_ordered_map}`; + rimraf.sync(`${third_ordered_map}/.git`); + console.log("Cloning ordered-map...done!"); + } + + if (!fs.existsSync(third_pybind11)) { + console.log("Cloning pybind11"); + execute`git clone https://github.com/pybind/pybind11.git ${third_pybind11}`; + rimraf.sync(`${third_pybind11}/.git`); + console.log("Cloning pybind11...done!"); + } + + if (!fs.existsSync(third_tbb)) { + console.log("Cloning tbb"); + execute`git clone https://github.com/wjakob/tbb.git ${third_tbb}`; + rimraf.sync(`${third_tbb}/.git`); + console.log("Cloning tbb...done!"); + } + console.log("Cloning third party dependencies...done!"); + fs.mkdirpSync(dist); - fs.copySync(cpp, dist, {preserveTimestamps: true}); + fs.mkdirpSync(dist_src); + fs.mkdirpSync(dist_third); + + console.log("Copying C++ to python dist"); + fs.copySync(cpp_src, dist_src, {preserveTimestamps: true}); + console.log("Copying C++ to python dist...done!"); + + if (!fs.existsSync(dist_third_boost)) { + console.log("Copying boost to python dist"); + fs.copySync(third_boost, dist_third_boost, {preserveTimestamps: true}); + console.log("Copying boost to python dist...done!"); + } + + if (!fs.existsSync(dist_third_date)) { + console.log("Copying date to python dist"); + fs.copySync(third_date, dist_third_date, {preserveTimestamps: true}); + console.log("Copying date to python dist...done!"); + } + + if (!fs.existsSync(dist_third_hopscotch)) { + console.log("Copying hopscotch to python dist"); + fs.copySync(third_hopscotch, dist_third_hopscotch, {preserveTimestamps: true}); + console.log("Copying hopscotch to python dist...done!"); + } + + if (!fs.existsSync(dist_third_ordered_map)) { + console.log("Copying ordered-map to python dist"); + fs.copySync(third_ordered_map, dist_third_ordered_map, {preserveTimestamps: true}); + console.log("Copying ordered-map to python dist...done!"); + } + + if (!fs.existsSync(dist_third_pybind11)) { + console.log("Copying pybind11 to python dist"); + fs.copySync(third_pybind11, dist_third_pybind11, {preserveTimestamps: true}); + console.log("Copying pybind11 to python dist...done!"); + } + if (!fs.existsSync(dist_third_tbb)) { + console.log("Copying tbb to python dist"); + fs.copySync(third_tbb, dist_third_tbb, {preserveTimestamps: true}); + console.log("Copying tbb to python dist...done!"); + } + + console.log("Copying LICENSE to python dist"); fs.copySync(lic, dlic, {preserveTimestamps: true}); - fs.copySync(cmake, dcmake, {preserveTimestamps: true}); + console.log("Copying LICENSE to python dist...done!"); clean(obj); let cmd; @@ -61,13 +178,16 @@ try { cmd = cmd + - `${PYTHON} -m pip install -e .[dev] && \ + `${PYTHON} -m pip install -v -e .[dev] && \ + ${PYTHON} setup.py build_ext --inplace && \ ${PYTHON} -m flake8 perspective && echo OK && \ - ${PYTHON} -m pytest -vvv --noconftest perspective/tests/client && \ + otool -L perspective/table/libbinding* && \ + ${PYTHON} -c "import perspective.table.libbinding" && \ ${PYTHON} -m pytest -vvv perspective \ --ignore=perspective/tests/client \ --junitxml=python_junit.xml --cov-report=xml --cov-branch \ - --cov=perspective`; + --cov=perspective && \ + ${PYTHON} -m pytest -vvv --noconftest perspective/tests/client`; if (IMAGE == "python") { cmd = cmd + @@ -78,7 +198,7 @@ try { } else if (IS_INSTALL) { cmd = `${PYTHON} -m pip install . --no-clean`; } else { - cmd = bash`${PYTHON} setup.py build -v`; + cmd = bash`${PYTHON} setup.py build -v && ${PYTHON} setup.py build_ext --inplace`; } if (IS_DOCKER) { diff --git a/scripts/clean.js b/scripts/clean.js index b2f8653deb..014e78941b 100644 --- a/scripts/clean.js +++ b/scripts/clean.js @@ -30,7 +30,7 @@ try { clean`cpp/perspective/obj`; } if (process.env.PSP_PROJECT === "python") { - clean("cpp/perspective/obj", "cpp/perspective/cppbuild", "python/perspective/dist", "python/perspective/build", "python/perspective/perspective_python.egg-info"); + clean("cpp/perspective/obj", "cpp/perspective/cppbuild", "cpp/perspective/third", "python/perspective/dist", "python/perspective/build", "python/perspective/perspective_python.egg-info"); return; } if (!IS_SCREENSHOTS && (!process.env.PACKAGE || minimatch("perspective", process.env.PACKAGE))) { diff --git a/yarn.lock b/yarn.lock index f0faf8460f..318ff060fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3587,6 +3587,11 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142" integrity sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw== +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + chroma-js@^1.3.4: version "1.4.1" resolved "https://registry.yarnpkg.com/chroma-js/-/chroma-js-1.4.1.tgz#eb2d9c4d1ff24616be84b35119f4d26f8205f134" @@ -6853,6 +6858,13 @@ fs-minipass@^1.2.5: dependencies: minipass "^2.6.0" +fs-minipass@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + fs-readdir-recursive@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz#e32fc030a2ccee44a6b5371308da54be0b397d27" @@ -10205,6 +10217,13 @@ minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0: safe-buffer "^5.1.2" yallist "^3.0.0" +minipass@^3.0.0: + version "3.1.3" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd" + integrity sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg== + dependencies: + yallist "^4.0.0" + minizlib@^1.2.1: version "1.3.3" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d" @@ -10212,6 +10231,14 @@ minizlib@^1.2.1: dependencies: minipass "^2.9.0" +minizlib@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.0.tgz#fd52c645301ef09a63a2c209697c294c6ce02cf3" + integrity sha512-EzTZN/fjSvifSX0SlqUERCN39o6T40AMarPbv0MrarSFtIITCBh7bi+dU8nxGFHuqs9jdIAeoYoKuQAAASsPPA== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + mississippi@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/mississippi/-/mississippi-3.0.0.tgz#ea0a3291f97e0b5e8776b363d5f0a12d94c67022" @@ -10248,6 +10275,11 @@ mkdirp@0.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: dependencies: minimist "0.0.8" +mkdirp@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + mobile-drag-drop@^2.3.0-rc.2: version "2.3.0-rc.2" resolved "https://registry.yarnpkg.com/mobile-drag-drop/-/mobile-drag-drop-2.3.0-rc.2.tgz#00d6e85e04512a620fd5357366e8786bd29aa7aa" @@ -14134,6 +14166,18 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.3" +tar@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.0.2.tgz#5df17813468a6264ff14f766886c622b84ae2f39" + integrity sha512-Glo3jkRtPcvpDlAs/0+hozav78yoXKFr+c4wgw62NNMO3oo4AaJdCo21Uu7lcwr55h39W2XD1LMERc64wtbItg== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.0" + mkdirp "^1.0.3" + yallist "^4.0.0" + tcp-port-used@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/tcp-port-used/-/tcp-port-used-1.0.1.tgz#46061078e2d38c73979a2c2c12b5a674e6689d70" @@ -15515,6 +15559,11 @@ yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yamljs@^0.2.1: version "0.2.10" resolved "https://registry.yarnpkg.com/yamljs/-/yamljs-0.2.10.tgz#481cc7c25ca73af59f591f0c96e3ce56c757a40f"