aschaffer · aschaffer · Oct 6, 2020 · Aug 26, 2020 · Aug 27, 2020 · Aug 28, 2020
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,15 +1,45 @@
 # cuGraph 0.16.0 (Date TBD)
 
 ## New Features
-- PR 1098 Add new graph classes to support 2D partitioning
+- PR #1098 Add new graph classes to support 2D partitioning
+- PR #1124 Sub-communicator initialization for 2D partitioning support
+- PR #838 Add pattern accelerator API functions and pattern accelerator API based implementations of PageRank, Katz Centrality, BFS, and SSSP
+- PR #1147 Added support for NetworkX graphs as input type
+- PR #1157 Louvain API update to use graph_container_t
+- PR #1151 MNMG extension for pattern accelerator based PageRank, Katz Centrality, BFS, and SSSP implementations (C++ part)
+- PR #1163 Integrated 2D shuffling and Louvain updates
+- PR #1178 Refactored cython graph factory code to scale to additional data types
 
 ## Improvements
 - PR 1081 MNMG Renumbering - sort partitions by degree
 - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource
+- PR #1133 added python 2D shuffling
+- PR 1129 Refactored test to use common dataset and added additional doc pages
+- PR 1135 SG Updates to Louvain et. al.
+- PR 1132 Upgrade Thrust to latest commit
 - PR #1129 Refactored test to use common dataset and added additional doc pages
+- PR #1145 Simple edge list generator
+- PR #1144 updated documentation and APIs
+- PR #1139 MNMG Louvain Python updates, Cython cleanup
+- PR #1149 Parquet read and concat within workers
+- PR #1152 graph container cleanup, added arg for instantiating legacy types and switch statements to factory function
+- PR #1164 MG symmetrize and conda env updates
+- PR #1162 enhanced networkx testing
+- PR #1169 Added RAPIDS cpp packages to cugraph dev env
+- PR #1165 updated remaining algorithms to be NetworkX compatible
+- PR #1176 Update ci/local/README.md
+- PR #1184 BLD getting latest tags
 
 ## Bug Fixes
 - PR #1131 Show style checker errors with set +e
+- PR #1150 Update RAFT git tag
+- PR #1155 Remove RMM library dependency and CXX11 ABI handling
+- PR #1158 Pass size_t* & size_t* instead of size_t[] & int[] for raft allgatherv's input parameters recvcounts & displs
+- PR #1168 Disabled MG tests on single GPU
+- PR #1166 Fix misspelling of function calls in asserts causing debug build to fail
+- PR #1180 BLD Adopt RAFT model for cuhornet dependency
+- PR #1181 Fix notebook error handling in CI
+- PR #1186 BLD Installing raft headers under cugraph 
 
 # cuGraph 0.15.0 (26 Aug 2020)
 

diff --git a/SOURCEBUILD.md b/SOURCEBUILD.md
@@ -244,22 +244,5 @@ unset LD_LIBRARY_PATH
 
 Python API documentation can be generated from [docs](docs) directory.
 
-## C++ ABI issues
-
-cuGraph builds with C++14 features.  By default, we build cuGraph with the latest ABI (the ABI changed with C++11).  The version of cuDF pointed to in
-the conda installation above is build with the new ABI.
-
-If you see link errors indicating trouble finding functions that use C++ strings when trying to build cuGraph you may have an ABI incompatibility.
-
-There are a couple of complications that may make this a problem:
-* if you need to link in a library built with the old ABI, you may need to build the entire tool chain from source using the old ABI.
-* if you build cudf from source (for whatever reason), the default behavior for cudf (at least through version 0.5.x) is to build using the old ABI.  You can build with the new ABI, but you need to follow the instructions in CUDF to explicitly turn that on.
-
-If you must build cugraph with the old ABI, you can use the following command (instead of the cmake call above):
-
-```bash
-cmake .. -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DCMAKE_CXX11_ABI=OFF
-```
-
 ## Attribution
 Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md
diff --git a/build.sh b/build.sh
@@ -105,7 +105,6 @@ if (( ${NUMARGS} == 0 )) || hasArg libcugraph; then
     mkdir -p ${LIBCUGRAPH_BUILD_DIR}
     cd ${LIBCUGRAPH_BUILD_DIR}
     cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
-          -DCMAKE_CXX11_ABI=${BUILD_ABI} \
 	  -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \
           -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ..
     make -j${PARALLEL_LEVEL} VERBOSE=${VERBOSE} ${INSTALL_TARGET}
@@ -131,8 +130,7 @@ if (( ${NUMARGS} == 0 )) || hasArg docs; then
         mkdir -p ${LIBCUGRAPH_BUILD_DIR}
         cd ${LIBCUGRAPH_BUILD_DIR}
         cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
-            -DCMAKE_CXX11_ABI=${BUILD_ABI} \
-	        -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \
+            -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \
             -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ..
     fi
 

diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
@@ -53,7 +53,7 @@ logger "Check GPU usage..."
 nvidia-smi
 
 logger "Activate conda env..."
-source activate gdf
+source activate rapids
 
 logger "conda install required packages"
 conda install -c nvidia -c rapidsai -c rapidsai-nightly -c conda-forge -c defaults \
@@ -98,6 +98,10 @@ fi
 # TEST - Run GoogleTest and py.tests for libcugraph and cuGraph
 ################################################################################
 
+set +e -Eo pipefail
+EXITCODE=0
+trap "EXITCODE=1" ERR
+
 if hasArg --skip-tests; then
     logger "Skipping Tests..."
 else
@@ -122,3 +126,5 @@ else
     ${WORKSPACE}/ci/gpu/test-notebooks.sh 2>&1 | tee nbtest.log
     python ${WORKSPACE}/ci/utils/nbtestlog2junitxml.py nbtest.log
 fi
+
+return ${EXITCODE}
diff --git a/ci/gpu/test-notebooks.sh b/ci/gpu/test-notebooks.sh
@@ -23,7 +23,7 @@ TOPLEVEL_NB_FOLDERS=$(find . -name *.ipynb |cut -d'/' -f2|sort -u)
 # Add notebooks that should be skipped here
 # (space-separated list of filenames without paths)
 
-SKIPNBS="uvm.ipynb bfs_benchmark.ipynb louvain_benchmark.ipynb pagerank_benchmark.ipynb sssp_benchmark.ipynb release.ipynb"
+SKIPNBS="uvm.ipynb bfs_benchmark.ipynb louvain_benchmark.ipynb pagerank_benchmark.ipynb sssp_benchmark.ipynb release.ipynb nx_cugraph_bc_benchmarking.ipynb"
 
 ## Check env
 env

diff --git a/ci/local/README.md b/ci/local/README.md
@@ -18,19 +18,19 @@ Build and test your local repository using a base gpuCI Docker image
 where:
     -H   Show this help text
     -r   Path to repository (defaults to working directory)
-    -i   Use Docker image (default is gpuci/rapidsai-base:cuda10.0-ubuntu16.04-gcc5-py3.6)
+    -i   Use Docker image (default is gpuci/rapidsai:${NIGHTLY_VERSION}-cuda10.1-devel-ubuntu16.04-py3.7)
     -s   Skip building and testing and start an interactive shell in a container of the Docker image
 ```
 
 Example Usage:
-`bash build.sh -r ~/rapids/cugraph -i gpuci/rapidsai-base:cuda10.1-ubuntu16.04-gcc5-py3.6`
+`bash build.sh -r ~/rapids/cugraph -i gpuci/rapidsai:0.16-cuda10.2-devel-ubuntu16.04-py3.7`
 
 For a full list of available gpuCI docker images, visit our [DockerHub](https://hub.docker.com/r/gpuci/rapidsai/tags) page.
 
 Style Check:
 ```bash
 $ bash ci/local/build.sh -r ~/rapids/cugraph -s
-$ source activate gdf    #Activate gpuCI conda environment
+$ source activate rapids    # Activate gpuCI conda environment
 $ cd rapids
 $ flake8 python
 ```
@@ -42,7 +42,7 @@ There are some caveats to be aware of when using this script, especially if you
 
 ### Docker Image Build Repository
 
-The docker image will generate build artifacts in a folder on your machine located in the `root` directory of the repository you passed to the script. For the above example, the directory is named `~/rapids/cugraph/build_rapidsai-base_cuda10.1-ubuntu16.04-gcc5-py3.6/`. Feel free to remove this directory after the script is finished.
+The docker image will generate build artifacts in a folder on your machine located in the `root` directory of the repository you passed to the script. For the above example, the directory is named `~/rapids/cugraph/build_rapidsai_cuda10.1-ubuntu16.04-py3.7/`. Feel free to remove this directory after the script is finished.
 
 *Note*: The script *will not* override your local build repository. Your local environment stays in tact.
 

diff --git a/conda/environments/cugraph_dev_cuda10.1.yml b/conda/environments/cugraph_dev_cuda10.1.yml
@@ -6,7 +6,9 @@ channels:
 - conda-forge
 dependencies:
 - cudf=0.16.*
+- libcudf=0.16.*
 - rmm=0.16.*
+- librmm=0.16.*
 - dask>=2.12.0
 - distributed>=2.12.0
 - dask-cuda=0.16*

diff --git a/conda/environments/cugraph_dev_cuda10.2.yml b/conda/environments/cugraph_dev_cuda10.2.yml
@@ -6,7 +6,9 @@ channels:
 - conda-forge
 dependencies:
 - cudf=0.16.*
+- libcudf=0.16.*
 - rmm=0.16.*
+- librmm=0.16.*
 - dask>=2.12.0
 - distributed>=2.12.0
 - dask-cuda=0.16*

diff --git a/conda/environments/cugraph_dev_cuda11.0.yml b/conda/environments/cugraph_dev_cuda11.0.yml
@@ -6,7 +6,9 @@ channels:
 - conda-forge
 dependencies:
 - cudf=0.16.*
+- libcudf=0.16.*
 - rmm=0.16.*
+- librmm=0.16.*
 - dask>=2.12.0
 - distributed>=2.12.0
 - dask-cuda=0.16*

diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml
@@ -32,6 +32,10 @@ requirements:
     - python x.x
     - libcugraph={{ version }}
     - cudf={{ minor_version }}
+    - dask-cudf {{ minor_version }}
+    - dask-cuda {{ minor_version }}
+    - dask>=2.12.0
+    - distributed>=2.12.0
     - nccl>=2.5
     - ucx-py {{ minor_version }}
 

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -46,29 +46,6 @@ set(CMAKE_CUDA_STANDARD_REQUIRED ON)
 
 if(CMAKE_COMPILER_IS_GNUCXX)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-error=deprecated-declarations")
-
-###################################################################################################
-###   C++ ABI changes.
-###
-###   By default, cugraph builds with the new C++ ABI.  In order to insure that thirdparty
-###   applications build with the properly setting (specifically RMM) we need to set
-###   the CMAKE_CXX11_ABI flag appropriately.
-###
-###   If a user wants to build with the OLD ABI, then they need to define CMAKE_CXX11_ABI
-###   to be OFF (typically on the cmake command line).
-###
-###   This block of code will configure the old ABI if the flag is set to OFF and
-###   do nothing (the default behavior of the C++14 compiler).
-###
-    option(CMAKE_CXX11_ABI "Enable the GLIBCXX11 ABI" ON)
-    if(CMAKE_CXX11_ABI)
-        message(STATUS "CUGRAPH: Enabling the GLIBCXX11 ABI")
-    else()
-        message(STATUS "CUGRAPH: Disabling the GLIBCXX11 ABI")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
-        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler -D_GLIBCXX_USE_CXX11_ABI=0")
-    endif(CMAKE_CXX11_ABI)
 endif(CMAKE_COMPILER_IS_GNUCXX)
 
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode=arch=compute_60,code=sm_60")
@@ -172,49 +149,20 @@ find_path(RMM_INCLUDE "rmm"
     "$ENV{CONDA_PREFIX}/include/rmm"
     "$ENV{CONDA_PREFIX}/include")
 
-find_library(RMM_LIBRARY "rmm"
-    HINTS
-    "$ENV{RMM_ROOT}/lib"
-    "$ENV{CONDA_PREFIX}/lib")
-
-message(STATUS "RMM: RMM_LIBRARY set to ${RMM_LIBRARY}")
 message(STATUS "RMM: RMM_INCLUDE set to ${RMM_INCLUDE}")
 
-add_library(rmm SHARED IMPORTED ${RMM_LIBRARY})
-if (RMM_INCLUDE AND RMM_LIBRARY)
-    set_target_properties(rmm PROPERTIES IMPORTED_LOCATION ${RMM_LIBRARY})
-endif (RMM_INCLUDE AND RMM_LIBRARY)
-
 ###################################################################################################
-# - Fetch Content -----------------------------------------------------------------------------
+# - Fetch Content ---------------------------------------------------------------------------------
 include(FetchContent)
 
-# - CUB
-message("Fetching CUB")
-
-FetchContent_Declare(
-    cub
-    GIT_REPOSITORY https://github.com/thrust/cub.git
-    GIT_TAG        1.9.10
-    GIT_SHALLOW    true
-)
-
-FetchContent_GetProperties(cub)
-if(NOT cub_POPULATED)
-  FetchContent_Populate(cub)
-  # We are not using the cub CMake targets, so no need to call `add_subdirectory()`.
-endif()
-set(CUB_INCLUDE_DIR "${cub_SOURCE_DIR}")
-
-# - THRUST
+# - THRUST/CUB
 message("Fetching Thrust")
 
 FetchContent_Declare(
     thrust
     GIT_REPOSITORY https://github.com/thrust/thrust.git
-    GIT_TAG        1.9.10
-    GIT_SHALLOW    true
-    PATCH_COMMAND COMMAND patch -p1 < "${CMAKE_CURRENT_SOURCE_DIR}/cmake/thrust-ret-if-fail.patch"
+    # August 28, 2020
+    GIT_TAG        52a8bda46c5c2128414d1d47f546b486ff0be2f0
 )
 
 FetchContent_GetProperties(thrust)
@@ -224,9 +172,6 @@ if(NOT thrust_POPULATED)
 endif()
 set(THRUST_INCLUDE_DIR "${thrust_SOURCE_DIR}")
 
-
-
-
 ###################################################################################################
 # - External Projects -----------------------------------------------------------------------------
 
@@ -240,7 +185,7 @@ set(CUHORNET_INCLUDE_DIR ${CUHORNET_DIR}/src/cuhornet CACHE STRING "Path to cuho
 
 ExternalProject_Add(cuhornet
   GIT_REPOSITORY    https://github.com/rapidsai/cuhornet.git
-  GIT_TAG           main
+  GIT_TAG           9cb8e8803852bd895a9c95c0fe778ad6eeefa7ad
   PREFIX            ${CUHORNET_DIR}
   CONFIGURE_COMMAND ""
   BUILD_COMMAND     ""
@@ -305,7 +250,7 @@ else(DEFINED ENV{RAFT_PATH})
 
   ExternalProject_Add(raft
     GIT_REPOSITORY    https://github.com/rapidsai/raft.git
-    GIT_TAG           099e2b874b05555a78bed1666fa2d22f784e56a7
+    GIT_TAG           53c1e2dde4045f386f9cc4bb7d3dc99d5690b886
     PREFIX            ${RAFT_DIR}
     CONFIGURE_COMMAND ""
     BUILD_COMMAND     ""
@@ -332,6 +277,7 @@ add_library(cugraph SHARED
     src/db/db_parser_integration_test.cu
     src/db/db_operators.cu
     src/utilities/spmv_1D.cu
+    src/utilities/cython.cu
     src/structure/graph.cu
     src/link_analysis/pagerank.cu
     src/link_analysis/pagerank_1D.cu
@@ -344,10 +290,8 @@ add_library(cugraph SHARED
     src/converters/renumber.cu
     src/converters/COOtoCSR.cu
     src/community/spectral_clustering.cu
-    src/community/louvain.cpp
-    src/community/louvain_kernels.cu
-    src/community/leiden.cpp
-    src/community/leiden_kernels.cu
+    src/community/louvain.cu
+    src/community/leiden.cu
     src/community/ktruss.cu
     src/community/ECG.cu
     src/community/triangles_counting.cu
@@ -359,6 +303,10 @@ add_library(cugraph SHARED
     src/centrality/betweenness_centrality.cu
     src/experimental/graph.cu
     src/experimental/graph_view.cu
+    src/experimental/bfs.cu
+    src/experimental/sssp.cu
+    src/experimental/pagerank.cu
+    src/experimental/katz_centrality.cu
 )
 
 #
@@ -372,7 +320,6 @@ add_dependencies(cugraph raft)
 # - include paths ---------------------------------------------------------------------------------
 target_include_directories(cugraph
     PRIVATE
-    "${CUB_INCLUDE_DIR}"
     "${THRUST_INCLUDE_DIR}"
     "${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}"
     "${LIBCYPHERPARSER_INCLUDE}"
@@ -395,7 +342,7 @@ target_include_directories(cugraph
 # - link libraries --------------------------------------------------------------------------------
 
 target_link_libraries(cugraph PRIVATE
-    ${RMM_LIBRARY} gunrock cublas cusparse curand cusolver cudart cuda ${LIBCYPHERPARSER_LIBRARY} ${MPI_CXX_LIBRARIES} ${NCCL_LIBRARIES})
+    gunrock cublas cusparse curand cusolver cudart cuda ${LIBCYPHERPARSER_LIBRARY} ${MPI_CXX_LIBRARIES} ${NCCL_LIBRARIES})
 
 if(OpenMP_CXX_FOUND)
 target_link_libraries(cugraph PRIVATE
@@ -408,7 +355,7 @@ target_link_libraries(cugraph PRIVATE
 ###   ...
 ###
 ###   libgomp.so is included in the conda base environment and copied to every new conda
-###   environment. If a full file path is provided (e.g ${RMM_LIBRARY}), cmake
+###   environment. If a full file path is provided (e.g ${NCCL_LIBRARIES}), cmake
 ###   extracts the directory path and adds the directory path to BUILD_RPATH (if BUILD_RPATH is not
 ###   disabled).
 ###
@@ -421,7 +368,7 @@ target_link_libraries(cugraph PRIVATE
 ###   If a full path to libgomp.so is provided (which is the case with OpenMP::OpenMP_CXX), cmake
 ###   checks whether there is any other libgomp.so with the different full path (after resolving
 ###   soft links) in the search paths (implicit directoires + BUILD_RAPTH). There is one in the
-###   path included in BUILD_RPATH when ${RMM_LIBRARY} are added; this one can
+###   path included in BUILD_RPATH when ${NCCL_LIBRARIES} are added; this one can
 ###   potentially hide the one in the provided full path and cmake generates a warning (and RPATH
 ###   is searched before the directories in /etc/ld.so/conf; ld.so.conf does not coincide but
 ###   overlaps with implicit directories).
@@ -470,6 +417,8 @@ install(TARGETS cugraph LIBRARY
 install(DIRECTORY include/
     DESTINATION include/cugraph)
 
+install(DIRECTORY ${RAFT_DIR}/cpp/include/raft/
+    DESTINATION include/cugraph/raft)
 ###################################################################################################
 # - make documentation ----------------------------------------------------------------------------
 # requires doxygen and graphviz to be installed

diff --git a/cpp/cmake/Modules/ConfigureArrow.cmake b/cpp/cmake/Modules/ConfigureArrow.cmake
@@ -21,14 +21,6 @@ set(ARROW_CMAKE_ARGS " -DARROW_WITH_LZ4=OFF"
                      " -DARROW_USE_GLOG=OFF"
                      " -DCMAKE_VERBOSE_MAKEFILE=ON")
 
-if(NOT CMAKE_CXX11_ABI)
-    message(STATUS "ARROW: Disabling the GLIBCXX11 ABI")
-    list(APPEND ARROW_CMAKE_ARGS " -DARROW_TENSORFLOW=ON")
-elseif(CMAKE_CXX11_ABI)
-    message(STATUS "ARROW: Enabling the GLIBCXX11 ABI")
-    list(APPEND ARROW_CMAKE_ARGS " -DARROW_TENSORFLOW=OFF")
-endif(NOT CMAKE_CXX11_ABI)
-
 configure_file("${CMAKE_SOURCE_DIR}/cmake/Templates/Arrow.CMakeLists.txt.cmake"
                "${ARROW_ROOT}/CMakeLists.txt")