From 01e352d08f279cca8d31d110f6262ad65a068941 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Mon, 17 Aug 2020 23:44:45 -0400 Subject: [PATCH 01/77] initial implementation of new graph class --- cpp/include/exp_graph.hpp | 220 +++++++++++++++++++++++ cpp/src/experimental/graph.cu | 320 ++++++++++++++++++++++++++++++++++ 2 files changed, 540 insertions(+) create mode 100644 cpp/include/exp_graph.hpp create mode 100644 cpp/src/experimental/graph.cu diff --git a/cpp/include/exp_graph.hpp b/cpp/include/exp_graph.hpp new file mode 100644 index 00000000000..fd0f5b0dc0d --- /dev/null +++ b/cpp/include/exp_graph.hpp @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +// FIXME: this file should be renamed to graph.hpp. + +#include + +#include +#include + +#include +#include +#include +#include + +namespace cugraph { +namespace experimental { + +// FIXME: threshold values require tuning +size_t constexpr low_degree_threshold{raft::warp_size()}; +size_t constexpr mid_degree_threshold{1024}; +size_t constexpr num_segments_per_vertex_partition{3}; + +struct graph_properties_t { + bool is_symmetric{false}; + bool is_multigraph{false}; + bool is_weighted{false}; +}; + +// FIXME: these should better be defined somewhere else. +std::string constexpr comm_p_row_key = "comm_p_row"; +std::string constexpr comm_p_col_key = "comm_p_key"; + +template +struct edgelist_t { + vertex_t const *p_src_vertices{nullptr}; + vertex_t const *p_dst_vertices{nullptr}; + vertex_t const *p_edge_weights{nullptr}; + edge_t number_of_edges{0}; +}; + +/** + * @brief store vertex partitioning map + * + * We need to partition 1D vertex arrays (storing per vertex values) and the 2D graph adjacency + * matrix (or transposed 2D graph adjacency matrix) of G. An 1D vertex array of size V is divided to + * P linear partitions; each partition has the size close to V / P. The 2D matrix is partitioned to + * 1) P or 2) P * P_col rectangular partitions. + * + * 1) is the default. One GPU will be responsible for 1 rectangular partition. The matrix will be + * horizontally partitioned first to P_row slabs (by combining P_col vertex partitions). Each slab + * will be further vertically partitioned to P_col rectangles (by combining P_row vertex + * partitions). Each rectangular partition will have the size close to V / P_row by V / P_col. + * + * 2) is to support hyper-graph partitioning. We may apply hyper-graph partitioning to divide V + * vertices to P groups minimizing edge cuts across groups while balancing the number of vertices in + * each group. We will also renumber vertices so the vertices in each group are mapped to + * consecutive integers. Then, there will be more non-zeros in the diagonal partitions of the 2D + * graph adjacency matrix (or the transposed 2D graph adjacency matrix) than the off-diagonal + * partitions. The strategy 1) does not balance the number of nonzeros if hyper-graph partitioning + * is applied. To solve this problem, the matrix is first horizontally partitioned to P (instead of + * P_row) slabs, then each slab will be further vertically partitioned to P_col rectangles. One GPU + * will be responsible P_col rectangular partitions in this case. See E. G. Boman et. al., “Scalable + * matrix computations on large scale-free graphs using 2D graph partitioning”, 2013 for additional + * detail. + * + * In case of 1), a GPU with (row_rank, col_rank) will be responsible for one rectangular partition + * [a,b) by [c,d) where + * a = vertex_partition_offsets[P_col * row_rank], + * b = vertex_partition_offsets[p_col * (row_rank + 1)], + * c = vertex_partition_offsets[P_row * col_rank], and + * d = vertex_partition_offsets[p_row * (col_rank + 1)] + * + * In case of 2), a GPU with (row_rank, col_rank) will be responsible for P_col rectangular + * partitions [a_i,b_i) by [c,d) where + * a_i = vertex_partition_offsets[P_row * i + row_rank] and + * b_i = vertex_partition_offsets[P_row * i + row_rank + 1]. + * c and d are same to 1) and i = [0, P_col). + * + * @tparam vertex_t Type of vertex ID + */ +template +struct partition_t { + std::vector vertex_partition_offsets{}; // size = P + 1 + bool hypergraph_partitioned{false}; +}; + +template +class graph_t; + +// multi-GPU version +template +class graph_t> { + public: + graph_t(raft::handle_t &handle, + std::vector> const &edge_lists, + partition_t const &partition, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool sorted_by_global_degree_within_vertex_partition, + bool do_expensive_check = false); + + vertex_t get_number_of_vertices() const { return number_of_vertices_; } + edge_t get_number_of_edges() const { return number_of_edges_; } + + bool is_symmetric() const { return properties_.is_symmetric; } + bool is_multigraph() const { return properties_.is_multigraph; } + bool is_weighted() const { return properties.is_weighted; } + + vertex_t get_number_of_local_vertices() const noexcept + { + auto comm_p_rank = handle_ptr_->get_comms().get_rank(); + return vertex_partition_.partition_offsets[comm_p_rank + 1] - + vertex_partition_.partition_offsets[comm_p_rank]; + } + + // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this + // function will be eventually removed. + edge_t const *offsets() const + { + CUGRAPH_EXPECTS(adj_matrix_partition_offsets_.size() == 1, + "offsets() is valid only when there is one rectangular partition per GPU."); + return adj_matrix_partition_offsets_[0].data(); + } + + // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this + // function will be eventually removed. + vertex_t const *indices() const + { + CUGRAPH_EXPECTS(adj_matrix_partition_indicess_.size() == 1, + "indices() is valid only when there is one rectangular partition per GPU."); + return adj_matrix_partition_indices_[0].data(); + } + + // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this + // function will be eventually removed. + weight_t const *weights() const + { + CUGRAPH_EXPECTS(adj_matrix_partition_weights_.size() == 1, + "weights() is valid only when there is one rectangular partition per GPU."); + return adj_matrix_partition_weights_[0].data(); + } + + private: + vertex_t number_of_vertices_{0}; + edge_t number_of_edges_{0}; + + graph_properties_t properties_{}; + + raft::handle_t const *handle_ptr_{nullptr}; + + partition_t partition_{}; + + std::vector> adj_matrix_partition_offsets_{}; + std::vector> adj_matrix_partition_indices_{}; + std::vector> adj_matrix_partition_weights_{}; + std::vector + vertex_partition_segment_offsets_{}; // segment offsets within the vertex partition based on + // vertex degree, relevant only if + // sorted_by_global_degree_within_vertex_partition is true +}; + +template +struct invalid_idx; + +template +struct invalid_idx< + T, + typename std::enable_if_t::value && std::is_signed::value>> + : std::integral_constant { +}; + +template +struct invalid_idx< + T, + typename std::enable_if_t::value && std::is_unsigned::value>> + : std::integral_constant::max()> { +}; + +template +struct invalid_vertex_id : invalid_idx { +}; + +template +struct invalid_edge_id : invalid_idx { +}; + +} // namespace experimental +} // namespace cugraph \ No newline at end of file diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu new file mode 100644 index 00000000000..38a8190e49f --- /dev/null +++ b/cpp/src/experimental/graph.cu @@ -0,0 +1,320 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace cugraph { +namespace experimental { + +namespace { + +template +auto edge_list_to_compressed_sparse(raft::handel_t &handle, + edgelist_t const &edgelist, + vertex_t row_first, + vertex_t row_last, + vertex_t col_first, + vertex_t col_last) +{ + rmm::device_uvector offsets( + store_transposed ? (row_last - row_first) + 1 : (col_last - col_first) + 1, + edge_t{0}, + handle.get_stream()); + rmm::device_uvector indices( + edge_list.number_of_edges, vertex_t{0}, handle.get_stream()); + rmm::device_uvector weights( + edge_list.p_edge_weights != nullptr ? edge_list.number_of_edges : 0, handle.get_stream()); + thrust::fill(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + offsets.begin(), + offsets.end(), + edge_t{0}); + + // FIXME: need to performance test this code with R-mat graphs having highly-skewed degree + // distribution. If there is a small number of vertices with very large degrees, atomicAdd can + // sequentialize execution. CUDA9+ & Kepler+ provide complier/architectural optimizations to + // mitigate this impact + // (https://developer.nvidia.com/blog/cuda-pro-tip-optimized-filtering-warp-aggregated-atomics/), + // and we need to check this thrust::for_each based approach delivers the expected performance. + + // FIXME: also need to verify this approach is at least not significantly slower than the sorting + // based approach (this approach does not use extra memory, so better stick to this approach + // unless performance is significantly worse). + + auto p_offsets = offsets.data(); + auto p_indices = indices.data(); + auto p_weights = + edge_list.p_edge_weights != nullptr ? weights.data() : reinterpret_cast(nullptr); + + auto major_first = store_transposed ? row_first : col_first; + thrust::for_each( + rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + store_transposed ? edgelist.p_src_vertices : edgelist.p_dst_vertices, + store_transposed ? edgelist.p_src_vertices + edgelist.number_of_edges + : edgelist.p_dst_vertices + edgelist.number_of_edges, + [p_offsets, major_first] __device__(auto v) { atomicAdd(p_offsets + (v - major_first), 1); }); + + thrust::exclusive_scan(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + offsets.begin(), + offsets.end(), + offsets.begin()); + + if (edgelist.edge_weights != nullptr) { + auto edge_first = thrust::make_zip_iterator(thrust::make_tuple( + edgelist.p_src_vertices, edgelist.p_dst_vertices, edgelist.p_edge_weights)); + thrust::for_each(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + edge_first, + edge_first + edgelist.number_of_edges, + [p_offsets, p_indices, p_weights, major_first] __device__(auto e) { + auto s = thrust::get<0>(e); + auto d = thrust::get<1>(e); + auto w = thrust::get<2>(e); + auto major = store_transposed ? s : d; + auto minor = store_transposed ? d : s; + auto start = p_offsets[major - major_first]; + auto degree = p_offsets[(major - major_first) + 1] - start; + auto idx = atmoicAdd(p_indices + (degree - 1), + 1); // use the last element as a counter + // FIXME: we can actually store minor - minor_first instead of minor to save + // memory if minor can be larger than 32 bit but minor - minor_first fits + // within 32 bit + p_indices[start + idx] = + minor; // overwrite the counter only if idx == degree - 1 (no race) + p_weights[start + idx] = w; + }); + } else { + auto edge_first = thrust::make_zip_iterator( + thrust::make_tuple(edgelist.p_src_vertices, edgelist.p_dst_vertices)); + thrust::for_each(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + edge_first, + edge_first + edgelist.number_of_edges, + [p_offsets, p_indices, p_weights, major_first] __device__(auto e) { + auto s = thrust::get<0>(e); + auto d = thrust::get<1>(e); + auto major = store_transposed ? s : d; + auto minor = store_transposed ? d : s; + auto start = p_offsets[major - major_first]; + auto degree = p_offsets[(major - major_first) + 1] - start; + auto idx = atmoicAdd(p_indices + (degree - 1), + 1); // use the last element as a counter + // FIXME: we can actually store minor - minor_first instead of minor to save + // memory if minor can be larger than 32 bit but minor - minor_first fits + // within 32 bit + p_indices[start + idx] = + minor; // overwrite the counter only if idx == degree - 1 (no race) + }); + } + + return std::make_tuple(offsets, indices, weights); +} + +// compute the numbers of nonzeros in rows of the (transposed) graph adjacency matrix +rmm::device_uvector compute_row_degree( + raft::handle_t &handle, + std::vector> const &adj_matrix_partition_offsets{}; + bool hypergraph_partitioned) +{ + auto &comm_p_row = handle.get_subcomm(comm_p_row_key); + auto comm_p_row_rank = comm_p_row.get_rank(); + auto comm_p_row_size = comm_p_row.get_size(); + auto &comm_p_col = handle.get_subcomm(comm_p_col_key); + auto comm_p_col_rank = comm_p_col.get_rank(); + auto comm_p_col_size = comm_p_col.get_size(); + + rmm::device_uvector local_degrees{}; + rmm::device_uvector degrees{}; + + vertex_t max_num_local_degrees{0}; + for (int i = 0; i < comm_p_col_size; ++i) { + auto vertex_partition_id = hypergraph_partitioned ? comm_p_row_size * i + row_rank + : comm_p_col_size * comm_p_row_rank + i; + auto row_first = vertex_partition_offsets[vertex_partition_id]; + auto row_last = vertex_partition_offsets[vertex_partition_id + 1]; + max_num_local_degrees = std::max(max_num_local_degrees, row_last - row_first); + if (i == comm_p_col_rank) { degrees.resize(row_last - row_first, handle.get_stream()); } + } + local_degrees.reisze(local_degree_size, handle.get_stream()); + for (int i = 0; i < comm_p_col_size; ++i) { + auto vertex_partition_id = hypergraph_partitioned ? comm_p_row_size * i + row_rank + : comm_p_col_size * comm_p_row_rank + i; + auto row_first = vertex_partition_offsets[vertex_partition_id]; + auto row_last = vertex_partition_offsets[vertex_partition_id + 1]; + auto p_offsets = + hypergraph_partitioned + ? adj_matrix_partition_offsets_[i].data() + : adj_matrix_partition_offsets_[0].data() + + (row_first - vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); + thrust::transform(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + thrust::make_counting_iterator(vertex_t{0}), + thrust::make_counting_iterator(row_last - row_first), + local_degrees.data(), + [p_offsets] __device__(auto i) { return p_offsets[i + 1] - p_offsets[i]; }); + comm_p_row.reduce(local_degrees.data(), + i == comm_p_col_rank ? degrees.data() : nullptr, + degrees.size(), + raft::comms::get_type(), + hanlde.get_stream()); + } + + return degrees; +} + +template +std::vector segment_degree_sorted_vertex_partition(raft::handle_t &handle, + DegreeIterator degree_first, + DegreeIterator degree_last, + ThresholdIterator threshold_first, + ThresholdIterator threshold_last) +{ + auto num_elements = thrust::distance(degree_first, degree_last); + auto num_segments = thrust::distance(threshold_first, threshold_last) + 1; + + std::vector h_segment_offsets(num_segments + 1); + h_segment_offsets[0] = 0; + h_segment_offsets.back() = num_elements; + + rmm::device_uvector d_segment_offsets(num_segments - 1, handle.get_stream()); + + thrust::upper_bound(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + degree_first, + degree_last, + threshold_first, + threshold_last, + d_segments_offsets.begin()); + + raft::update_host(h_segment_offsets.begin() + 1, + d_segment_offsets.begin(), + d_segment_offsets.size(), + handle.get_stream()); + + return h_segment_offsets; +} + +} // namespace + +} // namespace experimental + +template +graph_t>:: + graph_t(raft::handel_t &handle, + std::vector> const &edge_lists, + vertex_partition_t const &vertex_partition, + std::vector> const &adj_matrix_partitions, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool do_expensive_check) + : number_of_vertices_(number_of_vertices), + number_of_edges_(number_of_edges), + properties_({is_symmetric, is_multigraph, is_weighted}), + handle_ptr_(&handle), + vertex_partition_(vertex_partition), + adj_matrix_partitions_(adj_matrix_partitions) +{ + CUGRAPH_EXPECTS(edge_lists.size() == adj_matrix_partitions.size(), + "Invalid API parameter, edge_lists.size() != adj_matrix_partitions.size()"); + + // convert edge list (COO) to compressed sparse format (CSR or CSC) + + adj_matrix_partition_offsets_.resize(edge_lists.size()); + adj_matrix_partition_indices_.resize(edge_lists.size()); + adj_matrix_partition_weights_.resize(edge_lists.size()); + for (size_t i = 0; i < edge_lists.size(); ++i) { + CUGRAPH_EXPECTS((is_weighted == false) || (edge_lists[i].p_edge_weights != nullptr), + "Invalid API parameter, edge_lists[i].p_edge_weights shoud not be nullptr if " + "is_weighted == true"); + + rmm::device_uvector offets{}; + rmm::device_uvector indices{}; + rmm::device_uvector weights{}; + + std::tie(offsets, indices, weights) = detail::edge_list_to_compressed_sparse( + handle, + edge_lists[i], + vertex_partition_.partition_offsets[adj_matrix_partitions[i].row_vertex_partition_first], + vertex_partition_.partition_offsets[adj_matrix_partitions[i].row_vertex_partition_last], + vertex_partition_.partition_offsets[adj_matrix_partitions[i].col_vertex_partition_first], + vertex_partition_.partition_offsets[adj_matrix_partitions[i].col_vertex_partition_last]); + adj_matrix_partitions_offsets_[i] = std::move(offsets); + adj_matrix_partitions_indices_[i] = std::move(indices); + if (is_weighted) { adj_matrix_partitions_weights_[i] = std::move(weights); } + } + + // update degree-based segment offsets (to be used for graph analytics kernel optimization) + + auto &comm_p_row = handle_ptr_->get_subcomm(comm_p_row_key); + auto comm_p_row_rank = comm_p_row.get_rank(); + auto comm_p_row_size = comm_p_row.get_size(); + + auto degrees = compute_row_degree(); + + static_assert(detail::num_segments_per_subpartition == 3); + static_assert((detail::low_degree_threshold <= detail::high_degree_threshold) && + (detail::high_degree_threshold <= std::numeric_limits::max())); + auto segment_degree_threshold_first = + thrust::make_transform_iterator(thrust::make_counting_iterator(0), [] __device__(auto i) { + if (i == 0) { + return static_cast(detail::low_degree_threshold); + } else if (i == 1) { + return static_cast(detail::high_degree_threshold); + } else { + assert(0); // should not be reached + return vertex_t{0}; + } + }); + + auto segment_offsets = segment_degree_sorted_vertex_partition( + *handle_ptr_, + degrees.begin(), + degrees.end(), + segment_degree_threshold_first, + segment_degree_threshold_first + num_segments_per_subpartition - 1); + + comm_p_row.allgather(segment_offsets.begin(), + aggregate_segment_offsets_.begin(), + (num_segments_per_partition + 1), + raft::comms::get_type(), + hanlde_ptr_->get_stream()); +} + +// explicit instantiation + +template class graph_t; +template class graph_t; + +template class graph_t; +template class graph_t; + +} // namespace cugraph +} // namespace cugraph \ No newline at end of file From d33328e67edee5290c1cf751890998a95aae9127 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 18 Aug 2020 23:29:29 -0400 Subject: [PATCH 02/77] fix compile errors --- cpp/include/exp_graph.hpp | 105 ++++++++--- cpp/src/experimental/graph.cu | 316 ++++++++++++++++++++++------------ 2 files changed, 288 insertions(+), 133 deletions(-) diff --git a/cpp/include/exp_graph.hpp b/cpp/include/exp_graph.hpp index fd0f5b0dc0d..fce6e3096b4 100644 --- a/cpp/include/exp_graph.hpp +++ b/cpp/include/exp_graph.hpp @@ -35,23 +35,9 @@ size_t constexpr low_degree_threshold{raft::warp_size()}; size_t constexpr mid_degree_threshold{1024}; size_t constexpr num_segments_per_vertex_partition{3}; -struct graph_properties_t { - bool is_symmetric{false}; - bool is_multigraph{false}; - bool is_weighted{false}; -}; - // FIXME: these should better be defined somewhere else. -std::string constexpr comm_p_row_key = "comm_p_row"; -std::string constexpr comm_p_col_key = "comm_p_key"; - -template -struct edgelist_t { - vertex_t const *p_src_vertices{nullptr}; - vertex_t const *p_dst_vertices{nullptr}; - vertex_t const *p_edge_weights{nullptr}; - edge_t number_of_edges{0}; -}; +std::string const comm_p_row_key = "comm_p_row"; +std::string const comm_p_col_key = "comm_p_key"; /** * @brief store vertex partitioning map @@ -99,6 +85,20 @@ struct partition_t { bool hypergraph_partitioned{false}; }; +struct graph_properties_t { + bool is_symmetric{false}; + bool is_multigraph{false}; + bool is_weighted{false}; +}; + +template +struct edgelist_t { + vertex_t const *p_src_vertices{nullptr}; + vertex_t const *p_dst_vertices{nullptr}; + weight_t const *p_edge_weights{nullptr}; + edge_t number_of_edges{0}; +}; + template > { public: - graph_t(raft::handle_t &handle, - std::vector> const &edge_lists, + graph_t(raft::handle_t const &handle, + std::vector> const &edge_lists, partition_t const &partition, vertex_t number_of_vertices, edge_t number_of_edges, @@ -136,13 +136,13 @@ class graph_tget_comms().get_rank(); - return vertex_partition_.partition_offsets[comm_p_rank + 1] - - vertex_partition_.partition_offsets[comm_p_rank]; + return partition_.vertex_partition_offsets[comm_p_rank + 1] - + partition_.vertex_partition_offsets[comm_p_rank]; } // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this @@ -158,7 +158,7 @@ class graph_t +class graph_t> { + public: + graph_t(raft::handle_t const &handle, + edgelist_t const &edge_list, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool sorted_by_global_degree, + bool do_expensive_check = false); + + vertex_t get_number_of_vertices() const { return number_of_vertices_; } + edge_t get_number_of_edges() const { return number_of_edges_; } + + bool is_symmetric() const { return properties_.is_symmetric; } + bool is_multigraph() const { return properties_.is_multigraph; } + bool is_weighted() const { return properties_.is_weighted; } + + vertex_t get_number_of_local_vertices() const { return number_of_vertices_; } + + // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this + // function will be eventually removed. + edge_t const *offsets() const { return offsets_.data(); } + + // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this + // function will be eventually removed. + vertex_t const *indices() const { return indices_.data(); } + + // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this + // function will be eventually removed. + weight_t const *weights() const { return weights_.data(); } + + private: + vertex_t number_of_vertices_{0}; + edge_t number_of_edges_{0}; + + graph_properties_t properties_{}; + + raft::handle_t const *handle_ptr_{nullptr}; + + rmm::device_uvector offsets_; + rmm::device_uvector indices_; + rmm::device_uvector weights_; + std::vector segment_offsets_{}; // segment offsets based on vertex degree, relevant + // only if sorted_by_global_degree is true +}; + template struct invalid_idx; diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 38a8190e49f..c8c2867dad0 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -18,42 +18,52 @@ #include +#include #include #include #include +#include +#include #include #include #include #include #include +#include #include +#include + namespace cugraph { namespace experimental { namespace { -template -auto edge_list_to_compressed_sparse(raft::handel_t &handle, - edgelist_t const &edgelist, - vertex_t row_first, - vertex_t row_last, - vertex_t col_first, - vertex_t col_last) +template +std:: + tuple, rmm::device_uvector, rmm::device_uvector> + edge_list_to_compressed_sparse(raft::handle_t const &handle, + edgelist_t const &edgelist, + vertex_t row_first, + vertex_t row_last, + vertex_t col_first, + vertex_t col_last) { rmm::device_uvector offsets( store_transposed ? (row_last - row_first) + 1 : (col_last - col_first) + 1, - edge_t{0}, handle.get_stream()); - rmm::device_uvector indices( - edge_list.number_of_edges, vertex_t{0}, handle.get_stream()); + rmm::device_uvector indices(edgelist.number_of_edges, handle.get_stream()); rmm::device_uvector weights( - edge_list.p_edge_weights != nullptr ? edge_list.number_of_edges : 0, handle.get_stream()); + edgelist.p_edge_weights != nullptr ? edgelist.number_of_edges : 0, handle.get_stream()); thrust::fill(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), offsets.begin(), offsets.end(), edge_t{0}); + thrust::fill(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + indices.begin(), + indices.end(), + vertex_t{0}); // FIXME: need to performance test this code with R-mat graphs having highly-skewed degree // distribution. If there is a small number of vertices with very large degrees, atomicAdd can @@ -69,22 +79,23 @@ auto edge_list_to_compressed_sparse(raft::handel_t &handle, auto p_offsets = offsets.data(); auto p_indices = indices.data(); auto p_weights = - edge_list.p_edge_weights != nullptr ? weights.data() : reinterpret_cast(nullptr); + edgelist.p_edge_weights != nullptr ? weights.data() : static_cast(nullptr); auto major_first = store_transposed ? row_first : col_first; - thrust::for_each( - rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), - store_transposed ? edgelist.p_src_vertices : edgelist.p_dst_vertices, - store_transposed ? edgelist.p_src_vertices + edgelist.number_of_edges - : edgelist.p_dst_vertices + edgelist.number_of_edges, - [p_offsets, major_first] __device__(auto v) { atomicAdd(p_offsets + (v - major_first), 1); }); + thrust::for_each(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + store_transposed ? edgelist.p_src_vertices : edgelist.p_dst_vertices, + store_transposed ? edgelist.p_src_vertices + edgelist.number_of_edges + : edgelist.p_dst_vertices + edgelist.number_of_edges, + [p_offsets, major_first] __device__(auto v) { + atomicAdd(p_offsets + (v - major_first), edge_t{1}); + }); thrust::exclusive_scan(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), offsets.begin(), offsets.end(), offsets.begin()); - if (edgelist.edge_weights != nullptr) { + if (edgelist.p_edge_weights != nullptr) { auto edge_first = thrust::make_zip_iterator(thrust::make_tuple( edgelist.p_src_vertices, edgelist.p_dst_vertices, edgelist.p_edge_weights)); thrust::for_each(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), @@ -98,8 +109,8 @@ auto edge_list_to_compressed_sparse(raft::handel_t &handle, auto minor = store_transposed ? d : s; auto start = p_offsets[major - major_first]; auto degree = p_offsets[(major - major_first) + 1] - start; - auto idx = atmoicAdd(p_indices + (degree - 1), - 1); // use the last element as a counter + auto idx = atomicAdd(p_indices + (degree - 1), + vertex_t{1}); // use the last element as a counter // FIXME: we can actually store minor - minor_first instead of minor to save // memory if minor can be larger than 32 bit but minor - minor_first fits // within 32 bit @@ -120,8 +131,8 @@ auto edge_list_to_compressed_sparse(raft::handel_t &handle, auto minor = store_transposed ? d : s; auto start = p_offsets[major - major_first]; auto degree = p_offsets[(major - major_first) + 1] - start; - auto idx = atmoicAdd(p_indices + (degree - 1), - 1); // use the last element as a counter + auto idx = atomicAdd(p_indices + (degree - 1), + vertex_t{1}); // use the last element as a counter // FIXME: we can actually store minor - minor_first instead of minor to save // memory if minor can be larger than 32 bit but minor - minor_first fits // within 32 bit @@ -130,14 +141,17 @@ auto edge_list_to_compressed_sparse(raft::handel_t &handle, }); } - return std::make_tuple(offsets, indices, weights); + return std::make_tuple(std::move(offsets), std::move(indices), std::move(weights)); } +// FIXME: better move this elsewhere, this can be reused in graph_device_view.cuh to compute degree +// as well. // compute the numbers of nonzeros in rows of the (transposed) graph adjacency matrix +template rmm::device_uvector compute_row_degree( - raft::handle_t &handle, - std::vector> const &adj_matrix_partition_offsets{}; - bool hypergraph_partitioned) + raft::handle_t const &handle, + std::vector> const &adj_matrix_partition_offsets, + partition_t const &partition) { auto &comm_p_row = handle.get_subcomm(comm_p_row_key); auto comm_p_row_rank = comm_p_row.get_rank(); @@ -146,46 +160,49 @@ rmm::device_uvector compute_row_degree( auto comm_p_col_rank = comm_p_col.get_rank(); auto comm_p_col_size = comm_p_col.get_size(); - rmm::device_uvector local_degrees{}; - rmm::device_uvector degrees{}; + rmm::device_uvector local_degrees(0, handle.get_stream()); + rmm::device_uvector degrees(0, handle.get_stream()); vertex_t max_num_local_degrees{0}; for (int i = 0; i < comm_p_col_size; ++i) { - auto vertex_partition_id = hypergraph_partitioned ? comm_p_row_size * i + row_rank - : comm_p_col_size * comm_p_row_rank + i; - auto row_first = vertex_partition_offsets[vertex_partition_id]; - auto row_last = vertex_partition_offsets[vertex_partition_id + 1]; + auto vertex_partition_id = partition.hypergraph_partitioned + ? comm_p_row_size * i + comm_p_row_rank + : comm_p_col_size * comm_p_row_rank + i; + auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; max_num_local_degrees = std::max(max_num_local_degrees, row_last - row_first); if (i == comm_p_col_rank) { degrees.resize(row_last - row_first, handle.get_stream()); } } - local_degrees.reisze(local_degree_size, handle.get_stream()); + local_degrees.resize(max_num_local_degrees, handle.get_stream()); for (int i = 0; i < comm_p_col_size; ++i) { - auto vertex_partition_id = hypergraph_partitioned ? comm_p_row_size * i + row_rank - : comm_p_col_size * comm_p_row_rank + i; - auto row_first = vertex_partition_offsets[vertex_partition_id]; - auto row_last = vertex_partition_offsets[vertex_partition_id + 1]; + auto vertex_partition_id = partition.hypergraph_partitioned + ? comm_p_row_size * i + comm_p_row_rank + : comm_p_col_size * comm_p_row_rank + i; + auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; auto p_offsets = - hypergraph_partitioned - ? adj_matrix_partition_offsets_[i].data() - : adj_matrix_partition_offsets_[0].data() + - (row_first - vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); + partition.hypergraph_partitioned + ? adj_matrix_partition_offsets[i].data() + : adj_matrix_partition_offsets[0].data() + + (row_first - partition.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); thrust::transform(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), thrust::make_counting_iterator(vertex_t{0}), thrust::make_counting_iterator(row_last - row_first), local_degrees.data(), [p_offsets] __device__(auto i) { return p_offsets[i + 1] - p_offsets[i]; }); comm_p_row.reduce(local_degrees.data(), - i == comm_p_col_rank ? degrees.data() : nullptr, + i == comm_p_col_rank ? degrees.data() : static_cast(nullptr), degrees.size(), - raft::comms::get_type(), - hanlde.get_stream()); + raft::comms::op_t::SUM, + comm_p_col_rank, + handle.get_stream()); } return degrees; } template -std::vector segment_degree_sorted_vertex_partition(raft::handle_t &handle, +std::vector segment_degree_sorted_vertex_partition(raft::handle_t const &handle, DegreeIterator degree_first, DegreeIterator degree_last, ThresholdIterator threshold_first, @@ -205,7 +222,7 @@ std::vector segment_degree_sorted_vertex_partition(raft::handle_t &han degree_last, threshold_first, threshold_last, - d_segments_offsets.begin()); + d_segment_offsets.begin()); raft::update_host(h_segment_offsets.begin() + 1, d_segment_offsets.begin(), @@ -217,95 +234,174 @@ std::vector segment_degree_sorted_vertex_partition(raft::handle_t &han } // namespace -} // namespace experimental - template graph_t>:: - graph_t(raft::handel_t &handle, - std::vector> const &edge_lists, - vertex_partition_t const &vertex_partition, - std::vector> const &adj_matrix_partitions, + graph_t(raft::handle_t const &handle, + std::vector> const &edgelists, + partition_t const &partition, vertex_t number_of_vertices, edge_t number_of_edges, bool is_symmetric, bool is_multigraph, bool is_weighted, + bool sorted_by_global_degree_within_vertex_partition, bool do_expensive_check) : number_of_vertices_(number_of_vertices), number_of_edges_(number_of_edges), properties_({is_symmetric, is_multigraph, is_weighted}), handle_ptr_(&handle), - vertex_partition_(vertex_partition), - adj_matrix_partitions_(adj_matrix_partitions) + partition_(partition) { - CUGRAPH_EXPECTS(edge_lists.size() == adj_matrix_partitions.size(), - "Invalid API parameter, edge_lists.size() != adj_matrix_partitions.size()"); + auto &comm_p_row = handle_ptr_->get_subcomm(comm_p_row_key); + auto comm_p_row_rank = comm_p_row.get_rank(); + auto comm_p_row_size = comm_p_row.get_size(); + auto &comm_p_col = handle_ptr_->get_subcomm(comm_p_col_key); + auto comm_p_col_rank = comm_p_col.get_rank(); + auto comm_p_col_size = comm_p_col.get_size(); // convert edge list (COO) to compressed sparse format (CSR or CSC) - adj_matrix_partition_offsets_.resize(edge_lists.size()); - adj_matrix_partition_indices_.resize(edge_lists.size()); - adj_matrix_partition_weights_.resize(edge_lists.size()); - for (size_t i = 0; i < edge_lists.size(); ++i) { - CUGRAPH_EXPECTS((is_weighted == false) || (edge_lists[i].p_edge_weights != nullptr), - "Invalid API parameter, edge_lists[i].p_edge_weights shoud not be nullptr if " + adj_matrix_partition_offsets_.reserve(edgelists.size()); + adj_matrix_partition_indices_.reserve(edgelists.size()); + adj_matrix_partition_weights_.reserve(edgelists.size()); + for (size_t i = 0; i < edgelists.size(); ++i) { + CUGRAPH_EXPECTS((is_weighted == false) || (edgelists[i].p_edge_weights != nullptr), + "Invalid API parameter, edgelists[i].p_edge_weights shoud not be nullptr if " "is_weighted == true"); - rmm::device_uvector offets{}; - rmm::device_uvector indices{}; - rmm::device_uvector weights{}; - - std::tie(offsets, indices, weights) = detail::edge_list_to_compressed_sparse( - handle, - edge_lists[i], - vertex_partition_.partition_offsets[adj_matrix_partitions[i].row_vertex_partition_first], - vertex_partition_.partition_offsets[adj_matrix_partitions[i].row_vertex_partition_last], - vertex_partition_.partition_offsets[adj_matrix_partitions[i].col_vertex_partition_first], - vertex_partition_.partition_offsets[adj_matrix_partitions[i].col_vertex_partition_last]); - adj_matrix_partitions_offsets_[i] = std::move(offsets); - adj_matrix_partitions_indices_[i] = std::move(indices); - if (is_weighted) { adj_matrix_partitions_weights_[i] = std::move(weights); } + auto row_first = partition_.hypergraph_partitioned + ? partition_.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank] + : partition_.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]; + auto row_last = + partition_.hypergraph_partitioned + ? partition_.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank + 1] + : partition_.vertex_partition_offsets[comm_p_col_size * (comm_p_row_rank + 1)]; + auto col_first = partition_.vertex_partition_offsets[comm_p_row_size * comm_p_col_rank]; + auto col_last = partition_.vertex_partition_offsets[comm_p_row_size * (comm_p_col_rank + 1)]; + + rmm::device_uvector offsets(0, handle_ptr_->get_stream()); + rmm::device_uvector indices(0, handle_ptr_->get_stream()); + rmm::device_uvector weights(0, handle_ptr_->get_stream()); + std::tie(offsets, indices, weights) = edge_list_to_compressed_sparse( + *handle_ptr_, edgelists[i], row_first, row_last, col_first, col_last); + adj_matrix_partition_offsets_.push_back(std::move(offsets)); + adj_matrix_partition_indices_.push_back(std::move(indices)); + adj_matrix_partition_weights_.push_back(std::move(weights)); } // update degree-based segment offsets (to be used for graph analytics kernel optimization) - auto &comm_p_row = handle_ptr_->get_subcomm(comm_p_row_key); - auto comm_p_row_rank = comm_p_row.get_rank(); - auto comm_p_row_size = comm_p_row.get_size(); + auto degrees = compute_row_degree(*handle_ptr_, adj_matrix_partition_offsets_, partition_); + + static_assert(num_segments_per_vertex_partition == 3); + static_assert((low_degree_threshold <= mid_degree_threshold) && + (mid_degree_threshold <= std::numeric_limits::max())); + rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, + handle_ptr_->get_stream()); + std::vector h_thresholds = {static_cast(low_degree_threshold), + static_cast(mid_degree_threshold)}; + raft::update_device( + d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), handle_ptr_->get_stream()); + + rmm::device_uvector segment_offsets(num_segments_per_vertex_partition + 1, + handle_ptr_->get_stream()); + segment_offsets.set_element_async(0, 0, handle_ptr_->get_stream()); + segment_offsets.set_element_async( + num_segments_per_vertex_partition, degrees.size(), handle_ptr_->get_stream()); + + thrust::upper_bound(rmm::exec_policy(handle_ptr_->get_stream())->on(handle_ptr_->get_stream()), + degrees.begin(), + degrees.end(), + d_thresholds.begin(), + d_thresholds.end(), + segment_offsets.begin() + 1); + + rmm::device_uvector aggregate_segment_offsets(comm_p_row_size * segment_offsets.size(), + handle_ptr_->get_stream()); + comm_p_row.allgather(segment_offsets.data(), + aggregate_segment_offsets.data(), + segment_offsets.size(), + handle_ptr_->get_stream()); + + vertex_partition_segment_offsets_.resize(comm_p_row_size * (segment_offsets.size())); + raft::update_host(vertex_partition_segment_offsets_.data(), + aggregate_segment_offsets.data(), + aggregate_segment_offsets.size(), + handle_ptr_->get_stream()); +} + +template +graph_t>:: + graph_t(raft::handle_t const &handle, + edgelist_t const &edgelist, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool sorted_by_global_degree, + bool do_expensive_check) + : number_of_vertices_(number_of_vertices), + number_of_edges_(number_of_edges), + properties_({is_symmetric, is_multigraph, is_weighted}), + handle_ptr_(&handle), + offsets_(rmm::device_uvector(0, handle.get_stream())), + indices_(rmm::device_uvector(0, handle.get_stream())), + weights_(rmm::device_uvector(0, handle.get_stream())) +{ + // convert edge list (COO) to compressed sparse format (CSR or CSC) + + CUGRAPH_EXPECTS( + (is_weighted == false) || (edgelist.p_edge_weights != nullptr), + "Invalid API parameter, edgelist.p_edge_weights shoud not be nullptr if is_weighted == true"); - auto degrees = compute_row_degree(); - - static_assert(detail::num_segments_per_subpartition == 3); - static_assert((detail::low_degree_threshold <= detail::high_degree_threshold) && - (detail::high_degree_threshold <= std::numeric_limits::max())); - auto segment_degree_threshold_first = - thrust::make_transform_iterator(thrust::make_counting_iterator(0), [] __device__(auto i) { - if (i == 0) { - return static_cast(detail::low_degree_threshold); - } else if (i == 1) { - return static_cast(detail::high_degree_threshold); - } else { - assert(0); // should not be reached - return vertex_t{0}; - } - }); - - auto segment_offsets = segment_degree_sorted_vertex_partition( - *handle_ptr_, - degrees.begin(), - degrees.end(), - segment_degree_threshold_first, - segment_degree_threshold_first + num_segments_per_subpartition - 1); - - comm_p_row.allgather(segment_offsets.begin(), - aggregate_segment_offsets_.begin(), - (num_segments_per_partition + 1), - raft::comms::get_type(), - hanlde_ptr_->get_stream()); + std::tie(offsets_, indices_, weights_) = edge_list_to_compressed_sparse( + *handle_ptr_, edgelist, vertex_t{0}, number_of_vertices_, vertex_t{0}, number_of_vertices_); + + // update degree-based segment offsets (to be used for graph analytics kernel optimization) + + rmm::device_uvector degrees(number_of_vertices_, handle_ptr_->get_stream()); + thrust::adjacent_difference( + rmm::exec_policy(handle_ptr_->get_stream())->on(handle_ptr_->get_stream()), + offsets_.begin() + 1, + offsets_.end(), + degrees.begin()); + + static_assert(num_segments_per_vertex_partition == 3); + static_assert((low_degree_threshold <= mid_degree_threshold) && + (mid_degree_threshold <= std::numeric_limits::max())); + rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, + handle_ptr_->get_stream()); + std::vector h_thresholds = {static_cast(low_degree_threshold), + static_cast(mid_degree_threshold)}; + raft::update_device( + d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), handle_ptr_->get_stream()); + + rmm::device_uvector segment_offsets(num_segments_per_vertex_partition + 1, + handle_ptr_->get_stream()); + segment_offsets.set_element_async(0, 0, handle_ptr_->get_stream()); + segment_offsets.set_element_async( + num_segments_per_vertex_partition, degrees.size(), handle_ptr_->get_stream()); + + thrust::upper_bound(rmm::exec_policy(handle_ptr_->get_stream())->on(handle_ptr_->get_stream()), + degrees.begin(), + degrees.end(), + d_thresholds.begin(), + d_thresholds.end(), + segment_offsets.begin() + 1); + + raft::update_host(segment_offsets_.data(), + segment_offsets.data(), + segment_offsets.size(), + handle_ptr_->get_stream()); } // explicit instantiation @@ -316,5 +412,5 @@ template class graph_t; template class graph_t; template class graph_t; -} // namespace cugraph +} // namespace experimental } // namespace cugraph \ No newline at end of file From 83bf444c0a532bcdf18a6f69cdc0408874dd5b6f Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 18 Aug 2020 23:43:39 -0400 Subject: [PATCH 03/77] update CMakeLists.txt --- cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3e84789e6b0..424a444bf23 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -354,6 +354,7 @@ add_library(cugraph SHARED src/components/connectivity.cu src/centrality/katz_centrality.cu src/centrality/betweenness_centrality.cu + src/experimental/graph.cu ) # From c5a1fe588274f83267a6d31b9644b3c6df75bcc1 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Wed, 19 Aug 2020 14:24:45 -0400 Subject: [PATCH 04/77] move graph.hpp to the experimental directory --- cpp/include/{exp_graph.hpp => experimental/graph.hpp} | 0 cpp/src/experimental/graph.cu | 3 +-- 2 files changed, 1 insertion(+), 2 deletions(-) rename cpp/include/{exp_graph.hpp => experimental/graph.hpp} (100%) diff --git a/cpp/include/exp_graph.hpp b/cpp/include/experimental/graph.hpp similarity index 100% rename from cpp/include/exp_graph.hpp rename to cpp/include/experimental/graph.hpp diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index c8c2867dad0..3423c1a9cd8 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -14,8 +14,7 @@ * limitations under the License. */ -#include - +#include #include #include From 19bdb947ed3ef0c1f1de823ad41c750b0e103a42 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Wed, 19 Aug 2020 15:06:11 -0400 Subject: [PATCH 05/77] refactor graph classes (create graph_base_t common for both SG and MG) --- cpp/include/experimental/graph.hpp | 94 +++++++++++++++++------------- cpp/src/experimental/graph.cu | 88 ++++++++++++++-------------- 2 files changed, 96 insertions(+), 86 deletions(-) diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index fce6e3096b4..7dae1c4d379 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -99,6 +99,40 @@ struct edgelist_t { edge_t number_of_edges{0}; }; +// Common for both single-GPU and multi-GPU versions +template +class graph_base_t { + public: + graph_base_t(raft::handle_t const &handle, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted) + : handle_ptr_(&handle), + number_of_vertices_(number_of_vertices), + number_of_edges_(number_of_edges), + properties_({is_symmetric, is_multigraph, is_weighted}){}; + + vertex_t get_number_of_vertices() const { return number_of_vertices_; } + edge_t get_number_of_edges() const { return number_of_edges_; } + + bool is_symmetric() const { return properties_.is_symmetric; } + bool is_multigraph() const { return properties_.is_multigraph; } + bool is_weighted() const { return properties_.is_weighted; } + + protected: + raft::handle_t const *get_handle_ptr() const { return handle_ptr_; }; + + private: + raft::handle_t const *handle_ptr_{nullptr}; + + vertex_t number_of_vertices_{0}; + edge_t number_of_edges_{0}; + + graph_properties_t properties_{}; +}; + template -class graph_t> { +class graph_t> + : public graph_base_t { public: + using vertex_type = vertex_t; + using edge_type = edge_t; + using weight_type = weight_t; + static constexpr bool is_adj_matrix_transposed = store_transposed; + static constexpr bool is_multi_gpu = multi_gpu; + graph_t(raft::handle_t const &handle, std::vector> const &edge_lists, partition_t const &partition, @@ -131,16 +167,9 @@ class graph_tget_comms().get_rank(); + auto comm_p_rank = this->get_handle_ptr()->get_comms().get_rank(); return partition_.vertex_partition_offsets[comm_p_rank + 1] - partition_.vertex_partition_offsets[comm_p_rank]; } @@ -173,13 +202,6 @@ class graph_t partition_{}; std::vector> adj_matrix_partition_offsets_{}; @@ -197,13 +219,15 @@ template -class graph_t> { +class graph_t> + : public graph_base_t { public: + using vertex_type = vertex_t; + using edge_type = edge_t; + using weight_type = weight_t; + static constexpr bool is_adj_matrix_transposed = store_transposed; + static constexpr bool is_multi_gpu = multi_gpu; + graph_t(raft::handle_t const &handle, edgelist_t const &edge_list, vertex_t number_of_vertices, @@ -214,14 +238,7 @@ class graph_tget_number_of_vertices(); } // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this // function will be eventually removed. @@ -236,13 +253,6 @@ class graph_t offsets_; rmm::device_uvector indices_; rmm::device_uvector weights_; diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 3423c1a9cd8..e10bb869acf 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -249,18 +249,17 @@ graph_t( + handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), partition_(partition) { - auto &comm_p_row = handle_ptr_->get_subcomm(comm_p_row_key); + auto &comm_p_row = this->get_handle_ptr()->get_subcomm(comm_p_row_key); auto comm_p_row_rank = comm_p_row.get_rank(); auto comm_p_row_size = comm_p_row.get_size(); - auto &comm_p_col = handle_ptr_->get_subcomm(comm_p_col_key); + auto &comm_p_col = this->get_handle_ptr()->get_subcomm(comm_p_col_key); auto comm_p_col_rank = comm_p_col.get_rank(); auto comm_p_col_size = comm_p_col.get_size(); + auto default_stream = this->get_handle_ptr()->get_stream(); // convert edge list (COO) to compressed sparse format (CSR or CSC) @@ -282,11 +281,11 @@ graph_t offsets(0, handle_ptr_->get_stream()); - rmm::device_uvector indices(0, handle_ptr_->get_stream()); - rmm::device_uvector weights(0, handle_ptr_->get_stream()); + rmm::device_uvector offsets(0, this->get_handle_ptr()->get_stream()); + rmm::device_uvector indices(0, this->get_handle_ptr()->get_stream()); + rmm::device_uvector weights(0, this->get_handle_ptr()->get_stream()); std::tie(offsets, indices, weights) = edge_list_to_compressed_sparse( - *handle_ptr_, edgelists[i], row_first, row_last, col_first, col_last); + *(this->get_handle_ptr()), edgelists[i], row_first, row_last, col_first, col_last); adj_matrix_partition_offsets_.push_back(std::move(offsets)); adj_matrix_partition_indices_.push_back(std::move(indices)); adj_matrix_partition_weights_.push_back(std::move(weights)); @@ -294,25 +293,25 @@ graph_tget_handle_ptr()), adj_matrix_partition_offsets_, partition_); static_assert(num_segments_per_vertex_partition == 3); static_assert((low_degree_threshold <= mid_degree_threshold) && (mid_degree_threshold <= std::numeric_limits::max())); - rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, - handle_ptr_->get_stream()); + rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, default_stream); std::vector h_thresholds = {static_cast(low_degree_threshold), static_cast(mid_degree_threshold)}; raft::update_device( - d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), handle_ptr_->get_stream()); + d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); rmm::device_uvector segment_offsets(num_segments_per_vertex_partition + 1, - handle_ptr_->get_stream()); - segment_offsets.set_element_async(0, 0, handle_ptr_->get_stream()); + default_stream); + segment_offsets.set_element_async(0, 0, default_stream); segment_offsets.set_element_async( - num_segments_per_vertex_partition, degrees.size(), handle_ptr_->get_stream()); + num_segments_per_vertex_partition, degrees.size(), default_stream); - thrust::upper_bound(rmm::exec_policy(handle_ptr_->get_stream())->on(handle_ptr_->get_stream()), + thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), degrees.begin(), degrees.end(), d_thresholds.begin(), @@ -320,17 +319,17 @@ graph_t aggregate_segment_offsets(comm_p_row_size * segment_offsets.size(), - handle_ptr_->get_stream()); + default_stream); comm_p_row.allgather(segment_offsets.data(), aggregate_segment_offsets.data(), segment_offsets.size(), - handle_ptr_->get_stream()); + default_stream); vertex_partition_segment_offsets_.resize(comm_p_row_size * (segment_offsets.size())); raft::update_host(vertex_partition_segment_offsets_.data(), aggregate_segment_offsets.data(), aggregate_segment_offsets.size(), - handle_ptr_->get_stream()); + default_stream); } template ( + handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), offsets_(rmm::device_uvector(0, handle.get_stream())), indices_(rmm::device_uvector(0, handle.get_stream())), weights_(rmm::device_uvector(0, handle.get_stream())) { + auto default_stream = this->get_handle_ptr()->get_stream(); + // convert edge list (COO) to compressed sparse format (CSR or CSC) CUGRAPH_EXPECTS( (is_weighted == false) || (edgelist.p_edge_weights != nullptr), "Invalid API parameter, edgelist.p_edge_weights shoud not be nullptr if is_weighted == true"); - std::tie(offsets_, indices_, weights_) = edge_list_to_compressed_sparse( - *handle_ptr_, edgelist, vertex_t{0}, number_of_vertices_, vertex_t{0}, number_of_vertices_); + std::tie(offsets_, indices_, weights_) = + edge_list_to_compressed_sparse(*(this->get_handle_ptr()), + edgelist, + vertex_t{0}, + this->get_number_of_vertices(), + vertex_t{0}, + this->get_number_of_vertices()); // update degree-based segment offsets (to be used for graph analytics kernel optimization) - rmm::device_uvector degrees(number_of_vertices_, handle_ptr_->get_stream()); - thrust::adjacent_difference( - rmm::exec_policy(handle_ptr_->get_stream())->on(handle_ptr_->get_stream()), - offsets_.begin() + 1, - offsets_.end(), - degrees.begin()); + rmm::device_uvector degrees(this->get_number_of_vertices(), default_stream); + thrust::adjacent_difference(rmm::exec_policy(default_stream)->on(default_stream), + offsets_.begin() + 1, + offsets_.end(), + degrees.begin()); static_assert(num_segments_per_vertex_partition == 3); static_assert((low_degree_threshold <= mid_degree_threshold) && (mid_degree_threshold <= std::numeric_limits::max())); - rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, - handle_ptr_->get_stream()); + rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, default_stream); std::vector h_thresholds = {static_cast(low_degree_threshold), static_cast(mid_degree_threshold)}; raft::update_device( - d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), handle_ptr_->get_stream()); + d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); rmm::device_uvector segment_offsets(num_segments_per_vertex_partition + 1, - handle_ptr_->get_stream()); - segment_offsets.set_element_async(0, 0, handle_ptr_->get_stream()); + default_stream); + segment_offsets.set_element_async(0, 0, default_stream); segment_offsets.set_element_async( - num_segments_per_vertex_partition, degrees.size(), handle_ptr_->get_stream()); + num_segments_per_vertex_partition, degrees.size(), default_stream); - thrust::upper_bound(rmm::exec_policy(handle_ptr_->get_stream())->on(handle_ptr_->get_stream()), + thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), degrees.begin(), degrees.end(), d_thresholds.begin(), d_thresholds.end(), segment_offsets.begin() + 1); - raft::update_host(segment_offsets_.data(), - segment_offsets.data(), - segment_offsets.size(), - handle_ptr_->get_stream()); + raft::update_host( + segment_offsets_.data(), segment_offsets.data(), segment_offsets.size(), default_stream); } // explicit instantiation From 12cf609be8ae08f27140e2f9841d4f81ffb03af8 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 21 Aug 2020 00:59:16 -0400 Subject: [PATCH 06/77] add graph_view.hpp & graph_utils.cuh --- .../experimental/detail/graph_utils.cuh | 89 ++++++ cpp/include/experimental/graph.hpp | 180 +++-------- cpp/include/experimental/graph_view.hpp | 284 ++++++++++++++++++ cpp/src/experimental/graph.cu | 107 +++---- 4 files changed, 455 insertions(+), 205 deletions(-) create mode 100644 cpp/include/experimental/detail/graph_utils.cuh create mode 100644 cpp/include/experimental/graph_view.hpp diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh new file mode 100644 index 00000000000..7536c180471 --- /dev/null +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include +#include + +#include + +#include + +namespace cugraph { +namespace experimental { +namespace detail { + +// compute the numbers of nonzeros in rows of the (transposed) graph adjacency matrix +template +rmm::device_uvector compute_row_degree( + raft::handle_t const &handle, + std::vector> const &adj_matrix_partition_offsets, + partition_t const &partition) +{ + auto &comm_p_row = handle.get_subcomm(comm_p_row_key); + auto comm_p_row_rank = comm_p_row.get_rank(); + auto comm_p_row_size = comm_p_row.get_size(); + auto &comm_p_col = handle.get_subcomm(comm_p_col_key); + auto comm_p_col_rank = comm_p_col.get_rank(); + auto comm_p_col_size = comm_p_col.get_size(); + + rmm::device_uvector local_degrees(0, handle.get_stream()); + rmm::device_uvector degrees(0, handle.get_stream()); + + vertex_t max_num_local_degrees{0}; + for (int i = 0; i < comm_p_col_size; ++i) { + auto vertex_partition_id = partition.hypergraph_partitioned + ? comm_p_row_size * i + comm_p_row_rank + : comm_p_col_size * comm_p_row_rank + i; + auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + max_num_local_degrees = std::max(max_num_local_degrees, row_last - row_first); + if (i == comm_p_col_rank) { degrees.resize(row_last - row_first, handle.get_stream()); } + } + local_degrees.resize(max_num_local_degrees, handle.get_stream()); + for (int i = 0; i < comm_p_col_size; ++i) { + auto vertex_partition_id = partition.hypergraph_partitioned + ? comm_p_row_size * i + comm_p_row_rank + : comm_p_col_size * comm_p_row_rank + i; + auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + auto p_offsets = + partition.hypergraph_partitioned + ? adj_matrix_partition_offsets[i].data() + : adj_matrix_partition_offsets[0].data() + + (row_first - partition.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); + thrust::transform(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), + thrust::make_counting_iterator(vertex_t{0}), + thrust::make_counting_iterator(row_last - row_first), + local_degrees.data(), + [p_offsets] __device__(auto i) { return p_offsets[i + 1] - p_offsets[i]; }); + comm_p_row.reduce(local_degrees.data(), + i == comm_p_col_rank ? degrees.data() : static_cast(nullptr), + degrees.size(), + raft::comms::op_t::SUM, + comm_p_col_rank, + handle.get_stream()); + } + + return degrees; +} + +} // namespace detail +} // namespace experimental +} // namespace cugraph diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index 7dae1c4d379..95edb8e7201 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -15,8 +15,7 @@ */ #pragma once -// FIXME: this file should be renamed to graph.hpp. - +#include #include #include @@ -30,67 +29,10 @@ namespace cugraph { namespace experimental { -// FIXME: threshold values require tuning -size_t constexpr low_degree_threshold{raft::warp_size()}; -size_t constexpr mid_degree_threshold{1024}; -size_t constexpr num_segments_per_vertex_partition{3}; - // FIXME: these should better be defined somewhere else. std::string const comm_p_row_key = "comm_p_row"; std::string const comm_p_col_key = "comm_p_key"; -/** - * @brief store vertex partitioning map - * - * We need to partition 1D vertex arrays (storing per vertex values) and the 2D graph adjacency - * matrix (or transposed 2D graph adjacency matrix) of G. An 1D vertex array of size V is divided to - * P linear partitions; each partition has the size close to V / P. The 2D matrix is partitioned to - * 1) P or 2) P * P_col rectangular partitions. - * - * 1) is the default. One GPU will be responsible for 1 rectangular partition. The matrix will be - * horizontally partitioned first to P_row slabs (by combining P_col vertex partitions). Each slab - * will be further vertically partitioned to P_col rectangles (by combining P_row vertex - * partitions). Each rectangular partition will have the size close to V / P_row by V / P_col. - * - * 2) is to support hyper-graph partitioning. We may apply hyper-graph partitioning to divide V - * vertices to P groups minimizing edge cuts across groups while balancing the number of vertices in - * each group. We will also renumber vertices so the vertices in each group are mapped to - * consecutive integers. Then, there will be more non-zeros in the diagonal partitions of the 2D - * graph adjacency matrix (or the transposed 2D graph adjacency matrix) than the off-diagonal - * partitions. The strategy 1) does not balance the number of nonzeros if hyper-graph partitioning - * is applied. To solve this problem, the matrix is first horizontally partitioned to P (instead of - * P_row) slabs, then each slab will be further vertically partitioned to P_col rectangles. One GPU - * will be responsible P_col rectangular partitions in this case. See E. G. Boman et. al., “Scalable - * matrix computations on large scale-free graphs using 2D graph partitioning”, 2013 for additional - * detail. - * - * In case of 1), a GPU with (row_rank, col_rank) will be responsible for one rectangular partition - * [a,b) by [c,d) where - * a = vertex_partition_offsets[P_col * row_rank], - * b = vertex_partition_offsets[p_col * (row_rank + 1)], - * c = vertex_partition_offsets[P_row * col_rank], and - * d = vertex_partition_offsets[p_row * (col_rank + 1)] - * - * In case of 2), a GPU with (row_rank, col_rank) will be responsible for P_col rectangular - * partitions [a_i,b_i) by [c,d) where - * a_i = vertex_partition_offsets[P_row * i + row_rank] and - * b_i = vertex_partition_offsets[P_row * i + row_rank + 1]. - * c and d are same to 1) and i = [0, P_col). - * - * @tparam vertex_t Type of vertex ID - */ -template -struct partition_t { - std::vector vertex_partition_offsets{}; // size = P + 1 - bool hypergraph_partitioned{false}; -}; - -struct graph_properties_t { - bool is_symmetric{false}; - bool is_multigraph{false}; - bool is_weighted{false}; -}; - template struct edgelist_t { vertex_t const *p_src_vertices{nullptr}; @@ -99,40 +41,6 @@ struct edgelist_t { edge_t number_of_edges{0}; }; -// Common for both single-GPU and multi-GPU versions -template -class graph_base_t { - public: - graph_base_t(raft::handle_t const &handle, - vertex_t number_of_vertices, - edge_t number_of_edges, - bool is_symmetric, - bool is_multigraph, - bool is_weighted) - : handle_ptr_(&handle), - number_of_vertices_(number_of_vertices), - number_of_edges_(number_of_edges), - properties_({is_symmetric, is_multigraph, is_weighted}){}; - - vertex_t get_number_of_vertices() const { return number_of_vertices_; } - edge_t get_number_of_edges() const { return number_of_edges_; } - - bool is_symmetric() const { return properties_.is_symmetric; } - bool is_multigraph() const { return properties_.is_multigraph; } - bool is_weighted() const { return properties_.is_weighted; } - - protected: - raft::handle_t const *get_handle_ptr() const { return handle_ptr_; }; - - private: - raft::handle_t const *handle_ptr_{nullptr}; - - vertex_t number_of_vertices_{0}; - edge_t number_of_edges_{0}; - - graph_properties_t properties_{}; -}; - template class graph_t> - : public graph_base_t { + : public detail::graph_base_t { public: using vertex_type = vertex_t; using edge_type = edge_t; @@ -160,7 +68,6 @@ class graph_t> const &edge_lists, partition_t const &partition, vertex_t number_of_vertices, - edge_t number_of_edges, bool is_symmetric, bool is_multigraph, bool is_weighted, @@ -174,39 +81,40 @@ class graph_t view() { - CUGRAPH_EXPECTS(adj_matrix_partition_weights_.size() == 1, - "weights() is valid only when there is one rectangular partition per GPU."); - return adj_matrix_partition_weights_[0].data(); + std::vector offsets(adj_matrix_partition_offsets_.size(), nullptr); + std::vector indices(adj_matrix_partition_indices_.size(), nullptr); + std::vector weights(adj_matrix_partition_weights_.size(), nullptr); + for (size_t i = 0; i < offsets.size(); ++i) { + offsets[i] = adj_matrix_partition_offsets_[i].data(); + indices[i] = adj_matrix_partition_indices_[i].data(); + if (this->is_weighted()) { weights[i] = adj_matrix_partition_weights_[i].data(); } + } + + return graph_view_t( + *(this->get_handle_ptr()), + offsets, + indices, + weights, + vertex_partition_segment_offsets_, + partition_, + this->get_number_of_vertices(), + this->get_number_of_edges(), + this->is_symmetric(), + this->is_multigraph(), + this->is_weighted(), + vertex_partition_segment_offsets_.size() > 0, + false); } private: - partition_t partition_{}; - std::vector> adj_matrix_partition_offsets_{}; std::vector> adj_matrix_partition_indices_{}; std::vector> adj_matrix_partition_weights_{}; + + partition_t partition_{}; + std::vector vertex_partition_segment_offsets_{}; // segment offsets within the vertex partition based on // vertex degree, relevant only if @@ -220,7 +128,7 @@ template class graph_t> - : public graph_base_t { + : public detail::graph_base_t { public: using vertex_type = vertex_t; using edge_type = edge_t; @@ -231,7 +139,6 @@ class graph_t const &edge_list, vertex_t number_of_vertices, - edge_t number_of_edges, bool is_symmetric, bool is_multigraph, bool is_weighted, @@ -240,17 +147,22 @@ class graph_tget_number_of_vertices(); } - // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this - // function will be eventually removed. - edge_t const *offsets() const { return offsets_.data(); } - - // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this - // function will be eventually removed. - vertex_t const *indices() const { return indices_.data(); } - - // FIXME: for compatibility with cuGraph analytics expecting either CSR or CSC format, this - // function will be eventually removed. - weight_t const *weights() const { return weights_.data(); } + graph_view_t view() + { + return graph_view_t( + *(this->get_handle_ptr()), + offsets_.data(), + indices_.data(), + weights_.data(), + segment_offsets_, + this->get_number_of_vertices(), + this->get_number_of_edges(), + this->is_symmetric(), + this->is_multigraph(), + this->is_weighted(), + segment_offsets_.size() > 0, + false); + } private: rmm::device_uvector offsets_; diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp new file mode 100644 index 00000000000..f065662f401 --- /dev/null +++ b/cpp/include/experimental/graph_view.hpp @@ -0,0 +1,284 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include + +#include +#include +#include +#include + +namespace cugraph { +namespace experimental { + +/** + * @brief store vertex partitioning map + * + * We need to partition 1D vertex arrays (storing per vertex values) and the 2D graph adjacency + * matrix (or transposed 2D graph adjacency matrix) of G. An 1D vertex array of size V is divided to + * P linear partitions; each partition has the size close to V / P. The 2D matrix is partitioned to + * 1) P or 2) P * P_col rectangular partitions. + * + * 1) is the default. One GPU will be responsible for 1 rectangular partition. The matrix will be + * horizontally partitioned first to P_row slabs (by combining P_col vertex partitions). Each slab + * will be further vertically partitioned to P_col rectangles (by combining P_row vertex + * partitions). Each rectangular partition will have the size close to V / P_row by V / P_col. + * + * 2) is to support hyper-graph partitioning. We may apply hyper-graph partitioning to divide V + * vertices to P groups minimizing edge cuts across groups while balancing the number of vertices in + * each group. We will also renumber vertices so the vertices in each group are mapped to + * consecutive integers. Then, there will be more non-zeros in the diagonal partitions of the 2D + * graph adjacency matrix (or the transposed 2D graph adjacency matrix) than the off-diagonal + * partitions. The strategy 1) does not balance the number of nonzeros if hyper-graph partitioning + * is applied. To solve this problem, the matrix is first horizontally partitioned to P (instead of + * P_row) slabs, then each slab will be further vertically partitioned to P_col rectangles. One GPU + * will be responsible P_col rectangular partitions in this case. See E. G. Boman et. al., “Scalable + * matrix computations on large scale-free graphs using 2D graph partitioning”, 2013 for additional + * detail. + * + * In case of 1), a GPU with (row_rank, col_rank) will be responsible for one rectangular partition + * [a,b) by [c,d) where + * a = vertex_partition_offsets[P_col * row_rank], + * b = vertex_partition_offsets[p_col * (row_rank + 1)], + * c = vertex_partition_offsets[P_row * col_rank], and + * d = vertex_partition_offsets[p_row * (col_rank + 1)] + * + * In case of 2), a GPU with (row_rank, col_rank) will be responsible for P_col rectangular + * partitions [a_i,b_i) by [c,d) where + * a_i = vertex_partition_offsets[P_row * i + row_rank] and + * b_i = vertex_partition_offsets[P_row * i + row_rank + 1]. + * c and d are same to 1) and i = [0, P_col). + * + * @tparam vertex_t Type of vertex ID + */ +template +struct partition_t { + std::vector vertex_partition_offsets{}; // size = P + 1 + bool hypergraph_partitioned{false}; +}; + +namespace detail { + +struct graph_properties_t { + bool is_symmetric{false}; + bool is_multigraph{false}; + bool is_weighted{false}; +}; + +// Common for both graph_view_t & graph_t and both single-GPU & multi-GPU versions +template +class graph_base_t { + public: + graph_base_t(raft::handle_t const& handle, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted) + : handle_ptr_(&handle), + number_of_vertices_(number_of_vertices), + number_of_edges_(number_of_edges), + properties_({is_symmetric, is_multigraph, is_weighted}){}; + + vertex_t get_number_of_vertices() const { return number_of_vertices_; } + edge_t get_number_of_edges() const { return number_of_edges_; } + + bool is_symmetric() const { return properties_.is_symmetric; } + bool is_multigraph() const { return properties_.is_multigraph; } + bool is_weighted() const { return properties_.is_weighted; } + + protected: + raft::handle_t const* get_handle_ptr() const { return handle_ptr_; }; + + private: + raft::handle_t const* handle_ptr_{nullptr}; + + vertex_t number_of_vertices_{0}; + edge_t number_of_edges_{0}; + + graph_properties_t properties_{}; +}; + +} // namespace detail + +template +class graph_view_t; + +// multi-GPU version +template +class graph_view_t> + : public detail::graph_base_t { + public: + using vertex_type = vertex_t; + using edge_type = edge_t; + using weight_type = weight_t; + static constexpr bool is_adj_matrix_transposed = store_transposed; + static constexpr bool is_multi_gpu = multi_gpu; + + graph_view_t(raft::handle_t const& handle, + std::vector const& adj_matrix_partition_offsets, + std::vector const& adj_matrix_partition_indices, + std::vector const& adj_matrix_partition_weights, + std::vector const& vertex_partition_segment_offsets, + partition_t const& partition, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool sorted_by_global_degree_within_vertex_partition, + bool do_expensive_check = false) + : detail::graph_base_t( + handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + adj_matrix_partition_offsets_(adj_matrix_partition_offsets), + adj_matrix_partition_indices_(adj_matrix_partition_indices), + adj_matrix_partition_weights_(adj_matrix_partition_weights), + partition_(partition), + vertex_partition_segment_offsets_(vertex_partition_segment_offsets) + { + // FIXME: error check + } + + vertex_t get_number_of_local_vertices() const + { + auto comm_p_rank = this->get_handle_ptr()->get_comms().get_rank(); + return partition_.vertex_partition_offsets[comm_p_rank + 1] - + partition_.vertex_partition_offsets[comm_p_rank]; + } + + size_t get_number_of_adj_matrix_partitions() { return adj_matrix_partition_offsets_.size(); } + + // Better avoid direct invocation in application code. + // This is mainly for pattern accelerator implementation. + // This function may disappear in the future if we swtich to CSR + DCSR (or CSC + DCSC) + edge_t const* offsets(size_t adj_matrix_partition_idx) const + { + return adj_matrix_partition_offsets_[adj_matrix_partition_idx].data(); + } + + // Better avoid direct invocation in application code. + // This is mainly for pattern accelerator implementation. + // This function may disappear in the future if we swtich to CSR + DCSR (or CSC + DCSC) + vertex_t const* indices(size_t adj_matrix_partition_idx) const + { + return adj_matrix_partition_indices_[adj_matrix_partition_idx].data(); + } + + // Better avoid direct invocation in application code. + // This is mainly for pattern accelerator implementation. + // This function may disappear in the future if we swtich to CSR + DCSR (or CSC + DCSC) + weight_t const* weights(size_t adj_matrix_partition_idx) const + { + return adj_matrix_partition_weights_[adj_matrix_partition_idx].data(); + } + + private: + std::vector adj_matrix_partition_offsets_{}; + std::vector adj_matrix_partition_indices_{}; + std::vector adj_matrix_partition_weights_{}; + + partition_t partition_{}; + + std::vector + vertex_partition_segment_offsets_{}; // segment offsets within the vertex partition based on + // vertex degree, relevant only if + // sorted_by_global_degree_within_vertex_partition is true +}; + +// single-GPU version +template +class graph_view_t> + : public detail::graph_base_t { + public: + using vertex_type = vertex_t; + using edge_type = edge_t; + using weight_type = weight_t; + static constexpr bool is_adj_matrix_transposed = store_transposed; + static constexpr bool is_multi_gpu = multi_gpu; + + graph_view_t(raft::handle_t const& handle, + edge_t const* offsets, + vertex_t const* indices, + weight_t const* weights, + std::vector const& segment_offsets, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool sorted_by_global_degree, + bool do_expensive_check = false) + : detail::graph_base_t( + handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + offsets_(offsets), + indices_(indices), + weights_(weights), + segment_offsets_(segment_offsets) + { + // FIXME: error check + } + + vertex_t get_number_of_local_vertices() const { return this->get_number_of_vertices(); } + + // Better avoid direct invocation in application code. + // This is mainly for pattern accelerator implementation. + edge_t const* offsets() const { return offsets_; } + + // Better avoid direct invocation in application code. + // This is mainly for pattern accelerator implementation. + vertex_t const* indices() const { return indices_; } + + // Better avoid direct invocation in application code. + // This is mainly for pattern accelerator implementation. + weight_t const* weights() const { return weights_; } + + private: + edge_t const* offsets_{nullptr}; + vertex_t const* indices_{nullptr}; + weight_t const* weights_{nullptr}; + std::vector segment_offsets_{}; // segment offsets based on vertex degree, relevant + // only if sorted_by_global_degree is true +}; + +} // namespace experimental +} // namespace cugraph \ No newline at end of file diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index e10bb869acf..e50aad5b50c 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -14,6 +14,7 @@ * limitations under the License. */ +#include #include #include @@ -27,9 +28,7 @@ #include #include #include -#include #include -#include #include #include @@ -39,6 +38,19 @@ namespace experimental { namespace { +// FIXME: threshold values require tuning +size_t constexpr low_degree_threshold{raft::warp_size()}; +size_t constexpr mid_degree_threshold{1024}; +size_t constexpr num_segments_per_vertex_partition{3}; + +template +edge_t sum_number_of_edges(std::vector> const &edgelists) +{ + edge_t number_of_edges{0}; + for (size_t i = 0; i < edgelists.size(); ++i) { number_of_edges += edgelists[i].number_of_edges; } + return number_of_edges; +} + template std:: tuple, rmm::device_uvector, rmm::device_uvector> @@ -108,7 +120,7 @@ std:: auto minor = store_transposed ? d : s; auto start = p_offsets[major - major_first]; auto degree = p_offsets[(major - major_first) + 1] - start; - auto idx = atomicAdd(p_indices + (degree - 1), + auto idx = atomicAdd(p_indices + (start + degree - 1), vertex_t{1}); // use the last element as a counter // FIXME: we can actually store minor - minor_first instead of minor to save // memory if minor can be larger than 32 bit but minor - minor_first fits @@ -130,7 +142,7 @@ std:: auto minor = store_transposed ? d : s; auto start = p_offsets[major - major_first]; auto degree = p_offsets[(major - major_first) + 1] - start; - auto idx = atomicAdd(p_indices + (degree - 1), + auto idx = atomicAdd(p_indices + (start + degree - 1), vertex_t{1}); // use the last element as a counter // FIXME: we can actually store minor - minor_first instead of minor to save // memory if minor can be larger than 32 bit but minor - minor_first fits @@ -143,63 +155,6 @@ std:: return std::make_tuple(std::move(offsets), std::move(indices), std::move(weights)); } -// FIXME: better move this elsewhere, this can be reused in graph_device_view.cuh to compute degree -// as well. -// compute the numbers of nonzeros in rows of the (transposed) graph adjacency matrix -template -rmm::device_uvector compute_row_degree( - raft::handle_t const &handle, - std::vector> const &adj_matrix_partition_offsets, - partition_t const &partition) -{ - auto &comm_p_row = handle.get_subcomm(comm_p_row_key); - auto comm_p_row_rank = comm_p_row.get_rank(); - auto comm_p_row_size = comm_p_row.get_size(); - auto &comm_p_col = handle.get_subcomm(comm_p_col_key); - auto comm_p_col_rank = comm_p_col.get_rank(); - auto comm_p_col_size = comm_p_col.get_size(); - - rmm::device_uvector local_degrees(0, handle.get_stream()); - rmm::device_uvector degrees(0, handle.get_stream()); - - vertex_t max_num_local_degrees{0}; - for (int i = 0; i < comm_p_col_size; ++i) { - auto vertex_partition_id = partition.hypergraph_partitioned - ? comm_p_row_size * i + comm_p_row_rank - : comm_p_col_size * comm_p_row_rank + i; - auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; - max_num_local_degrees = std::max(max_num_local_degrees, row_last - row_first); - if (i == comm_p_col_rank) { degrees.resize(row_last - row_first, handle.get_stream()); } - } - local_degrees.resize(max_num_local_degrees, handle.get_stream()); - for (int i = 0; i < comm_p_col_size; ++i) { - auto vertex_partition_id = partition.hypergraph_partitioned - ? comm_p_row_size * i + comm_p_row_rank - : comm_p_col_size * comm_p_row_rank + i; - auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; - auto p_offsets = - partition.hypergraph_partitioned - ? adj_matrix_partition_offsets[i].data() - : adj_matrix_partition_offsets[0].data() + - (row_first - partition.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); - thrust::transform(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), - thrust::make_counting_iterator(vertex_t{0}), - thrust::make_counting_iterator(row_last - row_first), - local_degrees.data(), - [p_offsets] __device__(auto i) { return p_offsets[i + 1] - p_offsets[i]; }); - comm_p_row.reduce(local_degrees.data(), - i == comm_p_col_rank ? degrees.data() : static_cast(nullptr), - degrees.size(), - raft::comms::op_t::SUM, - comm_p_col_rank, - handle.get_stream()); - } - - return degrees; -} - template std::vector segment_degree_sorted_vertex_partition(raft::handle_t const &handle, DegreeIterator degree_first, @@ -243,14 +198,17 @@ graph_t> const &edgelists, partition_t const &partition, vertex_t number_of_vertices, - edge_t number_of_edges, bool is_symmetric, bool is_multigraph, bool is_weighted, bool sorted_by_global_degree_within_vertex_partition, bool do_expensive_check) - : graph_base_t( - handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + : detail::graph_base_t(handle, + number_of_vertices, + sum_number_of_edges(edgelists), + is_symmetric, + is_multigraph, + is_weighted), partition_(partition) { auto &comm_p_row = this->get_handle_ptr()->get_subcomm(comm_p_row_key); @@ -261,11 +219,13 @@ graph_tget_handle_ptr()->get_stream(); + // FIXME: error checks + // convert edge list (COO) to compressed sparse format (CSR or CSC) adj_matrix_partition_offsets_.reserve(edgelists.size()); adj_matrix_partition_indices_.reserve(edgelists.size()); - adj_matrix_partition_weights_.reserve(edgelists.size()); + adj_matrix_partition_weights_.reserve(this->is_weighted() ? edgelists.size() : 0); for (size_t i = 0; i < edgelists.size(); ++i) { CUGRAPH_EXPECTS((is_weighted == false) || (edgelists[i].p_edge_weights != nullptr), "Invalid API parameter, edgelists[i].p_edge_weights shoud not be nullptr if " @@ -288,13 +248,13 @@ graph_tget_handle_ptr()), edgelists[i], row_first, row_last, col_first, col_last); adj_matrix_partition_offsets_.push_back(std::move(offsets)); adj_matrix_partition_indices_.push_back(std::move(indices)); - adj_matrix_partition_weights_.push_back(std::move(weights)); + if (this->is_weighted()) { adj_matrix_partition_weights_.push_back(std::move(weights)); } } // update degree-based segment offsets (to be used for graph analytics kernel optimization) - auto degrees = - compute_row_degree(*(this->get_handle_ptr()), adj_matrix_partition_offsets_, partition_); + auto degrees = detail::compute_row_degree( + *(this->get_handle_ptr()), adj_matrix_partition_offsets_, partition_); static_assert(num_segments_per_vertex_partition == 3); static_assert((low_degree_threshold <= mid_degree_threshold) && @@ -341,20 +301,25 @@ graph_t const &edgelist, vertex_t number_of_vertices, - edge_t number_of_edges, bool is_symmetric, bool is_multigraph, bool is_weighted, bool sorted_by_global_degree, bool do_expensive_check) - : graph_base_t( - handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + : detail::graph_base_t(handle, + number_of_vertices, + edgelist.number_of_edges, + is_symmetric, + is_multigraph, + is_weighted), offsets_(rmm::device_uvector(0, handle.get_stream())), indices_(rmm::device_uvector(0, handle.get_stream())), weights_(rmm::device_uvector(0, handle.get_stream())) { auto default_stream = this->get_handle_ptr()->get_stream(); + // FIXME: error checks + // convert edge list (COO) to compressed sparse format (CSR or CSC) CUGRAPH_EXPECTS( From 8d784e44184d628c7a9e115761e8766d7791d68c Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 21 Aug 2020 00:59:37 -0400 Subject: [PATCH 07/77] add graph test --- cpp/tests/CMakeLists.txt | 9 + cpp/tests/experimental/graph_test.cpp | 245 ++++++++++++++++++++++++++ 2 files changed, 254 insertions(+) create mode 100644 cpp/tests/experimental/graph_test.cpp diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index e0f945639ca..0624e44bf09 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -268,6 +268,15 @@ if (BUILD_MPI) ConfigureTest(NCCL_DEGREE_TEST "${NCCL_DEGREE_TEST_SRC}" "") endif(BUILD_MPI) +################################################################################################### +# - Experimental Graph tests ---------------------------------------------------------------------- + +set(EXPERIMENTAL_GRAPH_TEST_SRCS + "${CMAKE_SOURCE_DIR}/../thirdparty/mmio/mmio.c" + "${CMAKE_CURRENT_SOURCE_DIR}/experimental/graph_test.cpp") + +ConfigureTest(EXPERIMENTAL_GRAPH_TEST "${EXPERIMENTAL_GRAPH_TEST_SRCS}" "") + ################################################################################################### ### enable testing ################################################################################ ################################################################################################### diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp new file mode 100644 index 00000000000..86ad13afa8b --- /dev/null +++ b/cpp/tests/experimental/graph_test.cpp @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governin_from_mtxg permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +template +std::tuple, std::vector, std::vector> graph_reference( + vertex_t const* p_src_vertices, + vertex_t const* p_dst_vertices, + weight_t const* p_edge_weights, + vertex_t number_of_vertices, + edge_t number_of_edges) +{ + std::vector offsets(number_of_vertices + 1, edge_t{0}); + std::vector indices(number_of_edges, vertex_t{0}); + std::vector weights(p_edge_weights != nullptr ? number_of_edges : 0, weight_t{0.0}); + + for (size_t i = 0; i < number_of_edges; ++i) { + if (store_transposed) { + auto d = p_dst_vertices[i]; + offsets[1 + d]++; + } else { + auto s = p_src_vertices[i]; + offsets[1 + s]++; + } + } + std::partial_sum(offsets.begin() + 1, offsets.end(), offsets.begin() + 1); + + for (size_t i = 0; i < number_of_edges; ++i) { + auto major = store_transposed ? p_src_vertices[i] : p_dst_vertices[i]; + auto minor = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; + auto start = offsets[major]; + auto degree = offsets[major + 1] - start; + auto idx = indices[start + degree - 1]++; + indices[idx] = minor; + if (p_edge_weights != nullptr) { weights[idx] = p_edge_weights[i]; } + } + + return std::make_tuple(std::move(offsets), std::move(indices), std::move(weights)); +} + +typedef struct Graph_Usecase_t { + std::string graph_file_path_; + std::string graph_file_full_path_; + bool test_weighted_; + + Graph_Usecase_t(std::string const& graph_file_path, bool test_weighted) + : graph_file_path_(graph_file_path), test_weighted_(test_weighted) + { + if ((graph_file_path_.length() > 0) && (graph_file_path[0] != '/')) { + graph_file_full_path_ = cugraph::test::get_rapids_dataset_root_dir() + "/" + graph_file_path_; + } else { + graph_file_full_path_ = graph_file_path_; + } + }; +} Graph_Usecase; + +class Tests_Graph : public ::testing::TestWithParam { + public: + Tests_Graph() {} + static void SetupTestCase() {} + static void TearDownTestCase() {} + + virtual void SetUp() {} + virtual void TearDown() {} + + template + void run_current_test(Graph_Usecase const& configuration) + { + MM_typecode mc{}; + vertex_t m{}; + vertex_t k{}; + edge_t nnz{}; + + FILE* file = fopen(configuration.graph_file_full_path_.c_str(), "r"); + ASSERT_NE(file, nullptr) << "fopen (" << configuration.graph_file_full_path_ << ") failure."; + + ASSERT_EQ(cugraph::test::mm_properties(file, 1, &mc, &m, &k, &nnz), 0) + << "could not read Matrix Market file properties\n"; + ASSERT_TRUE(mm_is_matrix(mc)); + ASSERT_TRUE(mm_is_coordinate(mc)); + ASSERT_FALSE(mm_is_complex(mc)); + ASSERT_FALSE(mm_is_skew(mc)); + + std::vector h_rows(nnz, vertex_t{0}); + std::vector h_cols(nnz, vertex_t{0}); + std::vector h_weights(nnz, weight_t{0.0}); + + ASSERT_EQ((cugraph::test::mm_to_coo( + file, 1, nnz, h_rows.data(), h_cols.data(), h_weights.data(), nullptr)), + 0) + << "could not read matrix data\n"; + ASSERT_EQ(fclose(file), 0); + + std::vector h_reference_offsets{}; + std::vector h_reference_indices{}; + std::vector h_reference_weights{}; + + std::tie(h_reference_offsets, h_reference_indices, h_reference_weights) = + graph_reference(h_rows.data(), + h_cols.data(), + configuration.test_weighted_ ? h_weights.data() : nullptr, + m, + nnz); + + raft::handle_t handle{}; + + rmm::device_uvector d_rows(nnz, handle.get_stream()); + rmm::device_uvector d_cols(nnz, handle.get_stream()); + rmm::device_uvector d_weights(configuration.test_weighted_ ? nnz : 0, + handle.get_stream()); + + raft::update_device(d_rows.data(), h_rows.data(), h_rows.size(), handle.get_stream()); + raft::update_device(d_cols.data(), h_cols.data(), h_cols.size(), handle.get_stream()); + if (configuration.test_weighted_) { + raft::update_device( + d_weights.data(), h_weights.data(), h_weights.size(), handle.get_stream()); + } + + CUDA_TRY(cudaStreamSynchronize(handle.get_stream())); + + cugraph::experimental::edgelist_t edgelist{ + d_rows.data(), d_cols.data(), configuration.test_weighted_ ? d_weights.data() : nullptr, nnz}; + + CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement + + auto graph = + cugraph::experimental::graph_t( + handle, edgelist, m, mm_is_symmetric(mc), false, configuration.test_weighted_, false, true); + + auto graph_view = graph.view(); + + CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement + + std::vector h_cugraph_offsets{}; + std::vector h_cugraph_indices{}; + std::vector h_cugraph_weights{}; + + raft::update_host(h_cugraph_offsets.data(), + graph_view.offsets(), + graph_view.get_number_of_vertices() + 1, + handle.get_stream()); + raft::update_host(h_cugraph_indices.data(), + graph_view.indices(), + graph_view.get_number_of_edges(), + handle.get_stream()); + raft::update_host(h_cugraph_weights.data(), + graph_view.weights(), + graph_view.get_number_of_edges(), + handle.get_stream()); + + CUDA_TRY(cudaStreamSynchronize(handle.get_stream())); + + ASSERT_TRUE( + std::equal(h_reference_offsets.begin(), h_reference_offsets.end(), h_cugraph_offsets.begin())) + << "Graph compressed sparse format offsets do not match with the reference values."; + ASSERT_EQ(h_reference_weights.size(), h_cugraph_weights.size()); + for (vertex_t i = 0; i < m; ++i) { + auto start = h_reference_offsets[i]; + auto degree = h_reference_offsets[i + 1] - start; + if (configuration.test_weighted_) { + std::vector> reference_pairs(degree); + std::vector> cugraph_pairs(degree); + for (edge_t j = 0; j < degree; ++j) { + reference_pairs[j] = + std::make_tuple(h_reference_indices[start + j], h_reference_weights[start + j]); + cugraph_pairs[j] = + std::make_tuple(h_cugraph_indices[start + j], h_cugraph_weights[start + j]); + } + std::sort(reference_pairs.begin(), reference_pairs.end()); + std::sort(cugraph_pairs.begin(), cugraph_pairs.end()); + ASSERT_TRUE( + std::equal(reference_pairs.begin(), reference_pairs.end(), cugraph_pairs.begin())) + << "Graph compressed sparse format indices & weights for vertex " << i + << " do not match with the reference values."; + } else { + std::vector reference_indices(h_reference_indices.begin() + start, + h_reference_indices.begin() + (start + degree)); + std::vector cugraph_indices(h_cugraph_indices.begin() + start, + h_cugraph_indices.begin() + (start + degree)); + std::sort(reference_indices.begin(), reference_indices.end()); + std::sort(cugraph_indices.begin(), cugraph_indices.end()); + ASSERT_TRUE( + std::equal(reference_indices.begin(), reference_indices.end(), cugraph_indices.begin())) + << "Graph compressed sparse format indices for vertex " << i + << " do not match with the reference values."; + } + } + } +}; + +// FIXME: add tests for type combinations +TEST_P(Tests_Graph, CheckInt32Int32FloatFalse) +{ + run_current_test(GetParam()); +} + +// FIXME: add tests for type combinations +TEST_P(Tests_Graph, CheckInt32Int32FloatTrue) +{ + run_current_test(GetParam()); +} + +INSTANTIATE_TEST_CASE_P(simple_test, + Tests_Graph, + ::testing::Values(Graph_Usecase("test/datasets/karate.mtx", false), + Graph_Usecase("test/datasets/karate.mtx", true), + Graph_Usecase("test/datasets/web-Google.mtx", false), + Graph_Usecase("test/datasets/web-Google.mtx", true), + Graph_Usecase("test/datasets/ljournal-2008.mtx", false), + Graph_Usecase("test/datasets/ljournal-2008.mtx", true), + Graph_Usecase("test/datasets/webbase-1M.mtx", false), + Graph_Usecase("test/datasets/webbase-1M.mtx", true))); + +CUGRAPH_TEST_PROGRAM_MAIN() \ No newline at end of file From e25b79d9ff84c4c1a53ab974b8b026d05bae7276 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 21 Aug 2020 01:07:06 -0400 Subject: [PATCH 08/77] rename compute_row_degree to compute_major_degree to avoid confusion --- .../experimental/detail/graph_utils.cuh | 20 +++++++++---------- cpp/include/experimental/graph.hpp | 4 ---- cpp/include/experimental/graph_view.hpp | 4 ++++ cpp/src/experimental/graph.cu | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index 7536c180471..0daa8f89d5c 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -29,9 +29,9 @@ namespace cugraph { namespace experimental { namespace detail { -// compute the numbers of nonzeros in rows of the (transposed) graph adjacency matrix +// compute the numbers of nonzeros in rows (of the graph adjacency matrix, if store_transposed = false) or columns (of the graph adjacency matrix, if store_transposed = true) template -rmm::device_uvector compute_row_degree( +rmm::device_uvector compute_major_degree( raft::handle_t const &handle, std::vector> const &adj_matrix_partition_offsets, partition_t const &partition) @@ -51,26 +51,26 @@ rmm::device_uvector compute_row_degree( auto vertex_partition_id = partition.hypergraph_partitioned ? comm_p_row_size * i + comm_p_row_rank : comm_p_col_size * comm_p_row_rank + i; - auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; - max_num_local_degrees = std::max(max_num_local_degrees, row_last - row_first); - if (i == comm_p_col_rank) { degrees.resize(row_last - row_first, handle.get_stream()); } + auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + max_num_local_degrees = std::max(max_num_local_degrees, major_last - major_first); + if (i == comm_p_col_rank) { degrees.resize(major_last - major_first, handle.get_stream()); } } local_degrees.resize(max_num_local_degrees, handle.get_stream()); for (int i = 0; i < comm_p_col_size; ++i) { auto vertex_partition_id = partition.hypergraph_partitioned ? comm_p_row_size * i + comm_p_row_rank : comm_p_col_size * comm_p_row_rank + i; - auto row_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto row_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; auto p_offsets = partition.hypergraph_partitioned ? adj_matrix_partition_offsets[i].data() : adj_matrix_partition_offsets[0].data() + - (row_first - partition.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); + (major_first - partition.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); thrust::transform(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), thrust::make_counting_iterator(vertex_t{0}), - thrust::make_counting_iterator(row_last - row_first), + thrust::make_counting_iterator(major_last - major_first), local_degrees.data(), [p_offsets] __device__(auto i) { return p_offsets[i + 1] - p_offsets[i]; }); comm_p_row.reduce(local_degrees.data(), diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index 95edb8e7201..e1f075f2c76 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -29,10 +29,6 @@ namespace cugraph { namespace experimental { -// FIXME: these should better be defined somewhere else. -std::string const comm_p_row_key = "comm_p_row"; -std::string const comm_p_col_key = "comm_p_key"; - template struct edgelist_t { vertex_t const *p_src_vertices{nullptr}; diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index f065662f401..60b87243377 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -28,6 +28,10 @@ namespace cugraph { namespace experimental { +// FIXME: these should better be defined somewhere else. +std::string const comm_p_row_key = "comm_p_row"; +std::string const comm_p_col_key = "comm_p_key"; + /** * @brief store vertex partitioning map * diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index e50aad5b50c..ada9308fa86 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -253,7 +253,7 @@ graph_tget_handle_ptr()), adj_matrix_partition_offsets_, partition_); static_assert(num_segments_per_vertex_partition == 3); From 47ce146296c69c10b171c62fc02bd11d51fb883b Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 21 Aug 2020 10:46:37 -0400 Subject: [PATCH 09/77] add a FIXME comment --- cpp/src/experimental/graph.cu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index ada9308fa86..9047b445bc0 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -152,6 +152,8 @@ std:: }); } + // FIXME: need to add an option to sort neighbor lists + return std::make_tuple(std::move(offsets), std::move(indices), std::move(weights)); } From a07782757ba04c0b665d1c48f426056a7fddd626 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Mon, 24 Aug 2020 12:58:48 -0400 Subject: [PATCH 10/77] bug fixes --- cpp/src/experimental/graph.cu | 15 +++--- cpp/tests/experimental/graph_test.cpp | 69 ++++++++++++++------------- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 9047b445bc0..650e42da8dc 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -94,9 +94,9 @@ std:: auto major_first = store_transposed ? row_first : col_first; thrust::for_each(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), - store_transposed ? edgelist.p_src_vertices : edgelist.p_dst_vertices, - store_transposed ? edgelist.p_src_vertices + edgelist.number_of_edges - : edgelist.p_dst_vertices + edgelist.number_of_edges, + store_transposed ? edgelist.p_dst_vertices : edgelist.p_src_vertices, + store_transposed ? edgelist.p_dst_vertices + edgelist.number_of_edges + : edgelist.p_src_vertices + edgelist.number_of_edges, [p_offsets, major_first] __device__(auto v) { atomicAdd(p_offsets + (v - major_first), edge_t{1}); }); @@ -116,8 +116,8 @@ std:: auto s = thrust::get<0>(e); auto d = thrust::get<1>(e); auto w = thrust::get<2>(e); - auto major = store_transposed ? s : d; - auto minor = store_transposed ? d : s; + auto major = store_transposed ? d : s; + auto minor = store_transposed ? s : d; auto start = p_offsets[major - major_first]; auto degree = p_offsets[(major - major_first) + 1] - start; auto idx = atomicAdd(p_indices + (start + degree - 1), @@ -138,8 +138,8 @@ std:: [p_offsets, p_indices, p_weights, major_first] __device__(auto e) { auto s = thrust::get<0>(e); auto d = thrust::get<1>(e); - auto major = store_transposed ? s : d; - auto minor = store_transposed ? d : s; + auto major = store_transposed ? d : s; + auto minor = store_transposed ? s : d; auto start = p_offsets[major - major_first]; auto degree = p_offsets[(major - major_first) + 1] - start; auto idx = atomicAdd(p_indices + (start + degree - 1), @@ -366,6 +366,7 @@ graph_t, std::vector, std::vector> gr std::vector weights(p_edge_weights != nullptr ? number_of_edges : 0, weight_t{0.0}); for (size_t i = 0; i < number_of_edges; ++i) { - if (store_transposed) { - auto d = p_dst_vertices[i]; - offsets[1 + d]++; - } else { - auto s = p_src_vertices[i]; - offsets[1 + s]++; - } + auto major = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; + offsets[1 + major]++; } std::partial_sum(offsets.begin() + 1, offsets.end(), offsets.begin() + 1); for (size_t i = 0; i < number_of_edges; ++i) { - auto major = store_transposed ? p_src_vertices[i] : p_dst_vertices[i]; - auto minor = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; + auto major = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; + auto minor = store_transposed ? p_src_vertices[i] : p_dst_vertices[i]; auto start = offsets[major]; auto degree = offsets[major + 1] - start; auto idx = indices[start + degree - 1]++; - indices[idx] = minor; - if (p_edge_weights != nullptr) { weights[idx] = p_edge_weights[i]; } + indices[start + idx] = minor; + if (p_edge_weights != nullptr) { weights[start + idx] = p_edge_weights[i]; } } return std::make_tuple(std::move(offsets), std::move(indices), std::move(weights)); } typedef struct Graph_Usecase_t { - std::string graph_file_path_; - std::string graph_file_full_path_; - bool test_weighted_; + std::string graph_file_path; + std::string graph_file_full_path; + bool test_weighted; Graph_Usecase_t(std::string const& graph_file_path, bool test_weighted) - : graph_file_path_(graph_file_path), test_weighted_(test_weighted) + : graph_file_path(graph_file_path), test_weighted(test_weighted) { - if ((graph_file_path_.length() > 0) && (graph_file_path[0] != '/')) { - graph_file_full_path_ = cugraph::test::get_rapids_dataset_root_dir() + "/" + graph_file_path_; + if ((graph_file_path.length() > 0) && (graph_file_path[0] != '/')) { + graph_file_full_path = cugraph::test::get_rapids_dataset_root_dir() + "/" + graph_file_path; } else { - graph_file_full_path_ = graph_file_path_; + graph_file_full_path = graph_file_path; } }; } Graph_Usecase; @@ -102,8 +97,8 @@ class Tests_Graph : public ::testing::TestWithParam { vertex_t k{}; edge_t nnz{}; - FILE* file = fopen(configuration.graph_file_full_path_.c_str(), "r"); - ASSERT_NE(file, nullptr) << "fopen (" << configuration.graph_file_full_path_ << ") failure."; + FILE* file = fopen(configuration.graph_file_full_path.c_str(), "r"); + ASSERT_NE(file, nullptr) << "fopen (" << configuration.graph_file_full_path << ") failure."; ASSERT_EQ(cugraph::test::mm_properties(file, 1, &mc, &m, &k, &nnz), 0) << "could not read Matrix Market file properties\n"; @@ -129,7 +124,7 @@ class Tests_Graph : public ::testing::TestWithParam { std::tie(h_reference_offsets, h_reference_indices, h_reference_weights) = graph_reference(h_rows.data(), h_cols.data(), - configuration.test_weighted_ ? h_weights.data() : nullptr, + configuration.test_weighted ? h_weights.data() : nullptr, m, nnz); @@ -137,12 +132,12 @@ class Tests_Graph : public ::testing::TestWithParam { rmm::device_uvector d_rows(nnz, handle.get_stream()); rmm::device_uvector d_cols(nnz, handle.get_stream()); - rmm::device_uvector d_weights(configuration.test_weighted_ ? nnz : 0, + rmm::device_uvector d_weights(configuration.test_weighted ? nnz : 0, handle.get_stream()); raft::update_device(d_rows.data(), h_rows.data(), h_rows.size(), handle.get_stream()); raft::update_device(d_cols.data(), h_cols.data(), h_cols.size(), handle.get_stream()); - if (configuration.test_weighted_) { + if (configuration.test_weighted) { raft::update_device( d_weights.data(), h_weights.data(), h_weights.size(), handle.get_stream()); } @@ -150,21 +145,25 @@ class Tests_Graph : public ::testing::TestWithParam { CUDA_TRY(cudaStreamSynchronize(handle.get_stream())); cugraph::experimental::edgelist_t edgelist{ - d_rows.data(), d_cols.data(), configuration.test_weighted_ ? d_weights.data() : nullptr, nnz}; + d_rows.data(), d_cols.data(), configuration.test_weighted ? d_weights.data() : nullptr, nnz}; CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement auto graph = cugraph::experimental::graph_t( - handle, edgelist, m, mm_is_symmetric(mc), false, configuration.test_weighted_, false, true); + handle, edgelist, m, mm_is_symmetric(mc), false, configuration.test_weighted, false, true); auto graph_view = graph.view(); + ASSERT_EQ(graph_view.get_number_of_vertices(), m); + ASSERT_EQ(graph_view.get_number_of_edges(), nnz); + CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement - std::vector h_cugraph_offsets{}; - std::vector h_cugraph_indices{}; - std::vector h_cugraph_weights{}; + std::vector h_cugraph_offsets(graph_view.get_number_of_vertices() + 1); + std::vector h_cugraph_indices(graph_view.get_number_of_edges()); + std::vector h_cugraph_weights( + configuration.test_weighted ? graph_view.get_number_of_edges() : 0); raft::update_host(h_cugraph_offsets.data(), graph_view.offsets(), @@ -174,10 +173,12 @@ class Tests_Graph : public ::testing::TestWithParam { graph_view.indices(), graph_view.get_number_of_edges(), handle.get_stream()); - raft::update_host(h_cugraph_weights.data(), - graph_view.weights(), - graph_view.get_number_of_edges(), - handle.get_stream()); + if (configuration.test_weighted) { + raft::update_host(h_cugraph_weights.data(), + graph_view.weights(), + graph_view.get_number_of_edges(), + handle.get_stream()); + } CUDA_TRY(cudaStreamSynchronize(handle.get_stream())); @@ -188,7 +189,7 @@ class Tests_Graph : public ::testing::TestWithParam { for (vertex_t i = 0; i < m; ++i) { auto start = h_reference_offsets[i]; auto degree = h_reference_offsets[i + 1] - start; - if (configuration.test_weighted_) { + if (configuration.test_weighted) { std::vector> reference_pairs(degree); std::vector> cugraph_pairs(degree); for (edge_t j = 0; j < degree; ++j) { @@ -242,4 +243,4 @@ INSTANTIATE_TEST_CASE_P(simple_test, Graph_Usecase("test/datasets/webbase-1M.mtx", false), Graph_Usecase("test/datasets/webbase-1M.mtx", true))); -CUGRAPH_TEST_PROGRAM_MAIN() \ No newline at end of file +CUGRAPH_TEST_PROGRAM_MAIN() From 24633cc095ada1561753078c01bd0c44f196fc25 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Mon, 24 Aug 2020 13:19:43 -0400 Subject: [PATCH 11/77] expand experimental graph explicit template instantiation --- cpp/src/experimental/graph.cu | 30 ++++++++++++++++++++++----- cpp/tests/experimental/graph_test.cpp | 24 ++++++++++++++++----- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 650e42da8dc..2b037e21f83 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -373,11 +373,31 @@ graph_t; -template class graph_t; - -template class graph_t; -template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; + +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; } // namespace experimental } // namespace cugraph \ No newline at end of file diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp index 089ff7a08a1..a2679d5003d 100644 --- a/cpp/tests/experimental/graph_test.cpp +++ b/cpp/tests/experimental/graph_test.cpp @@ -100,8 +100,12 @@ class Tests_Graph : public ::testing::TestWithParam { FILE* file = fopen(configuration.graph_file_full_path.c_str(), "r"); ASSERT_NE(file, nullptr) << "fopen (" << configuration.graph_file_full_path << ") failure."; - ASSERT_EQ(cugraph::test::mm_properties(file, 1, &mc, &m, &k, &nnz), 0) + edge_t tmp_m{}; + edge_t tmp_k{}; + ASSERT_EQ(cugraph::test::mm_properties(file, 1, &mc, &tmp_m, &tmp_k, &nnz), 0) << "could not read Matrix Market file properties\n"; + m = static_cast(tmp_m); + k = static_cast(tmp_k); ASSERT_TRUE(mm_is_matrix(mc)); ASSERT_TRUE(mm_is_coordinate(mc)); ASSERT_FALSE(mm_is_complex(mc)); @@ -221,15 +225,25 @@ class Tests_Graph : public ::testing::TestWithParam { }; // FIXME: add tests for type combinations -TEST_P(Tests_Graph, CheckInt32Int32FloatFalse) +TEST_P(Tests_Graph, CheckStoreTransposedFalse) { - run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); } // FIXME: add tests for type combinations -TEST_P(Tests_Graph, CheckInt32Int32FloatTrue) +TEST_P(Tests_Graph, CheckStoreTransposedTrue) { - run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); } INSTANTIATE_TEST_CASE_P(simple_test, From 905eddfcf722e07908c9c9e60a405f1586e70a1e Mon Sep 17 00:00:00 2001 From: Keith Kraus Date: Wed, 26 Aug 2020 20:15:15 -0400 Subject: [PATCH 12/77] thrust patch --- CHANGELOG.md | 3 +++ cpp/CMakeLists.txt | 1 + cpp/cmake/thrust-ret-if-fail.patch | 16 ++++++++++++++++ 3 files changed, 20 insertions(+) create mode 100644 cpp/cmake/thrust-ret-if-fail.patch diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1f9d037ab..187739e3cec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,8 @@ - PR #1087 Updated benchmarks README to better describe how to get plugin, added rapids-pytest-benchmark plugin to conda dev environments - PR #1101 Removed unnecessary device-to-host copy which caused a performance regression - PR #1106 Added new release.ipynb to notebook test skip list +- PR #1125 Patch Thrust to workaround `CUDA_CUB_RET_IF_FAIL` macro clearing CUDA errors + # cuGraph 0.14.0 (03 Jun 2020) @@ -168,6 +170,7 @@ - PR 935 Merge - PR #956 Use new gpuCI image in local build script + # cuGraph 0.13.0 (31 Mar 2020) ## New Features diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 70d7edf99a3..8812d71fd83 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -214,6 +214,7 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/thrust/thrust.git GIT_TAG 1.9.10 GIT_SHALLOW true + PATCH_COMMAND COMMAND patch -p1 < "${CMAKE_CURRENT_SOURCE_DIR}/cmake/thrust-ret-if-fail.patch" ) FetchContent_GetProperties(thrust) diff --git a/cpp/cmake/thrust-ret-if-fail.patch b/cpp/cmake/thrust-ret-if-fail.patch new file mode 100644 index 00000000000..990b3f993be --- /dev/null +++ b/cpp/cmake/thrust-ret-if-fail.patch @@ -0,0 +1,16 @@ +diff --git a/thrust/system/cuda/detail/core/util.h b/thrust/system/cuda/detail/core/util.h +index a2c87772..ea4ed640 100644 +--- a/thrust/system/cuda/detail/core/util.h ++++ b/thrust/system/cuda/detail/core/util.h +@@ -652,7 +652,10 @@ namespace core { + } + + #define CUDA_CUB_RET_IF_FAIL(e) \ +- if (cub::Debug((e), __FILE__, __LINE__)) return e; ++ { \ ++ auto const error = (e); \ ++ if (cub::Debug(error, __FILE__, __LINE__)) return error; \ ++ } + + // uninitialized + // ------- From 2107d53b1ae41ba9a0e75ddab47f63a43ce665f2 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Thu, 27 Aug 2020 14:18:07 -0400 Subject: [PATCH 13/77] update compute_major_degree to work with both a vector of rmm::device_uvector and pointers --- .../experimental/detail/graph_utils.cuh | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index 0daa8f89d5c..ca0a326f262 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -29,11 +29,12 @@ namespace cugraph { namespace experimental { namespace detail { -// compute the numbers of nonzeros in rows (of the graph adjacency matrix, if store_transposed = false) or columns (of the graph adjacency matrix, if store_transposed = true) +// compute the numbers of nonzeros in rows (of the graph adjacency matrix, if store_transposed = +// false) or columns (of the graph adjacency matrix, if store_transposed = true) template rmm::device_uvector compute_major_degree( raft::handle_t const &handle, - std::vector> const &adj_matrix_partition_offsets, + std::vector const &adj_matrix_partition_offsets, partition_t const &partition) { auto &comm_p_row = handle.get_subcomm(comm_p_row_key); @@ -51,9 +52,9 @@ rmm::device_uvector compute_major_degree( auto vertex_partition_id = partition.hypergraph_partitioned ? comm_p_row_size * i + comm_p_row_rank : comm_p_col_size * comm_p_row_rank + i; - auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; - max_num_local_degrees = std::max(max_num_local_degrees, major_last - major_first); + auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + max_num_local_degrees = std::max(max_num_local_degrees, major_last - major_first); if (i == comm_p_col_rank) { degrees.resize(major_last - major_first, handle.get_stream()); } } local_degrees.resize(max_num_local_degrees, handle.get_stream()); @@ -61,12 +62,12 @@ rmm::device_uvector compute_major_degree( auto vertex_partition_id = partition.hypergraph_partitioned ? comm_p_row_size * i + comm_p_row_rank : comm_p_col_size * comm_p_row_rank + i; - auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; auto p_offsets = partition.hypergraph_partitioned - ? adj_matrix_partition_offsets[i].data() - : adj_matrix_partition_offsets[0].data() + + ? adj_matrix_partition_offsets[i] + : adj_matrix_partition_offsets[0] + (major_first - partition.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); thrust::transform(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), thrust::make_counting_iterator(vertex_t{0}), @@ -84,6 +85,22 @@ rmm::device_uvector compute_major_degree( return degrees; } +// compute the numbers of nonzeros in rows (of the graph adjacency matrix, if store_transposed = +// false) or columns (of the graph adjacency matrix, if store_transposed = true) +template +rmm::device_uvector compute_major_degree( + raft::handle_t const &handle, + std::vector> const &adj_matrix_partition_offsets, + partition_t const &partition) +{ + // we can avoid creating this temporary with "if constexpr" supported from C++17 + std::vector tmp_offsets(adj_matrix_partition_offsets.size(), nullptr); + for (size_t i = 0; i < adj_matrix_partition_offsets.size(); ++i) { + tmp_offsets[i] = adj_matrix_partition_offsets[i].data(); + } + return compute_major_degree(handle, tmp_offsets, partition); +} + } // namespace detail } // namespace experimental } // namespace cugraph From 7d3e609b54d565f126abf7d767877bf12a649831 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 09:40:12 -0400 Subject: [PATCH 14/77] Centralizing test data to a few common sets --- .../test_mg_batch_betweenness_centrality.py | 2 +- ...st_mg_batch_edge_betweenness_centrality.py | 2 +- .../cugraph/tests/dask/test_mg_replication.py | 2 +- python/cugraph/tests/test_connectivity.py | 26 ++++++++++++++++--- python/cugraph/tests/test_graph.py | 15 ++++++----- python/cugraph/tests/test_leiden.py | 2 +- python/cugraph/tests/test_sssp.py | 6 ++--- .../cugraph/tests/test_subgraph_extraction.py | 4 +-- python/cugraph/tests/utils.py | 26 +++++++++---------- 9 files changed, 52 insertions(+), 33 deletions(-) diff --git a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py index 92782e7a0b3..0a0c372d9c5 100644 --- a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py @@ -24,7 +24,7 @@ # ============================================================================= # Parameters # ============================================================================= -DATASETS = utils.DATASETS_1 +DATASETS = utils.DATASETS MG_DEVICE_COUNT_OPTIONS = [1, 2, 3, 4] RESULT_DTYPE_OPTIONS = [np.float64] diff --git a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py index d4906ca04ef..31ca62bd5c6 100644 --- a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py @@ -24,7 +24,7 @@ # ============================================================================= # Parameters # ============================================================================= -DATASETS = utils.DATASETS_1 +DATASETS = utils.DATASETS MG_DEVICE_COUNT_OPTIONS = [1, 2, 3, 4] RESULT_DTYPE_OPTIONS = [np.float64] diff --git a/python/cugraph/tests/dask/test_mg_replication.py b/python/cugraph/tests/dask/test_mg_replication.py index b74e6e9a15f..5248998d801 100644 --- a/python/cugraph/tests/dask/test_mg_replication.py +++ b/python/cugraph/tests/dask/test_mg_replication.py @@ -19,7 +19,7 @@ import cugraph.tests.utils as utils import pytest -DATASETS_OPTIONS = utils.DATASETS_1 +DATASETS_OPTIONS = utils.DATASETS DIRECTED_GRAPH_OPTIONS = [False, True] MG_DEVICE_COUNT_OPTIONS = [1, 2, 3, 4] diff --git a/python/cugraph/tests/test_connectivity.py b/python/cugraph/tests/test_connectivity.py index b33b6f8e9a3..84057e54499 100644 --- a/python/cugraph/tests/test_connectivity.py +++ b/python/cugraph/tests/test_connectivity.py @@ -142,7 +142,17 @@ def test_weak_cc(graph_file): # Compare vertices of largest component nx_vertices = sorted(lst_nx_components[0]) - cg_vertices = sorted(lst_cg_components[0]) + first_vert = nx_vertices[0] + + idx = -1 + for i in range(len(lst_cg_components)): + if first_vert in lst_cg_components[i]: + idx = i + + assert idx != -1 + + cg_vertices = sorted(lst_cg_components[idx]) + assert nx_vertices == cg_vertices @@ -167,7 +177,7 @@ def test_strong_cc(graph_file): nx_n_components = len(netx_labels) cg_n_components = len(cugraph_labels) - # Comapre number of components + # Comapre number of components found assert nx_n_components == cg_n_components lst_nx_components = sorted(netx_labels, key=len, reverse=True) @@ -180,7 +190,15 @@ def test_strong_cc(graph_file): # Compare lengths of each component assert lst_nx_components_lens == lst_cg_components_lens - # Compare vertices of largest component + # Compare vertices of largest component - note that there might be more than one largest component nx_vertices = sorted(lst_nx_components[0]) - cg_vertices = sorted(lst_cg_components[0]) + first_vert = nx_vertices[0] + + idx = -1 + for i in range(len(lst_cg_components)): + if first_vert in lst_cg_components[i]: + idx = i + + assert idx != -1 + cg_vertices = sorted(lst_cg_components[idx]) assert nx_vertices == cg_vertices diff --git a/python/cugraph/tests/test_graph.py b/python/cugraph/tests/test_graph.py index 15aa96ec287..7a87d6202b0 100644 --- a/python/cugraph/tests/test_graph.py +++ b/python/cugraph/tests/test_graph.py @@ -479,7 +479,7 @@ def test_consolidation(graph_file): # Test -@pytest.mark.parametrize('graph_file', utils.DATASETS_2) +@pytest.mark.parametrize('graph_file', utils.TINY_DATASETS) def test_two_hop_neighbors(graph_file): gc.collect() @@ -593,7 +593,7 @@ def test_number_of_vertices(graph_file): # Test -@pytest.mark.parametrize("graph_file", utils.DATASETS_2) +@pytest.mark.parametrize("graph_file", utils.DATASETS) def test_to_directed(graph_file): gc.collect() @@ -622,12 +622,14 @@ def test_to_directed(graph_file): # Test -@pytest.mark.parametrize("graph_file", utils.DATASETS_2) +@pytest.mark.parametrize("graph_file", utils.TINY_DATASETS) def test_to_undirected(graph_file): gc.collect() + # Read data and then convert to directed by dropped some edges cu_M = utils.read_csv_file(graph_file) cu_M = cu_M[cu_M["0"] <= cu_M["1"]].reset_index(drop=True) + M = utils.read_csv_for_nx(graph_file) M = M[M["0"] <= M["1"]] assert len(cu_M) == len(M) @@ -635,6 +637,7 @@ def test_to_undirected(graph_file): # cugraph add_edge_list DiG = cugraph.DiGraph() DiG.from_cudf_edgelist(cu_M, source="0", destination="1") + DiGnx = nx.from_pandas_edgelist( M, source="0", target="1", create_using=nx.DiGraph() ) @@ -655,7 +658,7 @@ def test_to_undirected(graph_file): # Test -@pytest.mark.parametrize("graph_file", utils.DATASETS_2) +@pytest.mark.parametrize("graph_file", utils.DATASETS) def test_has_edge(graph_file): gc.collect() @@ -672,7 +675,7 @@ def test_has_edge(graph_file): # Test -@pytest.mark.parametrize("graph_file", utils.DATASETS_2) +@pytest.mark.parametrize("graph_file", utils.DATASETS) def test_has_node(graph_file): gc.collect() @@ -688,7 +691,7 @@ def test_has_node(graph_file): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize('graph_file', utils.DATASETS_2) +@pytest.mark.parametrize('graph_file', utils.DATASETS) def test_bipartite_api(graph_file): # This test only tests the functionality of adding set of nodes and # retrieving them. The datasets currently used are not truly bipartite. diff --git a/python/cugraph/tests/test_leiden.py b/python/cugraph/tests/test_leiden.py index a2b7424484d..7f7b4b577fe 100644 --- a/python/cugraph/tests/test_leiden.py +++ b/python/cugraph/tests/test_leiden.py @@ -62,7 +62,7 @@ def cugraph_louvain(cu_M, edgevals=False): return parts, mod -@pytest.mark.parametrize("graph_file", utils.DATASETS_3) +@pytest.mark.parametrize("graph_file", utils.DATASETS) def test_leiden(graph_file): gc.collect() diff --git a/python/cugraph/tests/test_sssp.py b/python/cugraph/tests/test_sssp.py index 5c3d6293dcd..3c3b575fdb5 100644 --- a/python/cugraph/tests/test_sssp.py +++ b/python/cugraph/tests/test_sssp.py @@ -94,7 +94,7 @@ def networkx_call(M, source, edgevals=False): # Test -@pytest.mark.parametrize("graph_file", utils.DATASETS_4) +@pytest.mark.parametrize("graph_file", utils.DATASETS) @pytest.mark.parametrize("source", SOURCES) def test_sssp(graph_file, source): print("DOING test_sssp : " + graph_file + "\n\n\n") @@ -127,7 +127,7 @@ def test_sssp(graph_file, source): # Test -@pytest.mark.parametrize("graph_file", utils.DATASETS_1) +@pytest.mark.parametrize("graph_file", utils.DATASETS) @pytest.mark.parametrize("source", SOURCES) def test_sssp_edgevals(graph_file, source): gc.collect() @@ -159,7 +159,7 @@ def test_sssp_edgevals(graph_file, source): assert err == 0 -@pytest.mark.parametrize("graph_file", utils.DATASETS_1) +@pytest.mark.parametrize("graph_file", utils.DATASETS) @pytest.mark.parametrize("source", SOURCES) def test_sssp_data_type_conversion(graph_file, source): gc.collect() diff --git a/python/cugraph/tests/test_subgraph_extraction.py b/python/cugraph/tests/test_subgraph_extraction.py index 9d9631c0f6d..9192495c6b2 100644 --- a/python/cugraph/tests/test_subgraph_extraction.py +++ b/python/cugraph/tests/test_subgraph_extraction.py @@ -71,7 +71,7 @@ def nx_call(M, verts, directed=True): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize("graph_file", utils.DATASETS_4) +@pytest.mark.parametrize("graph_file", utils.DATASETS) def test_subgraph_extraction_DiGraph(graph_file): gc.collect() @@ -88,7 +88,7 @@ def test_subgraph_extraction_DiGraph(graph_file): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize("graph_file", utils.DATASETS_4) +@pytest.mark.parametrize("graph_file", utils.DATASETS) def test_subgraph_extraction_Graph(graph_file): gc.collect() diff --git a/python/cugraph/tests/utils.py b/python/cugraph/tests/utils.py index fa0631f9c30..e00be028351 100644 --- a/python/cugraph/tests/utils.py +++ b/python/cugraph/tests/utils.py @@ -22,21 +22,23 @@ # # Datasets are numbered based on the number of elements in the array # -DATASETS_1 = ['../datasets/netscience.csv'] +# DATASETS_1 = ['../datasets/netscience.csv'] -DATASETS_2 = ['../datasets/karate.csv', - '../datasets/dolphins.csv'] +DATASETS_UNDIRECTED = ['../datasets/karate.csv', '../datasets/dolphins.csv'] -DATASETS_3 = ['../datasets/karate.csv', +DATASETS = ['../datasets/karate.csv', + '../datasets/karate-disjoint.csv', '../datasets/dolphins.csv', + '../datasets/netscience.csv', '../datasets/email-Eu-core.csv'] # FIXME: netscience.csv causes NetworkX pagerank to throw an exception. # (networkx/algorithms/link_analysis/pagerank_alg.py:152: KeyError: 1532) -DATASETS_4 = ['../datasets/karate.csv', - '../datasets/dolphins.csv', - '../datasets/netscience.csv', - '../datasets/email-Eu-core.csv'] +# DATASETS_4 = ['../datasets/karate.csv', +# '../datasets/karate-disjoint.csv', +# '../datasets/dolphins.csv', +# '../datasets/netscience.csv', +# '../datasets/email-Eu-core.csv'] DATASETS_5 = ['../datasets/karate.csv', '../datasets/dolphins.csv', @@ -57,15 +59,11 @@ '../datasets/dolphins.csv', '../datasets/polbooks.csv'] -SMALL_DATASETS = ['../datasets/netscience.csv', - '../datasets/email-Eu-core.csv'] +# SMALL_DATASETS = ['../datasets/netscience.csv', '../datasets/email-Eu-core.csv'] -UNRENUMBERED_DATASETS = ['../datasets/karate.csv'] +UNRENUMBERED_DATASETS = ['../datasets/karate-disjoint.csv'] -# define the base for tests to use -DATASETS = DATASETS_3 -DATASETS_UNDIRECTED = DATASETS_2 def read_csv_for_nx(csv_file, read_weights_in_sp=True): From 386acadcfdedcf5cb38f2cc421ab956e7eaf2048 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 09:40:34 -0400 Subject: [PATCH 15/77] reference doc --- docs/source/cugraph_blogs.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docs/source/cugraph_blogs.rst diff --git a/docs/source/cugraph_blogs.rst b/docs/source/cugraph_blogs.rst new file mode 100644 index 00000000000..3cc37a302cc --- /dev/null +++ b/docs/source/cugraph_blogs.rst @@ -0,0 +1,32 @@ +cuGraph blogs and other references +=============================== + +The RAPIDS team blogs at https://medium.com/rapids-ai, and many of +these blog posts provide deeper dives into models or key features from +cuGraph. Here, we've selected just a few that are of particular interest +to cuML users: + +BLOGS +------------------------------------------------ + +2019 +* `RAPIDS cuGraph `_ March +* `RAPIDS cuGraph — The vision and journey to version 1.0 and beyond `_ August + +2020 + + + + + +Academic Papers +--------------- + +* `Machine Learning in Python: Main developments and technology trends in data science, machine learning, and artificial intelligence (Sebastian Raschka, Joshua Patterson, Corey Nolet) `_ + + + +References +--------------- + + From 4900b4944ea2501f4367aceb645e344c86a8feb9 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 09:43:49 -0400 Subject: [PATCH 16/77] update --- docs/source/cugraph_blogs.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/cugraph_blogs.rst b/docs/source/cugraph_blogs.rst index 3cc37a302cc..b4150c459fe 100644 --- a/docs/source/cugraph_blogs.rst +++ b/docs/source/cugraph_blogs.rst @@ -10,8 +10,8 @@ BLOGS ------------------------------------------------ 2019 -* `RAPIDS cuGraph `_ March -* `RAPIDS cuGraph — The vision and journey to version 1.0 and beyond `_ August +* March `RAPIDS cuGraph `_ +* August `RAPIDS cuGraph — The vision and journey to version 1.0 and beyond `_ 2020 From e62978e2fceb63803ff0554627aa4c481d9a2aa7 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 28 Aug 2020 11:14:28 -0400 Subject: [PATCH 17/77] add graph_view.cu for error checks using thrust --- cpp/CMakeLists.txt | 1 + cpp/include/experimental/graph_view.hpp | 36 +-- cpp/src/experimental/graph_view.cu | 324 ++++++++++++++++++++++++ 3 files changed, 336 insertions(+), 25 deletions(-) create mode 100644 cpp/src/experimental/graph_view.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 37aa55f04b0..5776e60f736 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -357,6 +357,7 @@ add_library(cugraph SHARED src/centrality/katz_centrality.cu src/centrality/betweenness_centrality.cu src/experimental/graph.cu + src/experimental/graph_view.cu ) # diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index 60b87243377..1463d73b44d 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -80,6 +80,11 @@ struct partition_t { namespace detail { +// FIXME: threshold values require tuning +size_t constexpr low_degree_threshold{raft::warp_size()}; +size_t constexpr mid_degree_threshold{1024}; +size_t constexpr num_segments_per_vertex_partition{3}; + struct graph_properties_t { bool is_symmetric{false}; bool is_multigraph{false}; @@ -162,17 +167,7 @@ class graph_view_t( - handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), - adj_matrix_partition_offsets_(adj_matrix_partition_offsets), - adj_matrix_partition_indices_(adj_matrix_partition_indices), - adj_matrix_partition_weights_(adj_matrix_partition_weights), - partition_(partition), - vertex_partition_segment_offsets_(vertex_partition_segment_offsets) - { - // FIXME: error check - } + bool do_expensive_check = false); vertex_t get_number_of_local_vertices() const { @@ -188,7 +183,7 @@ class graph_view_t( - handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), - offsets_(offsets), - indices_(indices), - weights_(weights), - segment_offsets_(segment_offsets) - { - // FIXME: error check - } + bool sorted_by_degree, + bool do_expensive_check = false); vertex_t get_number_of_local_vertices() const { return this->get_number_of_vertices(); } diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu new file mode 100644 index 00000000000..b1b07413baa --- /dev/null +++ b/cpp/src/experimental/graph_view.cu @@ -0,0 +1,324 @@ +/* + * Copyright (c) 2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace cugraph { +namespace experimental { + +namespace { + +// can't use lambda due to nvcc limitations (The enclosing parent function ("graph_view_t") for an +// extended __device__ lambda must allow its address to be taken) +template +struct out_of_range_t { + vertex_t min{}; + vertex_t max{}; + + __device__ bool operator()(vertex_t v) { return (v < min) || (v >= max); } +}; + +// can't use lambda due to nvcc limitations (The enclosing parent function ("graph_view_t") for an +// extended __device__ lambda must allow its address to be taken) +template +struct degree_from_offsets_t { + edge_t const* offsets{nullptr}; + + __device__ edge_t operator()(vertex_t v) { return offsets[v + 1] - offsets[v]; } +}; + +} // namespace + +template +graph_view_t>:: + graph_view_t(raft::handle_t const& handle, + std::vector const& adj_matrix_partition_offsets, + std::vector const& adj_matrix_partition_indices, + std::vector const& adj_matrix_partition_weights, + std::vector const& vertex_partition_segment_offsets, + partition_t const& partition, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool sorted_by_global_degree_within_vertex_partition, + bool do_expensive_check) + : detail::graph_base_t( + handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + adj_matrix_partition_offsets_(adj_matrix_partition_offsets), + adj_matrix_partition_indices_(adj_matrix_partition_indices), + adj_matrix_partition_weights_(adj_matrix_partition_weights), + partition_(partition), + vertex_partition_segment_offsets_(vertex_partition_segment_offsets) +{ + // cheap error checks + + auto comm_p_size = this->get_handle_ptr()->get_comms().get_size(); + auto comm_p_row_size = this->get_handle_ptr()->get_subcomm(comm_p_row_key).get_size(); + auto comm_p_col_size = this->get_handle_ptr()->get_subcomm(comm_p_col_key).get_size(); + + CUGRAPH_EXPECTS(adj_matrix_partition_offsets.size() == adj_matrix_partition_indices.size(), + "Invalid API parameter: adj_matrix_partition_offsets.size() and " + "adj_matrix_partition_indices.size() should coincide."); + CUGRAPH_EXPECTS( + (is_weighted && (adj_matrix_partition_weights.size() == adj_matrix_partition_offsets.size())) || + (!is_weighted && (adj_matrix_partition_weights.size() == 0)), + "Invalid API parameter: adj_matrix_partition_weights.size() should coincide with " + "adj_matrix_partition_offsets.size() (if is_weighted is true) or 0 (if is_weighted is false)."); + + CUGRAPH_EXPECTS( + (partition.hypergraph_partitioned && + (adj_matrix_partition_offsets.size() == static_cast(comm_p_row_size))) || + (!(partition.hypergraph_partitioned) && (adj_matrix_partition_offsets.size() == 1)), + "Invalid API parameter: errneous adj_matrix_partition_offsets.size()."); + + CUGRAPH_EXPECTS((sorted_by_global_degree_within_vertex_partition && + (vertex_partition_segment_offsets.size() == + comm_p_col_size * (detail::num_segments_per_vertex_partition + 1))) || + (!sorted_by_global_degree_within_vertex_partition && + (vertex_partition_segment_offsets.size() == 0)), + "Invalid API parameter: vertex_partition_segment_offsets.size() does not match " + "with sorted_by_global_degree_within_vertex_partition."); + + CUGRAPH_EXPECTS(partition.vertex_partition_offsets.size() == static_cast(comm_p_size), + "Invalid API parameter: erroneous partition.vertex_partition_offsets.size()."); + + // optinoal expensive checks + + if (do_expensive_check) { + auto default_stream = this->get_handle_ptr()->get_stream(); + + auto comm_p_row_rank = this->get_handle_ptr()->get_subcomm(comm_p_row_key).get_rank(); + auto comm_p_col_rank = this->get_handle_ptr()->get_subcomm(comm_p_col_key).get_rank(); + + for (size_t i = 0; i < adj_matrix_partition_offsets.size(); ++i) { + auto major_first = + partition.hypergraph_partitioned + ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank] + : partition.vertex_partition_offsets[comm_p_row_rank * comm_p_col_size]; + auto major_last = + partition.hypergraph_partitioned + ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank + 1] + : partition.vertex_partition_offsets[(comm_p_row_rank + 1) * comm_p_col_size]; + auto minor_first = partition.vertex_partition_offsets[comm_p_col_rank * comm_p_row_size]; + auto minor_last = partition.vertex_partition_offsets[(comm_p_col_rank + 1) * comm_p_row_size]; + CUGRAPH_EXPECTS( + thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), + adj_matrix_partition_offsets[i], + adj_matrix_partition_offsets[i] + (major_last - major_first + 1)), + "Invalid API parameter: adj_matrix_partition_offsets[] is not sorted."); + edge_t number_of_local_edges{}; + raft::update_host(&number_of_local_edges, + adj_matrix_partition_offsets[i] + (major_last - major_first), + 1, + default_stream); + + // better use thrust::any_of once https://github.com/thrust/thrust/issues/1016 is resolved + CUGRAPH_EXPECTS( + thrust::count_if(rmm::exec_policy(default_stream)->on(default_stream), + adj_matrix_partition_indices[i], + adj_matrix_partition_indices[i] + number_of_local_edges, + out_of_range_t{minor_first, minor_last}) == 0, + "Invalid API parameter: adj_matrix_partition_indices[] have out-of-range vertex IDs."); + + edge_t number_of_local_edges_sum{}; + this->get_handle_ptr()->get_comms().allreduce(&number_of_local_edges, + &number_of_local_edges_sum, + 1, + raft::comms::op_t::SUM, + default_stream); + CUGRAPH_EXPECTS(number_of_local_edges_sum == number_of_edges, + "Invalid API parameter: the sum of local edges doe counts not match with " + "number_of_local_edges."); + } + + if (sorted_by_global_degree_within_vertex_partition) { + auto degrees = detail::compute_major_degree(handle, adj_matrix_partition_offsets, partition); + CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), + degrees.begin(), + degrees.end(), + thrust::greater{}), + "Invalid API parameter: sorted_by_global_degree_within_vertex_partition is " + "set to true, but degrees are not non-ascending."); + + for (int i = 0; i < comm_p_col_size; ++i) { + CUGRAPH_EXPECTS(std::is_sorted(vertex_partition_segment_offsets.begin() + + (detail::num_segments_per_vertex_partition + 1) * i, + vertex_partition_segment_offsets.begin() + + (detail::num_segments_per_vertex_partition + 1) * (i + 1)), + "Invalid API parameter: erroneous vertex_partition_segment_offsets."); + CUGRAPH_EXPECTS( + vertex_partition_segment_offsets[(detail::num_segments_per_vertex_partition + 1) * i] == + 0, + "Invalid API parameter: erroneous vertex_partition_segment_offsets."); + auto vertex_partition_idx = partition.hypergraph_partitioned + ? comm_p_row_size * i + comm_p_row_rank + : comm_p_col_size * comm_p_row_rank + i; + CUGRAPH_EXPECTS( + vertex_partition_segment_offsets[(detail::num_segments_per_vertex_partition + 1) * i + + detail::num_segments_per_vertex_partition] == + partition.vertex_partition_offsets[vertex_partition_idx], + "Invalid API parameter: erroneous vertex_partition_segment_offsets."); + } + } + + CUGRAPH_EXPECTS( + std::is_sorted(partition.vertex_partition_offsets.begin(), + partition.vertex_partition_offsets.end()), + "Invalid API parameter: partition.vertex_partition_offsets values should be non-descending."); + CUGRAPH_EXPECTS(partition.vertex_partition_offsets[0] == edge_t{0}, + "Invalid API parameter: partition.vertex_partition_offsets[0] should be 0."); + CUGRAPH_EXPECTS(partition.vertex_partition_offsets.back() == number_of_vertices, + "Invalid API parameter: partition.vertex_partition_offsets.back() should be " + "number_of_vertices."); + + if (is_symmetric) {} + if (!is_multigraph) {} + } +} + +template +graph_view_t>::graph_view_t(raft::handle_t const& handle, + edge_t const* offsets, + vertex_t const* indices, + weight_t const* weights, + std::vector const& + segment_offsets, + vertex_t number_of_vertices, + edge_t number_of_edges, + bool is_symmetric, + bool is_multigraph, + bool is_weighted, + bool sorted_by_degree, + bool do_expensive_check) + : detail::graph_base_t( + handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + offsets_(offsets), + indices_(indices), + weights_(weights), + segment_offsets_(segment_offsets) +{ + // cheap error checks + + CUGRAPH_EXPECTS((is_weighted && (weights != nullptr)) || (!is_weighted && (weights == nullptr)), + "Invalid API parameter: weights shouldn't be nullptr if is_weighted is true and " + "should be nullptr if is_weighted is false."); + + CUGRAPH_EXPECTS( + (sorted_by_degree && + (segment_offsets.size() == (detail::num_segments_per_vertex_partition + 1))) || + (!sorted_by_degree && (segment_offsets.size() == 0)), + "Invalid API parameter: segment_offsets.size() does not match with sorted_by_degree."); + + // optinoal expensive checks + + if (do_expensive_check) { + auto default_stream = this->get_handle_ptr()->get_stream(); + + CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), + offsets, + offsets + (this->get_number_of_vertices() + 1)), + "Invalid API parameter: offsets is not sorted."); + + // better use thrust::any_of once https://github.com/thrust/thrust/issues/1016 is resolved + CUGRAPH_EXPECTS( + thrust::count_if(rmm::exec_policy(default_stream)->on(default_stream), + indices, + indices + this->get_number_of_edges(), + out_of_range_t{0, this->get_number_of_vertices()}) == 0, + "Invalid API parameter: adj_matrix_partition_indices[] have out-of-range vertex IDs."); + + if (sorted_by_degree) { + auto degree_first = thrust::make_transform_iterator( + thrust::make_counting_iterator(vertex_t{0}), + degree_from_offsets_t{offsets}); + CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), + degree_first, + degree_first + this->get_number_of_vertices(), + thrust::greater{}), + "Invalid API parameter: sorted_by_degree is set to true, but degrees are not " + "non-ascending."); + + CUGRAPH_EXPECTS(std::is_sorted(segment_offsets.begin(), segment_offsets.end()), + "Invalid API parameter: erroneous segment_offsets."); + CUGRAPH_EXPECTS(segment_offsets[0] == 0, "Invalid API parameter: segment_offsets."); + CUGRAPH_EXPECTS(segment_offsets.back() == this->get_number_of_vertices(), + "Invalid API parameter: segment_offsets."); + } + + if (is_symmetric) {} + if (!is_multigraph) {} + } +} + +// explicit instantiation + +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; + +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; + +} // namespace experimental +} // namespace cugraph \ No newline at end of file From e08158051741e666e9b6ca2b9d7331805902eee2 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 14:01:17 -0400 Subject: [PATCH 18/77] docs --- docs/source/conf.py | 8 +++++++ docs/source/cugraph_blogs.rst | 39 +++++++++++++++++-------------- docs/source/cugraph_intro.md | 6 +++++ docs/source/cugraph_ref.md | 26 +++++++++++++++++++++ docs/source/index.rst | 4 ++++ python/cugraph/structure/graph.py | 1 + 6 files changed, 67 insertions(+), 17 deletions(-) create mode 100644 docs/source/cugraph_intro.md create mode 100644 docs/source/cugraph_ref.md diff --git a/docs/source/conf.py b/docs/source/conf.py index 6b144e93c94..0c8a0316278 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -188,3 +188,11 @@ def setup(app): app.add_stylesheet('params.css') + +from recommonmark.parser import CommonMarkParser + +source_parsers = { + '.md': CommonMarkParser, +} + +source_suffix = ['.rst', '.md'] \ No newline at end of file diff --git a/docs/source/cugraph_blogs.rst b/docs/source/cugraph_blogs.rst index b4150c459fe..e42151049ef 100644 --- a/docs/source/cugraph_blogs.rst +++ b/docs/source/cugraph_blogs.rst @@ -1,32 +1,37 @@ -cuGraph blogs and other references -=============================== + +cuGraph BLOGS and Presentations +************************************************ The RAPIDS team blogs at https://medium.com/rapids-ai, and many of -these blog posts provide deeper dives into models or key features from -cuGraph. Here, we've selected just a few that are of particular interest -to cuML users: +these blog posts provide deeper dives into features from cuGraph. +Here, we've selected just a few that are of particular interest to cuGraph users: + BLOGS ------------------------------------------------- +============== 2019 -* March `RAPIDS cuGraph `_ -* August `RAPIDS cuGraph — The vision and journey to version 1.0 and beyond `_ +------- + * `RAPIDS cuGraph `_ + * `RAPIDS cuGraph — The vision and journey to version 1.0 and beyond `_ + * `RAPIDS cuGraph : multi-GPU PageRank `_ + * `Similarity in graphs: Jaccard versus the Overlap Coefficient `_ + 2020 +------ + * `Status of RAPIDS cuGraph — Refactoring Code And Rethinking Graphs `_ + * `Tackling Large Graphs with RAPIDS cuGraph and CUDA Unified Memory on GPUs `_ - +Media +=============== + * `Nvidia Rapids cuGraph: Making graph analysis ubiquitous `_ Academic Papers ---------------- - -* `Machine Learning in Python: Main developments and technology trends in data science, machine learning, and artificial intelligence (Sebastian Raschka, Joshua Patterson, Corey Nolet) `_ - - - -References ---------------- +=============== + * HPEC +Copyright (c) 2020, NVIDIA CORPORATION. diff --git a/docs/source/cugraph_intro.md b/docs/source/cugraph_intro.md new file mode 100644 index 00000000000..ca3e62a8423 --- /dev/null +++ b/docs/source/cugraph_intro.md @@ -0,0 +1,6 @@ + +cuGraph Intro +------------------------------ + + + diff --git a/docs/source/cugraph_ref.md b/docs/source/cugraph_ref.md new file mode 100644 index 00000000000..7251cac0be9 --- /dev/null +++ b/docs/source/cugraph_ref.md @@ -0,0 +1,26 @@ +# References +------------------------------ + +Betweenness Centrality +* +Katz +* Cohen, J., “Trusses: Cohesive subgraphs for social network analysis” National security agency technical report, 2008 +* O. Green, J. Fox, E. Kim, F. Busato, et al. “Quickly Finding a Truss in a Haystack” IEEE High Performance Extreme Computing Conference (HPEC), 2017 https://doi.org/10.1109/HPEC.2017.8091038 +* O. Green, P. Yalamanchili, L.M. Munguia, “Fast Triangle Counting on GPU” Irregular Applications: Architectures and Algorithms (IA3), 2014 + + + +------------------------ +## Data Sets + +| Data File | Reference | +|------------------|-------------------------------------------------------------------------| +| karate | W. W. Zachary, Aninformation flow model for conflict and fission in small groups, Journal of Anthropological Research 33, 452-473 (1977). | +| dolphins | D. Lusseau, K. Schneider, O. J. Boisseau, P. Haase, E. Slooten, and S. M. Dawson, The bottlenose dolphin community of Doubtful Sound features a large proportion of long-lasting associations, Behavioral Ecology and Sociobiology 54, 396-405 (2003). | +| netscience | M. E. J. Newman, Finding community structure in networks using the eigenvectors of matrices, Preprint physics/0605087 (2006). | +| email-Eu-core | Hao Yin, Austin R. Benson, Jure Leskovec, and David F. Gleich. "Local Higher-order Graph Clustering." In Proceedings of the 23rd ACM SIGKDD nternational Conference on Knowledge Discovery and Data Mining. 2017.

J. Leskovec, J. Kleinberg and C. Faloutsos. Graph Evolution: Densification and Shrinking Diameters. ACM Transactions on Knowledge Discovery from Data (ACM TKDD), 1(1), 2007. http://www.cs.cmu.edu/~jure/pubs/powergrowth-tkdd.pdf | +| polbooks | V. Krebs, unpublished, http://www.orgnet.com/. | + + + + diff --git a/docs/source/index.rst b/docs/source/index.rst index 47360b88f91..cae4d9ed56e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,6 +7,10 @@ Welcome to cugraph's documentation! api.rst dask-cugraph.rst + cugraph_intro.md + cugraph_blogs.rst + cugraph_ref.rst + Indices and tables ================== diff --git a/python/cugraph/structure/graph.py b/python/cugraph/structure/graph.py index f579a32cab6..a9e84f2c038 100644 --- a/python/cugraph/structure/graph.py +++ b/python/cugraph/structure/graph.py @@ -931,6 +931,7 @@ def out_degree(self, vertex_subset=None): raise Exception("Not supported for distributed graph") return self._degree(vertex_subset, x=2) + def degree(self, vertex_subset=None): """ Compute vertex degree. By default, this method computes vertex From 8a554c5a2a4f13b43346720108cab0f7e48d7124 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 14:11:32 -0400 Subject: [PATCH 19/77] changed formating --- docs/source/cugraph_intro.md | 9 ++++++- docs/source/cugraph_ref.md | 26 ------------------ docs/source/cugraph_ref.rst | 52 ++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 27 deletions(-) delete mode 100644 docs/source/cugraph_ref.md create mode 100644 docs/source/cugraph_ref.rst diff --git a/docs/source/cugraph_intro.md b/docs/source/cugraph_intro.md index ca3e62a8423..3a875e58e80 100644 --- a/docs/source/cugraph_intro.md +++ b/docs/source/cugraph_intro.md @@ -1,6 +1,13 @@ -cuGraph Intro +# cuGraph Intro ------------------------------ +## Graph Type + + +## Algorithms + + +## Using diff --git a/docs/source/cugraph_ref.md b/docs/source/cugraph_ref.md deleted file mode 100644 index 7251cac0be9..00000000000 --- a/docs/source/cugraph_ref.md +++ /dev/null @@ -1,26 +0,0 @@ -# References ------------------------------- - -Betweenness Centrality -* -Katz -* Cohen, J., “Trusses: Cohesive subgraphs for social network analysis” National security agency technical report, 2008 -* O. Green, J. Fox, E. Kim, F. Busato, et al. “Quickly Finding a Truss in a Haystack” IEEE High Performance Extreme Computing Conference (HPEC), 2017 https://doi.org/10.1109/HPEC.2017.8091038 -* O. Green, P. Yalamanchili, L.M. Munguia, “Fast Triangle Counting on GPU” Irregular Applications: Architectures and Algorithms (IA3), 2014 - - - ------------------------- -## Data Sets - -| Data File | Reference | -|------------------|-------------------------------------------------------------------------| -| karate | W. W. Zachary, Aninformation flow model for conflict and fission in small groups, Journal of Anthropological Research 33, 452-473 (1977). | -| dolphins | D. Lusseau, K. Schneider, O. J. Boisseau, P. Haase, E. Slooten, and S. M. Dawson, The bottlenose dolphin community of Doubtful Sound features a large proportion of long-lasting associations, Behavioral Ecology and Sociobiology 54, 396-405 (2003). | -| netscience | M. E. J. Newman, Finding community structure in networks using the eigenvectors of matrices, Preprint physics/0605087 (2006). | -| email-Eu-core | Hao Yin, Austin R. Benson, Jure Leskovec, and David F. Gleich. "Local Higher-order Graph Clustering." In Proceedings of the 23rd ACM SIGKDD nternational Conference on Knowledge Discovery and Data Mining. 2017.

J. Leskovec, J. Kleinberg and C. Faloutsos. Graph Evolution: Densification and Shrinking Diameters. ACM Transactions on Knowledge Discovery from Data (ACM TKDD), 1(1), 2007. http://www.cs.cmu.edu/~jure/pubs/powergrowth-tkdd.pdf | -| polbooks | V. Krebs, unpublished, http://www.orgnet.com/. | - - - - diff --git a/docs/source/cugraph_ref.rst b/docs/source/cugraph_ref.rst new file mode 100644 index 00000000000..d5953bc9ed5 --- /dev/null +++ b/docs/source/cugraph_ref.rst @@ -0,0 +1,52 @@ +# References +=============== + +Betweenness Centrality +------------------------- + + +Katz +------------------------- + +- J. Cohen, *Trusses: Cohesive subgraphs for social network analysis* National security agency technical report, 2008 +- O. Green, J. Fox, E. Kim, F. Busato, et al. *Quickly Finding a Truss in a Haystack* IEEE High Performance Extreme Computing Conference (HPEC), 2017 https://doi.org/10.1109/HPEC.2017.8091038 +- O. Green, P. Yalamanchili, L.M. Munguia, “*ast Triangle Counting on GPU* Irregular Applications: Architectures and Algorithms (IA3), 2014 + + + + + + +Data Sets +=============== + ++------------------+-------------------------------------------------------------------------+ +| Data File | Reference | ++==================+=========================================================================+ +| karate | W. W. Zachary, | +| | Aninformation flow model for conflict and fission in small groups, | +| | Journal of Anthropological Research 33, 452-473 (1977). | ++------------------+-------------------------------------------------------------------------+ +| dolphins | D. Lusseau, K. Schneider, O. J. Boisseau, P. Haase, E. Slooten, and | +| | S. M. Dawson, The bottlenose dolphin community of Doubtful Sound | +| | features a large proportion of long-lasting associations, | +| | Behavioral Ecology and Sociobiology 54, 396-405 (2003). | ++------------------+-------------------------------------------------------------------------+ +| netscience | M. E. J. Newman, Finding community structure in networks using the | +| | eigenvectors of matrices, Preprint physics/0605087 (2006). | ++------------------+-------------------------------------------------------------------------+ +| email-Eu-core | Hao Yin, Austin R. Benson, Jure Leskovec, and David F. Gleich. "Local | +| | Higher-order Graph Clustering." In Proceedings of the 23rd ACM SIGKDD | +| | International Conference on Knowledge Discovery and Data Mining. 2017. | +| | | +| | J. Leskovec, J. Kleinberg and C. Faloutsos. Graph Evolution: | +| | Densification and Shrinking Diameters. ACM Transactions on Knowledge | +| | Discovery from Data (ACM TKDD), 1(1), 2007. | +| | http://www.cs.cmu.edu/~jure/pubs/powergrowth-tkdd.pdf | ++------------------+-------------------------------------------------------------------------+ +| polbooks | V. Krebs, unpublished, http://www.orgnet.com/. | ++------------------+-------------------------------------------------------------------------+ + + + + From 8b846bf22bbcff276bd50b8ccc28c63231bdf06c Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 14:34:46 -0400 Subject: [PATCH 20/77] better formating --- docs/source/cugraph_ref.rst | 49 +++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/docs/source/cugraph_ref.rst b/docs/source/cugraph_ref.rst index d5953bc9ed5..b5d0709a10b 100644 --- a/docs/source/cugraph_ref.rst +++ b/docs/source/cugraph_ref.rst @@ -3,6 +3,9 @@ Betweenness Centrality ------------------------- +- Brandes, U. (2001). A faster algorithm for betweenness centrality. Journal of mathematical sociology, 25(2), 163-177. +- Brandes, U. (2008). On variants of shortest-path betweenness centrality and their generic computation. Social Networks, 30(2), 136-145. +- McLaughlin, A., & Bader, D. A. (2018). Accelerating GPU betweenness centrality. Communications of the ACM, 61(8), 85-92. Katz @@ -19,33 +22,25 @@ Katz Data Sets =============== - -+------------------+-------------------------------------------------------------------------+ -| Data File | Reference | -+==================+=========================================================================+ -| karate | W. W. Zachary, | -| | Aninformation flow model for conflict and fission in small groups, | -| | Journal of Anthropological Research 33, 452-473 (1977). | -+------------------+-------------------------------------------------------------------------+ -| dolphins | D. Lusseau, K. Schneider, O. J. Boisseau, P. Haase, E. Slooten, and | -| | S. M. Dawson, The bottlenose dolphin community of Doubtful Sound | -| | features a large proportion of long-lasting associations, | -| | Behavioral Ecology and Sociobiology 54, 396-405 (2003). | -+------------------+-------------------------------------------------------------------------+ -| netscience | M. E. J. Newman, Finding community structure in networks using the | -| | eigenvectors of matrices, Preprint physics/0605087 (2006). | -+------------------+-------------------------------------------------------------------------+ -| email-Eu-core | Hao Yin, Austin R. Benson, Jure Leskovec, and David F. Gleich. "Local | -| | Higher-order Graph Clustering." In Proceedings of the 23rd ACM SIGKDD | -| | International Conference on Knowledge Discovery and Data Mining. 2017. | -| | | -| | J. Leskovec, J. Kleinberg and C. Faloutsos. Graph Evolution: | -| | Densification and Shrinking Diameters. ACM Transactions on Knowledge | -| | Discovery from Data (ACM TKDD), 1(1), 2007. | -| | http://www.cs.cmu.edu/~jure/pubs/powergrowth-tkdd.pdf | -+------------------+-------------------------------------------------------------------------+ -| polbooks | V. Krebs, unpublished, http://www.orgnet.com/. | -+------------------+-------------------------------------------------------------------------+ +karate + - W. W. Zachary, *Aninformation flow model for conflict and fission in small groups*, Journal of Anthropological Research 33, 452-473 (1977). +dolphins + - D. Lusseau, K. Schneider, O. J. Boisseau, P. Haase, E. Slooten, and S. M. Dawson, + *The bottlenose dolphin community of Doubtful Sound features a large proportion of long-lasting associations*, + Behavioral Ecology and Sociobiology 54, 396-405 (2003). +netscience + - M. E. J. Newman, + *Finding community structure in networks using the eigenvectors of matrices*, + Preprint physics/0605087 (2006). +email-Eu-core + - Hao Yin, Austin R. Benson, Jure Leskovec, and David F. Gleich. + *Local Higher-order Graph Clustering.* + In Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining. 2017. + - J. Leskovec, J. Kleinberg and C. Faloutsos. + *Graph Evolution: Densification and Shrinking Diameters*. + ACM Transactions on Knowledge Discovery from Data (ACM TKDD), 1(1), 2007. http://www.cs.cmu.edu/~jure/pubs/powergrowth-tkdd.pdf +polbooks + - V. Krebs, unpublished, http://www.orgnet.com/. From ed00428f215211b7b84c3988229d3bedd4d7917e Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 14:41:10 -0400 Subject: [PATCH 21/77] formating --- docs/source/cugraph_intro.md | 13 ------------- docs/source/cugraph_intro.rst | 13 +++++++++++++ docs/source/cugraph_ref.rst | 8 ++++++-- docs/source/index.rst | 2 +- 4 files changed, 20 insertions(+), 16 deletions(-) delete mode 100644 docs/source/cugraph_intro.md create mode 100644 docs/source/cugraph_intro.rst diff --git a/docs/source/cugraph_intro.md b/docs/source/cugraph_intro.md deleted file mode 100644 index 3a875e58e80..00000000000 --- a/docs/source/cugraph_intro.md +++ /dev/null @@ -1,13 +0,0 @@ - -# cuGraph Intro ------------------------------- - - - -## Graph Type - - -## Algorithms - - -## Using diff --git a/docs/source/cugraph_intro.rst b/docs/source/cugraph_intro.rst new file mode 100644 index 00000000000..cd2d750e35f --- /dev/null +++ b/docs/source/cugraph_intro.rst @@ -0,0 +1,13 @@ + +cuGraph Intro +------------------------------ + + + +Graph Type + + +Algorithms + + +Using diff --git a/docs/source/cugraph_ref.rst b/docs/source/cugraph_ref.rst index b5d0709a10b..4c627c4c245 100644 --- a/docs/source/cugraph_ref.rst +++ b/docs/source/cugraph_ref.rst @@ -1,6 +1,9 @@ -# References +References =============== +Algorithms +************** + Betweenness Centrality ------------------------- - Brandes, U. (2001). A faster algorithm for betweenness centrality. Journal of mathematical sociology, 25(2), 163-177. @@ -21,7 +24,8 @@ Katz Data Sets -=============== +************** + karate - W. W. Zachary, *Aninformation flow model for conflict and fission in small groups*, Journal of Anthropological Research 33, 452-473 (1977). dolphins diff --git a/docs/source/index.rst b/docs/source/index.rst index cae4d9ed56e..2cd95e7f129 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,7 @@ Welcome to cugraph's documentation! api.rst dask-cugraph.rst - cugraph_intro.md + cugraph_intro.rst cugraph_blogs.rst cugraph_ref.rst From 352d56b761632ce8bc5c77b5989ebb1a25c24262 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 14:45:39 -0400 Subject: [PATCH 22/77] pr 1129 changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e52c6d424a3..b69c6786756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree +- PR 1129 Updating test and documentation ## Bug Fixes From bbb70ab81553a280dcd8b8aaf18f958fdcc62f4f Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 14:46:25 -0400 Subject: [PATCH 23/77] type --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b69c6786756..9c54f3c1163 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree -- PR 1129 Updating test and documentation +- PR #1129 Updating test and documentation ## Bug Fixes From 4a1f52b9a2bcc356dedf6f69181a11a287f79192 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Fri, 28 Aug 2020 15:11:08 -0400 Subject: [PATCH 24/77] more ref --- docs/source/cugraph_blogs.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/cugraph_blogs.rst b/docs/source/cugraph_blogs.rst index e42151049ef..ec8c0f0e278 100644 --- a/docs/source/cugraph_blogs.rst +++ b/docs/source/cugraph_blogs.rst @@ -27,7 +27,7 @@ BLOGS Media =============== * `Nvidia Rapids cuGraph: Making graph analysis ubiquitous `_ - + * `RAPIDS cuGraph – Accelerating all your Graph needs `_ Academic Papers =============== From 6ab0a0eb48c616dfe5643830a0e5ab57c21b44bb Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 28 Aug 2020 15:33:54 -0400 Subject: [PATCH 25/77] re-add edge_t number_of_edges to graph_t constructor, sum edgelists[i].number_of_edges is sum of the local edges --- cpp/include/experimental/graph.hpp | 1 + cpp/src/experimental/graph.cu | 58 +++++++++++------------------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index e1f075f2c76..520577556c7 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -64,6 +64,7 @@ class graph_t> const &edge_lists, partition_t const &partition, vertex_t number_of_vertices, + edge_t number_of_edges, bool is_symmetric, bool is_multigraph, bool is_weighted, diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 2b037e21f83..47deb4b2682 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -38,19 +38,6 @@ namespace experimental { namespace { -// FIXME: threshold values require tuning -size_t constexpr low_degree_threshold{raft::warp_size()}; -size_t constexpr mid_degree_threshold{1024}; -size_t constexpr num_segments_per_vertex_partition{3}; - -template -edge_t sum_number_of_edges(std::vector> const &edgelists) -{ - edge_t number_of_edges{0}; - for (size_t i = 0; i < edgelists.size(); ++i) { number_of_edges += edgelists[i].number_of_edges; } - return number_of_edges; -} - template std:: tuple, rmm::device_uvector, rmm::device_uvector> @@ -200,17 +187,14 @@ graph_t> const &edgelists, partition_t const &partition, vertex_t number_of_vertices, + edge_t number_of_edges, bool is_symmetric, bool is_multigraph, bool is_weighted, bool sorted_by_global_degree_within_vertex_partition, bool do_expensive_check) - : detail::graph_base_t(handle, - number_of_vertices, - sum_number_of_edges(edgelists), - is_symmetric, - is_multigraph, - is_weighted), + : detail::graph_base_t( + handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), partition_(partition) { auto &comm_p_row = this->get_handle_ptr()->get_subcomm(comm_p_row_key); @@ -221,8 +205,6 @@ graph_tget_handle_ptr()->get_stream(); - // FIXME: error checks - // convert edge list (COO) to compressed sparse format (CSR or CSC) adj_matrix_partition_offsets_.reserve(edgelists.size()); @@ -258,20 +240,21 @@ graph_tget_handle_ptr()), adj_matrix_partition_offsets_, partition_); - static_assert(num_segments_per_vertex_partition == 3); - static_assert((low_degree_threshold <= mid_degree_threshold) && - (mid_degree_threshold <= std::numeric_limits::max())); - rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, default_stream); - std::vector h_thresholds = {static_cast(low_degree_threshold), - static_cast(mid_degree_threshold)}; + static_assert(detail::num_segments_per_vertex_partition == 3); + static_assert((detail::low_degree_threshold <= detail::mid_degree_threshold) && + (detail::mid_degree_threshold <= std::numeric_limits::max())); + rmm::device_uvector d_thresholds(detail::num_segments_per_vertex_partition - 1, + default_stream); + std::vector h_thresholds = {static_cast(detail::low_degree_threshold), + static_cast(detail::mid_degree_threshold)}; raft::update_device( d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); - rmm::device_uvector segment_offsets(num_segments_per_vertex_partition + 1, + rmm::device_uvector segment_offsets(detail::num_segments_per_vertex_partition + 1, default_stream); segment_offsets.set_element_async(0, 0, default_stream); segment_offsets.set_element_async( - num_segments_per_vertex_partition, degrees.size(), default_stream); + detail::num_segments_per_vertex_partition, degrees.size(), default_stream); thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), degrees.begin(), @@ -344,20 +327,21 @@ graph_t::max())); - rmm::device_uvector d_thresholds(num_segments_per_vertex_partition - 1, default_stream); - std::vector h_thresholds = {static_cast(low_degree_threshold), - static_cast(mid_degree_threshold)}; + static_assert(detail::num_segments_per_vertex_partition == 3); + static_assert((detail::low_degree_threshold <= detail::mid_degree_threshold) && + (detail::mid_degree_threshold <= std::numeric_limits::max())); + rmm::device_uvector d_thresholds(detail::num_segments_per_vertex_partition - 1, + default_stream); + std::vector h_thresholds = {static_cast(detail::low_degree_threshold), + static_cast(detail::mid_degree_threshold)}; raft::update_device( d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); - rmm::device_uvector segment_offsets(num_segments_per_vertex_partition + 1, + rmm::device_uvector segment_offsets(detail::num_segments_per_vertex_partition + 1, default_stream); segment_offsets.set_element_async(0, 0, default_stream); segment_offsets.set_element_async( - num_segments_per_vertex_partition, degrees.size(), default_stream); + detail::num_segments_per_vertex_partition, degrees.size(), default_stream); thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), degrees.begin(), From 72e16543d5b4c8741c0cbb7fa7e24c13c7df3660 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 28 Aug 2020 16:33:46 -0400 Subject: [PATCH 26/77] add error checks to graph_t --- .../experimental/detail/graph_utils.cuh | 16 ++ cpp/include/experimental/graph.hpp | 2 +- cpp/src/experimental/graph.cu | 145 +++++++++++++++++- cpp/src/experimental/graph_view.cu | 38 ++--- 4 files changed, 176 insertions(+), 25 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index ca0a326f262..0cdf113e2e7 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -21,9 +21,11 @@ #include #include +#include #include #include +#include namespace cugraph { namespace experimental { @@ -101,6 +103,20 @@ rmm::device_uvector compute_major_degree( return compute_major_degree(handle, tmp_offsets, partition); } +template +void check_vertex_partition_offsets(std::vector const& vertex_partition_offsets, vertex_t number_of_vertices) +{ + CUGRAPH_EXPECTS( + std::is_sorted(vertex_partition_offsets.begin(), + vertex_partition_offsets.end()), + "Invalid API parameter: partition.vertex_partition_offsets values should be non-descending."); + CUGRAPH_EXPECTS(vertex_partition_offsets[0] == vertex_t{0}, + "Invalid API parameter: partition.vertex_partition_offsets[0] should be 0."); + CUGRAPH_EXPECTS(vertex_partition_offsets.back() == number_of_vertices, + "Invalid API parameter: partition.vertex_partition_offsets.back() should be " + "number_of_vertices."); +} + } // namespace detail } // namespace experimental } // namespace cugraph diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index 520577556c7..74a7dc1f435 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -139,7 +139,7 @@ class graph_tget_number_of_vertices(); } diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 47deb4b2682..b7481f19f75 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -31,6 +31,7 @@ #include #include +#include #include namespace cugraph { @@ -38,6 +39,24 @@ namespace experimental { namespace { +// can't use lambda due to nvcc limitations (The enclosing parent function ("graph_view_t") for an +// extended __device__ lambda must allow its address to be taken) +template +struct out_of_range_t { + vertex_t major_first{}; + vertex_t major_last{}; + vertex_t minor_first{}; + vertex_t minor_last{}; + + __device__ bool operator()(thrust::tuple t) + { + auto major = thrust::get<0>(t); + auto minor = thrust::get<1>(t); + return (major < major_first) || (major >= major_last) || (minor < minor_first) || + (minor >= minor_last); + } +}; + template std:: tuple, rmm::device_uvector, rmm::device_uvector> @@ -197,6 +216,10 @@ graph_tget_handle_ptr()->get_comms(); + auto comm_p_size = comm_p.get_size(); auto &comm_p_row = this->get_handle_ptr()->get_subcomm(comm_p_row_key); auto comm_p_row_rank = comm_p_row.get_rank(); auto comm_p_row_size = comm_p_row.get_size(); @@ -205,6 +228,70 @@ graph_tget_handle_ptr()->get_stream(); + for (size_t i = 0; i < edgelists.size(); ++i) { + CUGRAPH_EXPECTS( + (edgelists[i].p_src_vertices != nullptr) && (edgelists[i].p_dst_vertices != nullptr), + "Invalid API parameter: edgelists[].p_src_vertices and edgelists[].p_dst_vertices should " + "not be nullptr."); + CUGRAPH_EXPECTS((is_weighted && (edgelists[i].p_edge_weights != nullptr)) || + (!is_weighted && (edgelists[i].p_edge_weights == nullptr)), + "Invalid API parameter: edgelists[].p_edge_weights should not be nullptr (if " + "is_weighted is true) or should be nullptr (if is_weighted is false)."); + } + + CUGRAPH_EXPECTS((partition.hypergraph_partitioned && + (edgelists.size() == static_cast(comm_p_row_size))) || + (!(partition.hypergraph_partitioned) && (edgelists.size() == 1)), + "Invalid API parameter: errneous edgelists.size()."); + + CUGRAPH_EXPECTS(partition.vertex_partition_offsets.size() == static_cast(comm_p_size), + "Invalid API parameter: erroneous partition.vertex_partition_offsets.size()."); + + // optinoal expensive checks (part 1/2) + + if (do_expensive_check) { + edge_t number_of_local_edges_sum{}; + for (size_t i = 0; i < edgelists.size(); ++i) { + auto major_first = + partition.hypergraph_partitioned + ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank] + : partition.vertex_partition_offsets[comm_p_row_rank * comm_p_col_size]; + auto major_last = + partition.hypergraph_partitioned + ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank + 1] + : partition.vertex_partition_offsets[(comm_p_row_rank + 1) * comm_p_col_size]; + auto minor_first = partition.vertex_partition_offsets[comm_p_col_rank * comm_p_row_size]; + auto minor_last = partition.vertex_partition_offsets[(comm_p_col_rank + 1) * comm_p_row_size]; + + number_of_local_edges_sum += edgelists[i].number_of_edges; + + auto edge_first = thrust::make_zip_iterator(thrust::make_tuple( + store_transposed ? edgelists[i].p_dst_vertices : edgelists[i].p_src_vertices, + store_transposed ? edgelists[i].p_src_vertices : edgelists[i].p_dst_vertices)); + // better use thrust::any_of once https://github.com/thrust/thrust/issues/1016 is resolved + CUGRAPH_EXPECTS(thrust::count_if(rmm::exec_policy(default_stream)->on(default_stream), + edge_first, + edge_first + edgelists[i].number_of_edges, + out_of_range_t{ + major_first, major_last, minor_first, minor_last}) == 0, + "Invalid API parameter: edgelists[] have out-of-range values."); + } + this->get_handle_ptr()->get_comms().allreduce(&number_of_local_edges_sum, + &number_of_local_edges_sum, + 1, + raft::comms::op_t::SUM, + default_stream); + CUGRAPH_EXPECTS(number_of_local_edges_sum == this->get_number_of_edges(), + "Invalid API parameter: the sum of local edges doe counts not match with " + "number_of_local_edges."); + + detail::check_vertex_partition_offsets(partition.vertex_partition_offsets, + this->get_number_of_vertices()); + + if (is_symmetric) {} + if (!is_multigraph) {} + } + // convert edge list (COO) to compressed sparse format (CSR or CSC) adj_matrix_partition_offsets_.reserve(edgelists.size()); @@ -240,6 +327,19 @@ graph_tget_handle_ptr()), adj_matrix_partition_offsets_, partition_); + // optinoal expensive checks (part 2/2) + + if (do_expensive_check) { + if (sorted_by_global_degree_within_vertex_partition) { + CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), + degrees.begin(), + degrees.end(), + thrust::greater{}), + "Invalid API parameter: sorted_by_global_degree_within_vertex_partition is " + "set to true, but degrees are not non-ascending."); + } + } + static_assert(detail::num_segments_per_vertex_partition == 3); static_assert((detail::low_degree_threshold <= detail::mid_degree_threshold) && (detail::mid_degree_threshold <= std::numeric_limits::max())); @@ -289,7 +389,7 @@ graph_t(handle, number_of_vertices, @@ -301,9 +401,37 @@ graph_t(0, handle.get_stream())), weights_(rmm::device_uvector(0, handle.get_stream())) { + // cheap error checks + auto default_stream = this->get_handle_ptr()->get_stream(); - // FIXME: error checks + CUGRAPH_EXPECTS( + (edgelist.p_src_vertices != nullptr) && (edgelist.p_dst_vertices != nullptr), + "Invalid API parameter: edgelist.p_src_vertices and edgelist.p_dst_vertices should " + "not be nullptr."); + CUGRAPH_EXPECTS((is_weighted && (edgelist.p_edge_weights != nullptr)) || + (!is_weighted && (edgelist.p_edge_weights == nullptr)), + "Invalid API parameter: edgelist.p_edge_weights should not be nullptr (if " + "is_weighted is true) or should be nullptr (if is_weighted is false)."); + + // optinoal expensive checks (part 1/2) + + if (do_expensive_check) { + auto edge_first = thrust::make_zip_iterator( + thrust::make_tuple(store_transposed ? edgelist.p_dst_vertices : edgelist.p_src_vertices, + store_transposed ? edgelist.p_src_vertices : edgelist.p_dst_vertices)); + // better use thrust::any_of once https://github.com/thrust/thrust/issues/1016 is resolved + CUGRAPH_EXPECTS(thrust::count_if( + rmm::exec_policy(default_stream)->on(default_stream), + edge_first, + edge_first + edgelist.number_of_edges, + out_of_range_t{ + 0, this->get_number_of_vertices(), 0, this->get_number_of_vertices()}) == 0, + "Invalid API parameter: edgelist have out-of-range values."); + + if (is_symmetric) {} + if (!is_multigraph) {} + } // convert edge list (COO) to compressed sparse format (CSR or CSC) @@ -327,6 +455,19 @@ graph_ton(default_stream), + degrees.begin(), + degrees.end(), + thrust::greater{}), + "Invalid API parameter: sorted_by_degree is set to true, but degrees are not " + "non-ascending."); + } + } + static_assert(detail::num_segments_per_vertex_partition == 3); static_assert((detail::low_degree_threshold <= detail::mid_degree_threshold) && (detail::mid_degree_threshold <= std::numeric_limits::max())); diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index b1b07413baa..998d79e1bff 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -23,6 +23,8 @@ #include #include +#include +#include #include #include @@ -123,6 +125,7 @@ graph_view_tget_handle_ptr()->get_subcomm(comm_p_row_key).get_rank(); auto comm_p_col_rank = this->get_handle_ptr()->get_subcomm(comm_p_col_key).get_rank(); + edge_t number_of_local_edges_sum{}; for (size_t i = 0; i < adj_matrix_partition_offsets.size(); ++i) { auto major_first = partition.hypergraph_partitioned @@ -144,6 +147,7 @@ graph_view_t{minor_first, minor_last}) == 0, "Invalid API parameter: adj_matrix_partition_indices[] have out-of-range vertex IDs."); - - edge_t number_of_local_edges_sum{}; - this->get_handle_ptr()->get_comms().allreduce(&number_of_local_edges, - &number_of_local_edges_sum, - 1, - raft::comms::op_t::SUM, - default_stream); - CUGRAPH_EXPECTS(number_of_local_edges_sum == number_of_edges, - "Invalid API parameter: the sum of local edges doe counts not match with " - "number_of_local_edges."); } + this->get_handle_ptr()->get_comms().allreduce(&number_of_local_edges_sum, + &number_of_local_edges_sum, + 1, + raft::comms::op_t::SUM, + default_stream); + CUGRAPH_EXPECTS(number_of_local_edges_sum == this->get_number_of_edges(), + "Invalid API parameter: the sum of local edges doe counts not match with " + "number_of_local_edges."); if (sorted_by_global_degree_within_vertex_partition) { auto degrees = detail::compute_major_degree(handle, adj_matrix_partition_offsets, partition); @@ -194,15 +196,7 @@ graph_view_tget_number_of_vertices()); if (is_symmetric) {} if (!is_multigraph) {} @@ -270,9 +264,9 @@ graph_view_t{offsets}); + auto degree_first = + thrust::make_transform_iterator(thrust::make_counting_iterator(vertex_t{0}), + degree_from_offsets_t{offsets}); CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), degree_first, degree_first + this->get_number_of_vertices(), From 8f59a9e57fb70f6ceb876d270736e839f5ddd392 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Mon, 31 Aug 2020 09:27:12 -0400 Subject: [PATCH 27/77] add error checkes/comments for graph_t & graph_view_t --- .../experimental/detail/graph_utils.cuh | 15 +- cpp/src/experimental/graph.cu | 175 ++++++++++-------- cpp/src/experimental/graph_view.cu | 17 +- 3 files changed, 113 insertions(+), 94 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index 0cdf113e2e7..44ad157a309 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -24,8 +24,8 @@ #include #include -#include #include +#include namespace cugraph { namespace experimental { @@ -104,11 +104,11 @@ rmm::device_uvector compute_major_degree( } template -void check_vertex_partition_offsets(std::vector const& vertex_partition_offsets, vertex_t number_of_vertices) +void check_vertex_partition_offsets(std::vector const &vertex_partition_offsets, + vertex_t number_of_vertices) { CUGRAPH_EXPECTS( - std::is_sorted(vertex_partition_offsets.begin(), - vertex_partition_offsets.end()), + std::is_sorted(vertex_partition_offsets.begin(), vertex_partition_offsets.end()), "Invalid API parameter: partition.vertex_partition_offsets values should be non-descending."); CUGRAPH_EXPECTS(vertex_partition_offsets[0] == vertex_t{0}, "Invalid API parameter: partition.vertex_partition_offsets[0] should be 0."); @@ -117,6 +117,13 @@ void check_vertex_partition_offsets(std::vector const& vertex_partitio "number_of_vertices."); } +template +struct degree_from_offsets_t { + edge_t const* offsets{nullptr}; + + __device__ edge_t operator()(vertex_t v) { return offsets[v + 1] - offsets[v]; } +}; + } // namespace detail } // namespace experimental } // namespace cugraph diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index b7481f19f75..87ea812ce46 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -247,7 +247,7 @@ graph_t(comm_p_size), "Invalid API parameter: erroneous partition.vertex_partition_offsets.size()."); - // optinoal expensive checks (part 1/2) + // optinoal expensive checks (part 1/3) if (do_expensive_check) { edge_t number_of_local_edges_sum{}; @@ -287,9 +287,6 @@ graph_tget_number_of_vertices()); - - if (is_symmetric) {} - if (!is_multigraph) {} } // convert edge list (COO) to compressed sparse format (CSR or CSC) @@ -324,13 +321,13 @@ graph_tget_handle_ptr()), adj_matrix_partition_offsets_, partition_); + if (sorted_by_global_degree_within_vertex_partition) { + auto degrees = detail::compute_major_degree( + *(this->get_handle_ptr()), adj_matrix_partition_offsets_, partition_); - // optinoal expensive checks (part 2/2) + // optinoal expensive checks (part 2/3) - if (do_expensive_check) { - if (sorted_by_global_degree_within_vertex_partition) { + if (do_expensive_check) { CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), degrees.begin(), degrees.end(), @@ -338,43 +335,53 @@ graph_t::max())); + rmm::device_uvector d_thresholds(detail::num_segments_per_vertex_partition - 1, + default_stream); + std::vector h_thresholds = {static_cast(detail::low_degree_threshold), + static_cast(detail::mid_degree_threshold)}; + raft::update_device( + d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); + + rmm::device_uvector segment_offsets(detail::num_segments_per_vertex_partition + 1, + default_stream); + segment_offsets.set_element_async(0, 0, default_stream); + segment_offsets.set_element_async( + detail::num_segments_per_vertex_partition, degrees.size(), default_stream); + + thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), + degrees.begin(), + degrees.end(), + d_thresholds.begin(), + d_thresholds.end(), + segment_offsets.begin() + 1); + + rmm::device_uvector aggregate_segment_offsets( + comm_p_row_size * segment_offsets.size(), default_stream); + comm_p_row.allgather(segment_offsets.data(), + aggregate_segment_offsets.data(), + segment_offsets.size(), + default_stream); + + vertex_partition_segment_offsets_.resize(comm_p_row_size * (segment_offsets.size())); + raft::update_host(vertex_partition_segment_offsets_.data(), + aggregate_segment_offsets.data(), + aggregate_segment_offsets.size(), + default_stream); } - static_assert(detail::num_segments_per_vertex_partition == 3); - static_assert((detail::low_degree_threshold <= detail::mid_degree_threshold) && - (detail::mid_degree_threshold <= std::numeric_limits::max())); - rmm::device_uvector d_thresholds(detail::num_segments_per_vertex_partition - 1, - default_stream); - std::vector h_thresholds = {static_cast(detail::low_degree_threshold), - static_cast(detail::mid_degree_threshold)}; - raft::update_device( - d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); - - rmm::device_uvector segment_offsets(detail::num_segments_per_vertex_partition + 1, - default_stream); - segment_offsets.set_element_async(0, 0, default_stream); - segment_offsets.set_element_async( - detail::num_segments_per_vertex_partition, degrees.size(), default_stream); - - thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), - degrees.begin(), - degrees.end(), - d_thresholds.begin(), - d_thresholds.end(), - segment_offsets.begin() + 1); - - rmm::device_uvector aggregate_segment_offsets(comm_p_row_size * segment_offsets.size(), - default_stream); - comm_p_row.allgather(segment_offsets.data(), - aggregate_segment_offsets.data(), - segment_offsets.size(), - default_stream); - - vertex_partition_segment_offsets_.resize(comm_p_row_size * (segment_offsets.size())); - raft::update_host(vertex_partition_segment_offsets_.data(), - aggregate_segment_offsets.data(), - aggregate_segment_offsets.size(), - default_stream); + // optinoal expensive checks (part 3/3) + + if (do_expensive_check) { + // FIXME: check for symmetricity may better be implemetned with transpose(). + if (is_symmetric) {} + // FIXME: check for duplicate edges may better be implemented after deciding whether to sort + // neighbor list or not. + if (!is_multigraph) {} + } } template degrees(this->get_number_of_vertices(), default_stream); - thrust::adjacent_difference(rmm::exec_policy(default_stream)->on(default_stream), - offsets_.begin() + 1, - offsets_.end(), - degrees.begin()); + if (sorted_by_degree) { + auto degree_first = thrust::make_transform_iterator( + thrust::make_counting_iterator(vertex_t{0}), + detail::degree_from_offsets_t{offsets_.data()}); - // optinoal expensive checks (part 2/2) + // optinoal expensive checks (part 2/2) - if (do_expensive_check) { - if (sorted_by_degree) { + if (do_expensive_check) { CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), - degrees.begin(), - degrees.end(), + degree_first, + degree_first + this->get_number_of_vertices(), thrust::greater{}), "Invalid API parameter: sorted_by_degree is set to true, but degrees are not " "non-ascending."); } + + static_assert(detail::num_segments_per_vertex_partition == 3); + static_assert((detail::low_degree_threshold <= detail::mid_degree_threshold) && + (detail::mid_degree_threshold <= std::numeric_limits::max())); + rmm::device_uvector d_thresholds(detail::num_segments_per_vertex_partition - 1, + default_stream); + std::vector h_thresholds = {static_cast(detail::low_degree_threshold), + static_cast(detail::mid_degree_threshold)}; + raft::update_device( + d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); + + rmm::device_uvector segment_offsets(detail::num_segments_per_vertex_partition + 1, + default_stream); + segment_offsets.set_element_async(0, 0, default_stream); + segment_offsets.set_element_async( + detail::num_segments_per_vertex_partition, this->get_number_of_vertices(), default_stream); + + thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), + degree_first, + degree_first + this->get_number_of_vertices(), + d_thresholds.begin(), + d_thresholds.end(), + segment_offsets.begin() + 1); + + segment_offsets_.resize(segment_offsets.size()); + raft::update_host( + segment_offsets_.data(), segment_offsets.data(), segment_offsets.size(), default_stream); } - static_assert(detail::num_segments_per_vertex_partition == 3); - static_assert((detail::low_degree_threshold <= detail::mid_degree_threshold) && - (detail::mid_degree_threshold <= std::numeric_limits::max())); - rmm::device_uvector d_thresholds(detail::num_segments_per_vertex_partition - 1, - default_stream); - std::vector h_thresholds = {static_cast(detail::low_degree_threshold), - static_cast(detail::mid_degree_threshold)}; - raft::update_device( - d_thresholds.data(), h_thresholds.data(), h_thresholds.size(), default_stream); - - rmm::device_uvector segment_offsets(detail::num_segments_per_vertex_partition + 1, - default_stream); - segment_offsets.set_element_async(0, 0, default_stream); - segment_offsets.set_element_async( - detail::num_segments_per_vertex_partition, degrees.size(), default_stream); - - thrust::upper_bound(rmm::exec_policy(default_stream)->on(default_stream), - degrees.begin(), - degrees.end(), - d_thresholds.begin(), - d_thresholds.end(), - segment_offsets.begin() + 1); - - segment_offsets_.resize(segment_offsets.size()); - raft::update_host( - segment_offsets_.data(), segment_offsets.data(), segment_offsets.size(), default_stream); + // optinoal expensive checks (part 3/3) + + if (do_expensive_check) { + // FIXME: check for symmetricity may better be implemetned with transpose(). + if (is_symmetric) {} + // FIXME: check for duplicate edges may better be implemented after deciding whether to sort + // neighbor list or not. + if (!is_multigraph) {} + } } // explicit instantiation diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index 998d79e1bff..0f3d29831b6 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -47,15 +47,6 @@ struct out_of_range_t { __device__ bool operator()(vertex_t v) { return (v < min) || (v >= max); } }; -// can't use lambda due to nvcc limitations (The enclosing parent function ("graph_view_t") for an -// extended __device__ lambda must allow its address to be taken) -template -struct degree_from_offsets_t { - edge_t const* offsets{nullptr}; - - __device__ edge_t operator()(vertex_t v) { return offsets[v + 1] - offsets[v]; } -}; - } // namespace template get_number_of_vertices()); + // FIXME: check for symmetricity may better be implemetned with transpose(). if (is_symmetric) {} + // FIXME: check for duplicate edges may better be implemented after deciding whether to sort + // neighbor list or not. if (!is_multigraph) {} } } @@ -266,7 +260,7 @@ graph_view_t{offsets}); + detail::degree_from_offsets_t{offsets}); CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), degree_first, degree_first + this->get_number_of_vertices(), @@ -281,7 +275,10 @@ graph_view_t Date: Mon, 31 Aug 2020 09:32:25 -0400 Subject: [PATCH 28/77] update change log --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616bcda886e..da9af33dc96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # cuGraph 0.16.0 (Date TBD) ## New Features +- PR 1098 Add new graph classes to support 2D partitioning ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree From 8cfdd3b74209919f23c19b40c14202d10252d82a Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Mon, 31 Aug 2020 09:39:47 -0400 Subject: [PATCH 29/77] clang-format --- cpp/include/experimental/detail/graph_utils.cuh | 2 +- cpp/src/experimental/graph_view.cu | 3 ++- cpp/tests/experimental/graph_test.cpp | 12 ++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index 44ad157a309..a1ab473a6ee 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -119,7 +119,7 @@ void check_vertex_partition_offsets(std::vector const &vertex_partitio template struct degree_from_offsets_t { - edge_t const* offsets{nullptr}; + edge_t const *offsets{nullptr}; __device__ edge_t operator()(vertex_t v) { return offsets[v + 1] - offsets[v]; } }; diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index 0f3d29831b6..a38a9ef06ba 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -187,7 +187,8 @@ graph_view_tget_number_of_vertices()); + detail::check_vertex_partition_offsets(partition.vertex_partition_offsets, + this->get_number_of_vertices()); // FIXME: check for symmetricity may better be implemetned with transpose(). if (is_symmetric) {} diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp index a2679d5003d..f8699799e8a 100644 --- a/cpp/tests/experimental/graph_test.cpp +++ b/cpp/tests/experimental/graph_test.cpp @@ -46,17 +46,17 @@ std::tuple, std::vector, std::vector> gr std::vector weights(p_edge_weights != nullptr ? number_of_edges : 0, weight_t{0.0}); for (size_t i = 0; i < number_of_edges; ++i) { - auto major = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; + auto major = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; offsets[1 + major]++; } std::partial_sum(offsets.begin() + 1, offsets.end(), offsets.begin() + 1); for (size_t i = 0; i < number_of_edges; ++i) { - auto major = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; - auto minor = store_transposed ? p_src_vertices[i] : p_dst_vertices[i]; - auto start = offsets[major]; - auto degree = offsets[major + 1] - start; - auto idx = indices[start + degree - 1]++; + auto major = store_transposed ? p_dst_vertices[i] : p_src_vertices[i]; + auto minor = store_transposed ? p_src_vertices[i] : p_dst_vertices[i]; + auto start = offsets[major]; + auto degree = offsets[major + 1] - start; + auto idx = indices[start + degree - 1]++; indices[start + idx] = minor; if (p_edge_weights != nullptr) { weights[start + idx] = p_edge_weights[i]; } } From 0cc71a214e518c552e3fc30f806a11c52135ae95 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:15:45 -0400 Subject: [PATCH 30/77] updates --- CHANGELOG.md | 7 +++-- .../tests/test_betweenness_centrality.py | 14 ++++------ python/cugraph/tests/test_bfs.py | 4 +-- .../tests/test_edge_betweenness_centrality.py | 14 ++++------ python/cugraph/tests/test_graph.py | 4 +-- python/cugraph/tests/test_hits.py | 4 +-- python/cugraph/tests/utils.py | 26 +++---------------- 7 files changed, 23 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c54f3c1163..616bcda886e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,11 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree -- PR #1129 Updating test and documentation +- PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource ## Bug Fixes -# cuGraph 0.15.0 (Date TBD) +# cuGraph 0.15.0 (26 Aug 2020) ## New Features - PR #940 Add MG Batch BC @@ -100,6 +100,8 @@ - PR #1087 Updated benchmarks README to better describe how to get plugin, added rapids-pytest-benchmark plugin to conda dev environments - PR #1101 Removed unnecessary device-to-host copy which caused a performance regression - PR #1106 Added new release.ipynb to notebook test skip list +- PR #1125 Patch Thrust to workaround `CUDA_CUB_RET_IF_FAIL` macro clearing CUDA errors + # cuGraph 0.14.0 (03 Jun 2020) @@ -178,6 +180,7 @@ - PR 935 Merge - PR #956 Use new gpuCI image in local build script + # cuGraph 0.13.0 (31 Mar 2020) ## New Features diff --git a/python/cugraph/tests/test_betweenness_centrality.py b/python/cugraph/tests/test_betweenness_centrality.py index 165ef56e549..13b8ab10397 100644 --- a/python/cugraph/tests/test_betweenness_centrality.py +++ b/python/cugraph/tests/test_betweenness_centrality.py @@ -41,10 +41,6 @@ NORMALIZED_OPTIONS = [False, True] DEFAULT_EPSILON = 0.0001 -DATASETS = ["../datasets/karate.csv", "../datasets/netscience.csv"] - -UNRENUMBERED_DATASETS = ["../datasets/karate.csv"] - SUBSET_SIZE_OPTIONS = [4, None] SUBSET_SEED_OPTIONS = [42] @@ -298,7 +294,7 @@ def prepare_test(): # ============================================================================= # Tests # ============================================================================= -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -330,7 +326,7 @@ def test_betweenness_centrality( compare_scores(sorted_df, first_key="cu_bc", second_key="ref_bc") -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", [None]) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -371,7 +367,7 @@ def test_betweenness_centrality_k_full( # the function operating the comparison inside is first proceeding # to a random sampling over the number of vertices (thus direct offsets) # in the graph structure instead of actual vertices identifiers -@pytest.mark.parametrize("graph_file", UNRENUMBERED_DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_UNRENUMBERED ) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -407,7 +403,7 @@ def test_betweenness_centrality_fixed_sample( compare_scores(sorted_df, first_key="cu_bc", second_key="ref_bc") -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -445,7 +441,7 @@ def test_betweenness_centrality_weight_except( compare_scores(sorted_df, first_key="cu_bc", second_key="ref_bc") -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) diff --git a/python/cugraph/tests/test_bfs.py b/python/cugraph/tests/test_bfs.py index 8f76e25031e..8eb175ad66d 100644 --- a/python/cugraph/tests/test_bfs.py +++ b/python/cugraph/tests/test_bfs.py @@ -235,7 +235,7 @@ def _compare_bfs_spc(G, Gnx, source): # ============================================================================= # Tests # ============================================================================= -@pytest.mark.parametrize("graph_file", utils.DATASETS_5) +@pytest.mark.parametrize("graph_file", utils.DATASETS) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("seed", SUBSET_SEED_OPTIONS) def test_bfs(graph_file, directed, seed): @@ -257,7 +257,7 @@ def test_bfs_spc(graph_file, directed, seed): ) -@pytest.mark.parametrize("graph_file", utils.TINY_DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) def test_bfs_spc_full(graph_file, directed): """Test BFS traversal on every vertex with shortest path counting""" diff --git a/python/cugraph/tests/test_edge_betweenness_centrality.py b/python/cugraph/tests/test_edge_betweenness_centrality.py index c32f58f1156..e23fdc210ff 100644 --- a/python/cugraph/tests/test_edge_betweenness_centrality.py +++ b/python/cugraph/tests/test_edge_betweenness_centrality.py @@ -42,10 +42,6 @@ NORMALIZED_OPTIONS = [False, True] DEFAULT_EPSILON = 0.0001 -DATASETS = ["../datasets/karate.csv", "../datasets/netscience.csv"] - -UNRENUMBERED_DATASETS = ["../datasets/karate.csv"] - SUBSET_SIZE_OPTIONS = [4, None] SUBSET_SEED_OPTIONS = [42] @@ -297,7 +293,7 @@ def prepare_test(): gc.collect() -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -326,7 +322,7 @@ def test_edge_betweenness_centrality( compare_scores(sorted_df, first_key="cu_bc", second_key="ref_bc") -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", [None]) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -364,7 +360,7 @@ def test_edge_betweenness_centrality_k_full( # the function operating the comparison inside is first proceeding # to a random sampling over the number of vertices (thus direct offsets) # in the graph structure instead of actual vertices identifiers -@pytest.mark.parametrize("graph_file", UNRENUMBERED_DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_UNRENUMBERED) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -397,7 +393,7 @@ def test_edge_betweenness_centrality_fixed_sample( compare_scores(sorted_df, first_key="cu_bc", second_key="ref_bc") -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @@ -432,7 +428,7 @@ def test_edge_betweenness_centrality_weight_except( compare_scores(sorted_df, first_key="cu_bc", second_key="ref_bc") -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) diff --git a/python/cugraph/tests/test_graph.py b/python/cugraph/tests/test_graph.py index 7a87d6202b0..10243a0ffee 100644 --- a/python/cugraph/tests/test_graph.py +++ b/python/cugraph/tests/test_graph.py @@ -479,7 +479,7 @@ def test_consolidation(graph_file): # Test -@pytest.mark.parametrize('graph_file', utils.TINY_DATASETS) +@pytest.mark.parametrize('graph_file', utils.DATASETS_SMALL) def test_two_hop_neighbors(graph_file): gc.collect() @@ -622,7 +622,7 @@ def test_to_directed(graph_file): # Test -@pytest.mark.parametrize("graph_file", utils.TINY_DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) def test_to_undirected(graph_file): gc.collect() diff --git a/python/cugraph/tests/test_hits.py b/python/cugraph/tests/test_hits.py index 77471f57601..5778fcebe72 100644 --- a/python/cugraph/tests/test_hits.py +++ b/python/cugraph/tests/test_hits.py @@ -86,8 +86,6 @@ def networkx_call(M, max_iter, tol): return pr -DATASETS = ["../datasets/dolphins.csv", "../datasets/karate.csv"] - MAX_ITERATIONS = [50] TOLERANCE = [1.0e-06] @@ -95,7 +93,7 @@ def networkx_call(M, max_iter, tol): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize("graph_file", DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) @pytest.mark.parametrize("max_iter", MAX_ITERATIONS) @pytest.mark.parametrize("tol", TOLERANCE) def test_hits(graph_file, max_iter, tol): diff --git a/python/cugraph/tests/utils.py b/python/cugraph/tests/utils.py index e00be028351..8583bea054d 100644 --- a/python/cugraph/tests/utils.py +++ b/python/cugraph/tests/utils.py @@ -20,27 +20,13 @@ from cugraph.dask.common.mg_utils import (get_client) # -# Datasets are numbered based on the number of elements in the array +# Datasets # -# DATASETS_1 = ['../datasets/netscience.csv'] - DATASETS_UNDIRECTED = ['../datasets/karate.csv', '../datasets/dolphins.csv'] +DATASETS_UNRENUMBERED = ['../datasets/karate-disjoint.csv'] DATASETS = ['../datasets/karate.csv', '../datasets/karate-disjoint.csv', - '../datasets/dolphins.csv', - '../datasets/netscience.csv', - '../datasets/email-Eu-core.csv'] - -# FIXME: netscience.csv causes NetworkX pagerank to throw an exception. -# (networkx/algorithms/link_analysis/pagerank_alg.py:152: KeyError: 1532) -# DATASETS_4 = ['../datasets/karate.csv', -# '../datasets/karate-disjoint.csv', -# '../datasets/dolphins.csv', -# '../datasets/netscience.csv', -# '../datasets/email-Eu-core.csv'] - -DATASETS_5 = ['../datasets/karate.csv', '../datasets/dolphins.csv', '../datasets/polbooks.csv', '../datasets/netscience.csv', @@ -55,16 +41,10 @@ ('../datasets/netscience.csv', '../datasets/ref/ktruss/netscience.csv')] -TINY_DATASETS = ['../datasets/karate.csv', +DATASETS_SMALL = ['../datasets/karate.csv', '../datasets/dolphins.csv', '../datasets/polbooks.csv'] -# SMALL_DATASETS = ['../datasets/netscience.csv', '../datasets/email-Eu-core.csv'] - -UNRENUMBERED_DATASETS = ['../datasets/karate-disjoint.csv'] - - - def read_csv_for_nx(csv_file, read_weights_in_sp=True): print('Reading ' + str(csv_file) + '...') From 19cc333885d5818c31b09fe5dff90d6d7f5717ec Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:19:21 -0400 Subject: [PATCH 31/77] change log --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616bcda886e..379110c3620 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,9 @@ ## New Features ## Improvements -- PR 1081 MNMG Renumbering - sort partitions by degree -- PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource +- PR #1081 MNMG Renumbering - sort partitions by degree +- PR #1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource +- PR #1129 Refactored test to use common datasets and added additional documentation pages ## Bug Fixes From 124c07417d7e125f64f93fbc35744cafa8069624 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:24:34 -0400 Subject: [PATCH 32/77] fixed --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 379110c3620..92bda52afb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,8 @@ ## New Features ## Improvements -- PR #1081 MNMG Renumbering - sort partitions by degree -- PR #1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource +- PR 1081 MNMG Renumbering - sort partitions by degree +- PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource - PR #1129 Refactored test to use common datasets and added additional documentation pages ## Bug Fixes From a0d5156b1ea40010f7045a49f3aacfe5d58811d4 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:29:45 -0400 Subject: [PATCH 33/77] dropped changelog entry --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92bda52afb9..616bcda886e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource -- PR #1129 Refactored test to use common datasets and added additional documentation pages ## Bug Fixes From 4aab511f967b006376f0fca071ec9444a4ecabd1 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:31:51 -0400 Subject: [PATCH 34/77] updated changelog for pr 1129 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616bcda886e..06249cdf25c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource +- PR 1129 Refactored test to use common dataset and added additional doc pages ## Bug Fixes From 9834defa5c22645b9fe62e58a6cb41711b2e50fb Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:43:31 -0400 Subject: [PATCH 35/77] change dataset type to speed to tests --- python/cugraph/tests/test_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cugraph/tests/test_graph.py b/python/cugraph/tests/test_graph.py index 10243a0ffee..44c856cf3dc 100644 --- a/python/cugraph/tests/test_graph.py +++ b/python/cugraph/tests/test_graph.py @@ -593,7 +593,7 @@ def test_number_of_vertices(graph_file): # Test -@pytest.mark.parametrize("graph_file", utils.DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) def test_to_directed(graph_file): gc.collect() From a118ab117095b0e86a01f7cba2c589e8365b14d7 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:44:30 -0400 Subject: [PATCH 36/77] fixed typos --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06249cdf25c..e824090e66c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource -- PR 1129 Refactored test to use common dataset and added additional doc pages +- PR 1129 Refactored tests to use common dataset and added additional doc pages ## Bug Fixes From e4291ce022439dd887a0d37d6247e45840f58418 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:46:04 -0400 Subject: [PATCH 37/77] reset --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e824090e66c..616bcda886e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource -- PR 1129 Refactored tests to use common dataset and added additional doc pages ## Bug Fixes From ba2e92981fd6d8ce8f9e1cf5993b509d217c5062 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 11:48:22 -0400 Subject: [PATCH 38/77] one more try --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616bcda886e..28389e11861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource - +- PR Refactored test to use common dataset and added additional doc pages +- ## Bug Fixes # cuGraph 0.15.0 (26 Aug 2020) From 3b648c15b3cbdd521d4ea2e757393e6e27ff93ee Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Mon, 31 Aug 2020 14:12:19 -0400 Subject: [PATCH 39/77] addressing code review --- README.md | 2 +- docs/source/cugraph_ref.rst | 2 +- python/cugraph/tests/test_connectivity.py | 23 ++++++++++++----------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 45405d902bf..a51b9fb4e0c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The [RAPIDS](https://rapids.ai) cuGraph library is a collection of GPU accelerat For more project details, see [rapids.ai](https://rapids.ai/). -**NOTE:** For the latest stable [README.md](https://github.com/rapidsai/cudf/blob/main/README.md) ensure you are on the latest branch. +**NOTE:** For the latest stable [README.md](https://github.com/rapidsai/cugraph/blob/main/README.md) ensure you are on the latest branch. diff --git a/docs/source/cugraph_ref.rst b/docs/source/cugraph_ref.rst index 4c627c4c245..0742d5f5aa7 100644 --- a/docs/source/cugraph_ref.rst +++ b/docs/source/cugraph_ref.rst @@ -27,7 +27,7 @@ Data Sets ************** karate - - W. W. Zachary, *Aninformation flow model for conflict and fission in small groups*, Journal of Anthropological Research 33, 452-473 (1977). + - W. W. Zachary, *An information flow model for conflict and fission in small groups*, Journal of Anthropological Research 33, 452-473 (1977). dolphins - D. Lusseau, K. Schneider, O. J. Boisseau, P. Haase, E. Slooten, and S. M. Dawson, *The bottlenose dolphin community of Doubtful Sound features a large proportion of long-lasting associations*, diff --git a/python/cugraph/tests/test_connectivity.py b/python/cugraph/tests/test_connectivity.py index 84057e54499..6d84e43ee4d 100644 --- a/python/cugraph/tests/test_connectivity.py +++ b/python/cugraph/tests/test_connectivity.py @@ -107,6 +107,14 @@ def cugraph_strong_call(cu_M): label_vertex_dict[df["labels"][i]].append(df["vertices"][i]) return label_vertex_dict +def which_cluster_idx(_cluster, _find_vertex): + idx = -1 + for i in range(len(_cluster)): + if _find_vertex in _cluster[i]: + idx = i + break + return idx + # Test all combinations of default/managed and pooled/non-pooled allocation @pytest.mark.parametrize("graph_file", utils.DATASETS) @@ -144,12 +152,8 @@ def test_weak_cc(graph_file): nx_vertices = sorted(lst_nx_components[0]) first_vert = nx_vertices[0] - idx = -1 - for i in range(len(lst_cg_components)): - if first_vert in lst_cg_components[i]: - idx = i - - assert idx != -1 + idx = which_cluster_idx(lst_cg_components, first_vert) + assert idx != -1, "Check for Nx vertex in cuGraph results failed" cg_vertices = sorted(lst_cg_components[idx]) @@ -194,11 +198,8 @@ def test_strong_cc(graph_file): nx_vertices = sorted(lst_nx_components[0]) first_vert = nx_vertices[0] - idx = -1 - for i in range(len(lst_cg_components)): - if first_vert in lst_cg_components[i]: - idx = i + idx = which_cluster_idx(lst_cg_components, first_vert) + assert idx != -1, "Check for Nx vertex in cuGraph results failed" - assert idx != -1 cg_vertices = sorted(lst_cg_components[idx]) assert nx_vertices == cg_vertices From 9628a1b9487203a9e6570ecca5b1c9845432d7e0 Mon Sep 17 00:00:00 2001 From: Brad Rees <34135411+BradReesWork@users.noreply.github.com> Date: Mon, 31 Aug 2020 16:07:37 -0400 Subject: [PATCH 40/77] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28389e11861..622cb213b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource -- PR Refactored test to use common dataset and added additional doc pages +- PR 1129 Refactored test to use common dataset and added additional doc pages - ## Bug Fixes From b72837c7391ee683da578713badf0124328b66df Mon Sep 17 00:00:00 2001 From: Brad Rees <34135411+BradReesWork@users.noreply.github.com> Date: Mon, 31 Aug 2020 16:10:29 -0400 Subject: [PATCH 41/77] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 622cb213b40..eb4fd19ed7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ## Improvements - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource -- PR 1129 Refactored test to use common dataset and added additional doc pages +- PR #1129 Refactored test to use common dataset and added additional doc pages - ## Bug Fixes From dbff65a6dac3e64244a4d7e6528920d63d637756 Mon Sep 17 00:00:00 2001 From: Brad Rees <34135411+BradReesWork@users.noreply.github.com> Date: Mon, 31 Aug 2020 16:24:32 -0400 Subject: [PATCH 42/77] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb4fd19ed7e..82dfb678bbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - PR 1081 MNMG Renumbering - sort partitions by degree - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource - PR #1129 Refactored test to use common dataset and added additional doc pages -- + ## Bug Fixes # cuGraph 0.15.0 (26 Aug 2020) From b852bb88b423145d4ad134ce8cacd44acd67ceb2 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Tue, 1 Sep 2020 13:12:06 -0400 Subject: [PATCH 43/77] flake8 errors --- python/cugraph/structure/graph.py | 1 - .../cugraph/tests/test_betweenness_centrality.py | 2 +- python/cugraph/tests/test_connectivity.py | 4 +++- python/cugraph/tests/utils.py | 14 +++++++------- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/python/cugraph/structure/graph.py b/python/cugraph/structure/graph.py index a9e84f2c038..f579a32cab6 100644 --- a/python/cugraph/structure/graph.py +++ b/python/cugraph/structure/graph.py @@ -931,7 +931,6 @@ def out_degree(self, vertex_subset=None): raise Exception("Not supported for distributed graph") return self._degree(vertex_subset, x=2) - def degree(self, vertex_subset=None): """ Compute vertex degree. By default, this method computes vertex diff --git a/python/cugraph/tests/test_betweenness_centrality.py b/python/cugraph/tests/test_betweenness_centrality.py index 13b8ab10397..1ef1601edd5 100644 --- a/python/cugraph/tests/test_betweenness_centrality.py +++ b/python/cugraph/tests/test_betweenness_centrality.py @@ -367,7 +367,7 @@ def test_betweenness_centrality_k_full( # the function operating the comparison inside is first proceeding # to a random sampling over the number of vertices (thus direct offsets) # in the graph structure instead of actual vertices identifiers -@pytest.mark.parametrize("graph_file", utils.DATASETS_UNRENUMBERED ) +@pytest.mark.parametrize("graph_file", utils.DATASETS_UNRENUMBERED) @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("subset_size", SUBSET_SIZE_OPTIONS) @pytest.mark.parametrize("normalized", NORMALIZED_OPTIONS) diff --git a/python/cugraph/tests/test_connectivity.py b/python/cugraph/tests/test_connectivity.py index 6d84e43ee4d..508be9bb58d 100644 --- a/python/cugraph/tests/test_connectivity.py +++ b/python/cugraph/tests/test_connectivity.py @@ -107,6 +107,7 @@ def cugraph_strong_call(cu_M): label_vertex_dict[df["labels"][i]].append(df["vertices"][i]) return label_vertex_dict + def which_cluster_idx(_cluster, _find_vertex): idx = -1 for i in range(len(_cluster)): @@ -194,7 +195,8 @@ def test_strong_cc(graph_file): # Compare lengths of each component assert lst_nx_components_lens == lst_cg_components_lens - # Compare vertices of largest component - note that there might be more than one largest component + # Compare vertices of largest component + # note that there might be more than one largest component nx_vertices = sorted(lst_nx_components[0]) first_vert = nx_vertices[0] diff --git a/python/cugraph/tests/utils.py b/python/cugraph/tests/utils.py index 8583bea054d..3b823348bc8 100644 --- a/python/cugraph/tests/utils.py +++ b/python/cugraph/tests/utils.py @@ -26,11 +26,11 @@ DATASETS_UNRENUMBERED = ['../datasets/karate-disjoint.csv'] DATASETS = ['../datasets/karate.csv', - '../datasets/karate-disjoint.csv', - '../datasets/dolphins.csv', - '../datasets/polbooks.csv', - '../datasets/netscience.csv', - '../datasets/email-Eu-core.csv'] + '../datasets/karate-disjoint.csv', + '../datasets/dolphins.csv', + '../datasets/polbooks.csv', + '../datasets/netscience.csv', + '../datasets/email-Eu-core.csv'] STRONGDATASETS = ['../datasets/dolphins.csv', '../datasets/netscience.csv', @@ -42,8 +42,8 @@ '../datasets/ref/ktruss/netscience.csv')] DATASETS_SMALL = ['../datasets/karate.csv', - '../datasets/dolphins.csv', - '../datasets/polbooks.csv'] + '../datasets/dolphins.csv', + '../datasets/polbooks.csv'] def read_csv_for_nx(csv_file, read_weights_in_sp=True): From 88b9896506d8bcdf5410e3bc7ae1580d3515ef35 Mon Sep 17 00:00:00 2001 From: Raymond Douglass Date: Tue, 1 Sep 2020 13:18:14 -0400 Subject: [PATCH 44/77] FIX set +e on style check script --- ci/checks/style.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/checks/style.sh b/ci/checks/style.sh index 696f566a96a..a673074e141 100755 --- a/ci/checks/style.sh +++ b/ci/checks/style.sh @@ -11,6 +11,7 @@ # captured for returning a final status code. This allows all style check to # take place to provide a more comprehensive list of style violations. set -o pipefail +set +e ERRORCODE=0 PATH=/conda/bin:$PATH From 86edcaed49f2845ed6055cb45b2977464938da39 Mon Sep 17 00:00:00 2001 From: Raymond Douglass Date: Tue, 1 Sep 2020 13:24:37 -0400 Subject: [PATCH 45/77] DOC Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 616bcda886e..28000de4e77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - PR 1115 Replace deprecated rmm::mr::get_default_resource with rmm::mr::get_current_device_resource ## Bug Fixes +- PR #1131 Show style checker errors with set +e # cuGraph 0.15.0 (26 Aug 2020) From 4a684b34a7d182447e110d209f35de0bea17bafe Mon Sep 17 00:00:00 2001 From: Raymond Douglass Date: Tue, 1 Sep 2020 13:43:26 -0400 Subject: [PATCH 46/77] DOC Document why set +x is needed --- ci/checks/style.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/checks/style.sh b/ci/checks/style.sh index a673074e141..978ac03d85b 100755 --- a/ci/checks/style.sh +++ b/ci/checks/style.sh @@ -11,6 +11,8 @@ # captured for returning a final status code. This allows all style check to # take place to provide a more comprehensive list of style violations. set -o pipefail +# CI does `set -e` then sources this file, so we override that so we can output +# the results from the various style checkers set +e ERRORCODE=0 PATH=/conda/bin:$PATH From dcd10448817dac7fb41181414ae026495a462a81 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 14:51:05 -0400 Subject: [PATCH 47/77] minor fixes --- cpp/tests/experimental/graph_test.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp index f8699799e8a..fbd6b3674c6 100644 --- a/cpp/tests/experimental/graph_test.cpp +++ b/cpp/tests/experimental/graph_test.cpp @@ -15,10 +15,10 @@ */ #include +#include #include #include -#include #include #include @@ -65,12 +65,11 @@ std::tuple, std::vector, std::vector> gr } typedef struct Graph_Usecase_t { - std::string graph_file_path; - std::string graph_file_full_path; - bool test_weighted; + std::string graph_file_full_path{}; + bool test_weighted{false}; Graph_Usecase_t(std::string const& graph_file_path, bool test_weighted) - : graph_file_path(graph_file_path), test_weighted(test_weighted) + : test_weighted(test_weighted) { if ((graph_file_path.length() > 0) && (graph_file_path[0] != '/')) { graph_file_full_path = cugraph::test::get_rapids_dataset_root_dir() + "/" + graph_file_path; From 778dc44f5c6cad8afe860ce8b8e6d661d3828b97 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 14:51:42 -0400 Subject: [PATCH 48/77] address reviewer comments --- .../experimental/detail/graph_utils.cuh | 19 +++--- cpp/include/experimental/graph.hpp | 1 + cpp/include/experimental/graph_view.hpp | 65 +++++++++---------- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index a1ab473a6ee..0747f996850 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -39,12 +39,12 @@ rmm::device_uvector compute_major_degree( std::vector const &adj_matrix_partition_offsets, partition_t const &partition) { - auto &comm_p_row = handle.get_subcomm(comm_p_row_key); - auto comm_p_row_rank = comm_p_row.get_rank(); - auto comm_p_row_size = comm_p_row.get_size(); - auto &comm_p_col = handle.get_subcomm(comm_p_col_key); - auto comm_p_col_rank = comm_p_col.get_rank(); - auto comm_p_col_size = comm_p_col.get_size(); + auto &comm_p_row = handle.get_subcomm(comm_p_row_key); + auto const comm_p_row_rank = comm_p_row.get_rank(); + auto const comm_p_row_size = comm_p_row.get_size(); + auto &comm_p_col = handle.get_subcomm(comm_p_col_key); + auto const comm_p_col_rank = comm_p_col.get_rank(); + auto const comm_p_col_size = comm_p_col.get_size(); rmm::device_uvector local_degrees(0, handle.get_stream()); rmm::device_uvector degrees(0, handle.get_stream()); @@ -97,9 +97,10 @@ rmm::device_uvector compute_major_degree( { // we can avoid creating this temporary with "if constexpr" supported from C++17 std::vector tmp_offsets(adj_matrix_partition_offsets.size(), nullptr); - for (size_t i = 0; i < adj_matrix_partition_offsets.size(); ++i) { - tmp_offsets[i] = adj_matrix_partition_offsets[i].data(); - } + std::transform(adj_matrix_partition_offsets.begin(), + adj_matrix_partition_offsets.end(), + tmp_offsets.begin(), + [](auto const &offsets) { return offsets.data(); }); return compute_major_degree(handle, tmp_offsets, partition); } diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index 74a7dc1f435..70922bad420 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -37,6 +37,7 @@ struct edgelist_t { edge_t number_of_edges{0}; }; +// graph_t is an owning graph class (note that graph_view_t is a non-owning graph class) template Date: Tue, 1 Sep 2020 15:08:14 -0400 Subject: [PATCH 49/77] update comments on currently public (but will become private in the future) APIs --- cpp/include/experimental/graph_view.hpp | 39 +++++++++++++++---------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index 46c5748468f..53e66a78832 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -177,25 +177,28 @@ class graph_view_tget_number_of_vertices(); } - // Better avoid direct invocation in application code. - // This is mainly for pattern accelerator implementation. + // FIXME: this function is not part of the public stable API.This function is mainly for pattern + // accelerator implementation. This function is currently public to support the legacy + // implementations directly accessing CSR/CSC data, but this function will eventually become + // private. edge_t const* offsets() const { return offsets_; } - // Better avoid direct invocation in application code. - // This is mainly for pattern accelerator implementation. + // FIXME: this function is not part of the public stable API.This function is mainly for pattern + // accelerator implementation. This function is currently public to support the legacy + // implementations directly accessing CSR/CSC data, but this function will eventually become + // private. vertex_t const* indices() const { return indices_; } - // Better avoid direct invocation in application code. - // This is mainly for pattern accelerator implementation. + // FIXME: this function is not part of the public stable API.This function is mainly for pattern + // accelerator implementation. This function is currently public to support the legacy + // implementations directly accessing CSR/CSC data, but this function will eventually become + // private. weight_t const* weights() const { return weights_; } private: From 474941fefdad0a36e290343b3f1309ab479f7cc5 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 15:11:08 -0400 Subject: [PATCH 50/77] fix spelling errors --- cpp/src/experimental/graph.cu | 14 +++++++------- cpp/src/experimental/graph_view.cu | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 87ea812ce46..5b8aa5089a2 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -247,7 +247,7 @@ graph_t(comm_p_size), "Invalid API parameter: erroneous partition.vertex_partition_offsets.size()."); - // optinoal expensive checks (part 1/3) + // optional expensive checks (part 1/3) if (do_expensive_check) { edge_t number_of_local_edges_sum{}; @@ -325,7 +325,7 @@ graph_tget_handle_ptr()), adj_matrix_partition_offsets_, partition_); - // optinoal expensive checks (part 2/3) + // optional expensive checks (part 2/3) if (do_expensive_check) { CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), @@ -373,7 +373,7 @@ graph_t{offsets_.data()}); - // optinoal expensive checks (part 2/2) + // optional expensive checks (part 2/2) if (do_expensive_check) { CUGRAPH_EXPECTS(thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), @@ -500,7 +500,7 @@ graph_t; template class graph_t; } // namespace experimental -} // namespace cugraph \ No newline at end of file +} // namespace cugraph diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index a38a9ef06ba..d426f71fef6 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -108,7 +108,7 @@ graph_view_t(comm_p_size), "Invalid API parameter: erroneous partition.vertex_partition_offsets.size()."); - // optinoal expensive checks + // optional expensive checks if (do_expensive_check) { auto default_stream = this->get_handle_ptr()->get_stream(); @@ -240,7 +240,7 @@ graph_view_tget_handle_ptr()->get_stream(); @@ -313,4 +313,4 @@ template class graph_view_t; template class graph_view_t; } // namespace experimental -} // namespace cugraph \ No newline at end of file +} // namespace cugraph From 99f91c3e7d01f4eb742226ba6c98b6cfae8f1c90 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 15:30:02 -0400 Subject: [PATCH 51/77] clang-format --- cpp/include/experimental/detail/graph_utils.cuh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index 0747f996850..c3da891afab 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -54,9 +54,9 @@ rmm::device_uvector compute_major_degree( auto vertex_partition_id = partition.hypergraph_partitioned ? comm_p_row_size * i + comm_p_row_rank : comm_p_col_size * comm_p_row_rank + i; - auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; - max_num_local_degrees = std::max(max_num_local_degrees, major_last - major_first); + auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + max_num_local_degrees = std::max(max_num_local_degrees, major_last - major_first); if (i == comm_p_col_rank) { degrees.resize(major_last - major_first, handle.get_stream()); } } local_degrees.resize(max_num_local_degrees, handle.get_stream()); @@ -64,8 +64,8 @@ rmm::device_uvector compute_major_degree( auto vertex_partition_id = partition.hypergraph_partitioned ? comm_p_row_size * i + comm_p_row_rank : comm_p_col_size * comm_p_row_rank + i; - auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; + auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; + auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; auto p_offsets = partition.hypergraph_partitioned ? adj_matrix_partition_offsets[i] From c52257d3ce84c889691d1594f6a859984eae51bb Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 21:14:51 -0400 Subject: [PATCH 52/77] struct_t partition_t to class_t partition_t --- .../experimental/detail/graph_utils.cuh | 54 ++++----- cpp/include/experimental/graph.hpp | 7 +- cpp/include/experimental/graph_view.hpp | 103 +++++++++++++++++- cpp/src/experimental/graph.cu | 76 ++++++------- cpp/src/experimental/graph_view.cu | 42 +++---- 5 files changed, 175 insertions(+), 107 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index c3da891afab..a3e9b95ed39 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -51,26 +51,34 @@ rmm::device_uvector compute_major_degree( vertex_t max_num_local_degrees{0}; for (int i = 0; i < comm_p_col_size; ++i) { - auto vertex_partition_id = partition.hypergraph_partitioned - ? comm_p_row_size * i + comm_p_row_rank - : comm_p_col_size * comm_p_row_rank + i; - auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; - max_num_local_degrees = std::max(max_num_local_degrees, major_last - major_first); + auto vertex_partition_idx = + partition.is_hypergraph_partitioned() + ? static_cast(comm_p_row_size) * static_cast(i) + + static_cast(comm_p_row_rank) + : static_cast(comm_p_col_size) * static_cast(comm_p_row_rank) + + static_cast(i); + vertex_t major_first{}; + vertex_t major_last{}; + std::tie(major_first, major_last) = partition.get_vertex_partition_range(vertex_partition_idx); + max_num_local_degrees = std::max(max_num_local_degrees, major_last - major_first); if (i == comm_p_col_rank) { degrees.resize(major_last - major_first, handle.get_stream()); } } local_degrees.resize(max_num_local_degrees, handle.get_stream()); for (int i = 0; i < comm_p_col_size; ++i) { - auto vertex_partition_id = partition.hypergraph_partitioned - ? comm_p_row_size * i + comm_p_row_rank - : comm_p_col_size * comm_p_row_rank + i; - auto major_first = partition.vertex_partition_offsets[vertex_partition_id]; - auto major_last = partition.vertex_partition_offsets[vertex_partition_id + 1]; - auto p_offsets = - partition.hypergraph_partitioned - ? adj_matrix_partition_offsets[i] - : adj_matrix_partition_offsets[0] + - (major_first - partition.vertex_partition_offsets[comm_p_col_size * comm_p_row_rank]); + auto vertex_partition_idx = + partition.is_hypergraph_partitioned() + ? static_cast(comm_p_row_size) * static_cast(i) + + static_cast(comm_p_row_rank) + : static_cast(comm_p_col_size) * static_cast(comm_p_row_rank) + + static_cast(i); + vertex_t major_first{}; + vertex_t major_last{}; + std::tie(major_first, major_last) = partition.get_vertex_partition_range(vertex_partition_idx); + auto p_offsets = partition.is_hypergraph_partitioned() + ? adj_matrix_partition_offsets[i] + : adj_matrix_partition_offsets[0] + + (major_first - partition.get_vertex_partition_range_first( + comm_p_col_size * comm_p_row_rank)); thrust::transform(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), thrust::make_counting_iterator(vertex_t{0}), thrust::make_counting_iterator(major_last - major_first), @@ -104,20 +112,6 @@ rmm::device_uvector compute_major_degree( return compute_major_degree(handle, tmp_offsets, partition); } -template -void check_vertex_partition_offsets(std::vector const &vertex_partition_offsets, - vertex_t number_of_vertices) -{ - CUGRAPH_EXPECTS( - std::is_sorted(vertex_partition_offsets.begin(), vertex_partition_offsets.end()), - "Invalid API parameter: partition.vertex_partition_offsets values should be non-descending."); - CUGRAPH_EXPECTS(vertex_partition_offsets[0] == vertex_t{0}, - "Invalid API parameter: partition.vertex_partition_offsets[0] should be 0."); - CUGRAPH_EXPECTS(vertex_partition_offsets.back() == number_of_vertices, - "Invalid API parameter: partition.vertex_partition_offsets.back() should be " - "number_of_vertices."); -} - template struct degree_from_offsets_t { edge_t const *offsets{nullptr}; diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index 70922bad420..d014125acea 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -74,9 +74,10 @@ class graph_tget_handle_ptr()->get_comms().get_rank(); - return partition_.vertex_partition_offsets[comm_p_rank + 1] - - partition_.vertex_partition_offsets[comm_p_rank]; + vertex_t first{}; + vertex_t last{}; + std::tie(first, last) = partition_.get_vertex_partition_range(); + return last - first; } graph_view_t view() diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index 53e66a78832..964cad024b6 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -20,8 +20,10 @@ #include #include +#include #include #include +#include #include #include @@ -71,10 +73,99 @@ std::string const comm_p_col_key = "comm_p_key"; * @tparam vertex_t Type of vertex ID */ template -struct partition_t { - std::vector vertex_partition_offsets{}; // size = P + 1 - bool hypergraph_partitioned{false}; -}; +class partition_t { + public: + partition_t(std::vector const& vertex_partition_offsets, + bool hypergraph_partitioned, + int comm_p_row_size, + int comm_p_col_size, + int comm_p_row_rank, + int comm_p_col_rank) + : vertex_partition_offsets_(vertex_partition_offsets), + hypergraph_partitioned_(hypergraph_partitioned), + comm_p_rank_(comm_p_col_size * comm_p_row_rank + comm_p_col_rank), + comm_p_row_size_(comm_p_row_size), + comm_p_col_size_(comm_p_col_size), + comm_p_row_rank_(comm_p_row_rank), + comm_p_col_rank_(comm_p_col_rank) + { + CUGRAPH_EXPECTS( + vertex_partition_offsets.size() == static_cast(comm_p_row_size * comm_p_col_size), + "Invalid API parameter: erroneous vertex_partition_offsets.size()."); + + CUGRAPH_EXPECTS( + std::is_sorted(vertex_partition_offsets_.begin(), vertex_partition_offsets_.end()), + "Invalid API parameter: partition.vertex_partition_offsets values should be non-descending."); + CUGRAPH_EXPECTS(vertex_partition_offsets_[0] == vertex_t{0}, + "Invalid API parameter: partition.vertex_partition_offsets[0] should be 0."); + } + + std::tuple get_vertex_partition_range() const + { + return std::make_tuple(vertex_partition_offsets_[comm_p_rank_], + vertex_partition_offsets_[comm_p_rank_ + 1]); + } + + vertex_t get_vertex_partition_range_first() const + { + return vertex_partition_offsets_[comm_p_rank_]; + } + + vertex_t get_vertex_partition_range_last() const + { + return vertex_partition_offsets_[comm_p_rank_ + 1]; + } + + std::tuple get_vertex_partition_range(size_t vertex_partition_idx) const + { + return std::make_tuple(vertex_partition_offsets_[vertex_partition_idx], + vertex_partition_offsets_[vertex_partition_idx + 1]); + } + + vertex_t get_vertex_partition_range_first(size_t vertex_partition_idx) const + { + return vertex_partition_offsets_[vertex_partition_idx]; + } + + vertex_t get_vertex_partition_range_last(size_t vertex_partition_idx) const + { + return vertex_partition_offsets_[vertex_partition_idx + 1]; + } + + std::tuple get_matrix_partition_major_range(int partition_idx) const + { + auto major_first = + hypergraph_partitioned_ + ? vertex_partition_offsets_[comm_p_row_size_ * partition_idx + comm_p_row_rank_] + : vertex_partition_offsets_[comm_p_row_rank_ * comm_p_col_size_]; + auto major_last = + hypergraph_partitioned_ + ? vertex_partition_offsets_[comm_p_row_size_ * partition_idx + comm_p_row_rank_ + 1] + : vertex_partition_offsets_[(comm_p_row_rank_ + 1) * comm_p_col_size_]; + + return std::make_tuple(major_first, major_last); + } + + std::tuple get_matrix_partition_minor_range() const + { + auto minor_first = vertex_partition_offsets_[comm_p_col_rank_ * comm_p_row_size_]; + auto minor_last = vertex_partition_offsets_[(comm_p_col_rank_ + 1) * comm_p_row_size_]; + + return std::make_tuple(minor_first, minor_last); + } + + bool is_hypergraph_partitioned() const { return hypergraph_partitioned_; } + + private: + std::vector vertex_partition_offsets_{}; // size = P + 1 + bool hypergraph_partitioned_{false}; + + int comm_p_rank_{0}; + int comm_p_row_size_{0}; + int comm_p_col_size_{0}; + int comm_p_row_rank_{0}; + int comm_p_col_rank_{0}; +}; // namespace experimental namespace detail { @@ -171,8 +262,8 @@ class graph_view_tget_handle_ptr()->get_comms().get_rank(); - return partition_.vertex_partition_offsets[comm_p_rank + 1] - - partition_.vertex_partition_offsets[comm_p_rank]; + return partition_.get_vertex_partition_range_last() - + partition_.get_vertex_partition_range_first(); } size_t get_number_of_adj_matrix_partitions() { return adj_matrix_partition_offsets_.size(); } diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 5b8aa5089a2..07bfbb55339 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -62,14 +62,12 @@ std:: tuple, rmm::device_uvector, rmm::device_uvector> edge_list_to_compressed_sparse(raft::handle_t const &handle, edgelist_t const &edgelist, - vertex_t row_first, - vertex_t row_last, - vertex_t col_first, - vertex_t col_last) + vertex_t major_first, + vertex_t major_last, + vertex_t minor_first, + vertex_t minor_last) { - rmm::device_uvector offsets( - store_transposed ? (row_last - row_first) + 1 : (col_last - col_first) + 1, - handle.get_stream()); + rmm::device_uvector offsets((major_last - major_first) + 1, handle.get_stream()); rmm::device_uvector indices(edgelist.number_of_edges, handle.get_stream()); rmm::device_uvector weights( edgelist.p_edge_weights != nullptr ? edgelist.number_of_edges : 0, handle.get_stream()); @@ -98,7 +96,6 @@ std:: auto p_weights = edgelist.p_edge_weights != nullptr ? weights.data() : static_cast(nullptr); - auto major_first = store_transposed ? row_first : col_first; thrust::for_each(rmm::exec_policy(handle.get_stream())->on(handle.get_stream()), store_transposed ? edgelist.p_dst_vertices : edgelist.p_src_vertices, store_transposed ? edgelist.p_dst_vertices + edgelist.number_of_edges @@ -218,15 +215,15 @@ graph_tget_handle_ptr()->get_comms(); - auto comm_p_size = comm_p.get_size(); - auto &comm_p_row = this->get_handle_ptr()->get_subcomm(comm_p_row_key); - auto comm_p_row_rank = comm_p_row.get_rank(); - auto comm_p_row_size = comm_p_row.get_size(); - auto &comm_p_col = this->get_handle_ptr()->get_subcomm(comm_p_col_key); - auto comm_p_col_rank = comm_p_col.get_rank(); - auto comm_p_col_size = comm_p_col.get_size(); - auto default_stream = this->get_handle_ptr()->get_stream(); + auto &comm_p = this->get_handle_ptr()->get_comms(); + auto const comm_p_size = comm_p.get_size(); + auto &comm_p_row = this->get_handle_ptr()->get_subcomm(comm_p_row_key); + auto const comm_p_row_rank = comm_p_row.get_rank(); + auto const comm_p_row_size = comm_p_row.get_size(); + auto &comm_p_col = this->get_handle_ptr()->get_subcomm(comm_p_col_key); + auto const comm_p_col_rank = comm_p_col.get_rank(); + auto const comm_p_col_size = comm_p_col.get_size(); + auto default_stream = this->get_handle_ptr()->get_stream(); for (size_t i = 0; i < edgelists.size(); ++i) { CUGRAPH_EXPECTS( @@ -239,29 +236,22 @@ graph_t(comm_p_row_size))) || - (!(partition.hypergraph_partitioned) && (edgelists.size() == 1)), + (!(partition.is_hypergraph_partitioned()) && (edgelists.size() == 1)), "Invalid API parameter: errneous edgelists.size()."); - CUGRAPH_EXPECTS(partition.vertex_partition_offsets.size() == static_cast(comm_p_size), - "Invalid API parameter: erroneous partition.vertex_partition_offsets.size()."); - // optional expensive checks (part 1/3) if (do_expensive_check) { edge_t number_of_local_edges_sum{}; for (size_t i = 0; i < edgelists.size(); ++i) { - auto major_first = - partition.hypergraph_partitioned - ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank] - : partition.vertex_partition_offsets[comm_p_row_rank * comm_p_col_size]; - auto major_last = - partition.hypergraph_partitioned - ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank + 1] - : partition.vertex_partition_offsets[(comm_p_row_rank + 1) * comm_p_col_size]; - auto minor_first = partition.vertex_partition_offsets[comm_p_col_rank * comm_p_row_size]; - auto minor_last = partition.vertex_partition_offsets[(comm_p_col_rank + 1) * comm_p_row_size]; + vertex_t major_first{}; + vertex_t major_last{}; + vertex_t minor_first{}; + vertex_t minor_last{}; + std::tie(major_first, major_last) = partition.get_matrix_partition_major_range(i); + std::tie(minor_first, minor_last) = partition.get_matrix_partition_minor_range(); number_of_local_edges_sum += edgelists[i].number_of_edges; @@ -285,8 +275,9 @@ graph_tget_number_of_vertices()); + CUGRAPH_EXPECTS( + partition.get_vertex_partition_range_last(comm_p_size - 1) == number_of_vertices, + "Invalid API parameter: vertex partition should cover [0, number_of_vertices)."); } // convert edge list (COO) to compressed sparse format (CSR or CSC) @@ -299,21 +290,18 @@ graph_t offsets(0, this->get_handle_ptr()->get_stream()); rmm::device_uvector indices(0, this->get_handle_ptr()->get_stream()); rmm::device_uvector weights(0, this->get_handle_ptr()->get_stream()); std::tie(offsets, indices, weights) = edge_list_to_compressed_sparse( - *(this->get_handle_ptr()), edgelists[i], row_first, row_last, col_first, col_last); + *(this->get_handle_ptr()), edgelists[i], major_first, major_last, minor_first, minor_last); adj_matrix_partition_offsets_.push_back(std::move(offsets)); adj_matrix_partition_indices_.push_back(std::move(indices)); if (this->is_weighted()) { adj_matrix_partition_weights_.push_back(std::move(weights)); } diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index d426f71fef6..d238aaf5dfc 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -78,9 +78,9 @@ graph_view_tget_handle_ptr()->get_comms().get_size(); - auto comm_p_row_size = this->get_handle_ptr()->get_subcomm(comm_p_row_key).get_size(); - auto comm_p_col_size = this->get_handle_ptr()->get_subcomm(comm_p_col_key).get_size(); + auto const comm_p_size = this->get_handle_ptr()->get_comms().get_size(); + auto const comm_p_row_size = this->get_handle_ptr()->get_subcomm(comm_p_row_key).get_size(); + auto const comm_p_col_size = this->get_handle_ptr()->get_subcomm(comm_p_col_key).get_size(); CUGRAPH_EXPECTS(adj_matrix_partition_offsets.size() == adj_matrix_partition_indices.size(), "Invalid API parameter: adj_matrix_partition_offsets.size() and " @@ -92,9 +92,9 @@ graph_view_t(comm_p_row_size))) || - (!(partition.hypergraph_partitioned) && (adj_matrix_partition_offsets.size() == 1)), + (!(partition.is_hypergraph_partitioned()) && (adj_matrix_partition_offsets.size() == 1)), "Invalid API parameter: errneous adj_matrix_partition_offsets.size()."); CUGRAPH_EXPECTS((sorted_by_global_degree_within_vertex_partition && @@ -105,29 +105,22 @@ graph_view_t(comm_p_size), - "Invalid API parameter: erroneous partition.vertex_partition_offsets.size()."); - // optional expensive checks if (do_expensive_check) { auto default_stream = this->get_handle_ptr()->get_stream(); - auto comm_p_row_rank = this->get_handle_ptr()->get_subcomm(comm_p_row_key).get_rank(); - auto comm_p_col_rank = this->get_handle_ptr()->get_subcomm(comm_p_col_key).get_rank(); + auto const comm_p_row_rank = this->get_handle_ptr()->get_subcomm(comm_p_row_key).get_rank(); + auto const comm_p_col_rank = this->get_handle_ptr()->get_subcomm(comm_p_col_key).get_rank(); edge_t number_of_local_edges_sum{}; for (size_t i = 0; i < adj_matrix_partition_offsets.size(); ++i) { - auto major_first = - partition.hypergraph_partitioned - ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank] - : partition.vertex_partition_offsets[comm_p_row_rank * comm_p_col_size]; - auto major_last = - partition.hypergraph_partitioned - ? partition.vertex_partition_offsets[comm_p_row_size * i + comm_p_row_rank + 1] - : partition.vertex_partition_offsets[(comm_p_row_rank + 1) * comm_p_col_size]; - auto minor_first = partition.vertex_partition_offsets[comm_p_col_rank * comm_p_row_size]; - auto minor_last = partition.vertex_partition_offsets[(comm_p_col_rank + 1) * comm_p_row_size]; + vertex_t major_first{}; + vertex_t major_last{}; + vertex_t minor_first{}; + vertex_t minor_last{}; + std::tie(major_first, major_last) = partition.get_matrix_partition_major_range(i); + std::tie(minor_first, minor_last) = partition.get_matrix_partition_minor_range(); CUGRAPH_EXPECTS( thrust::is_sorted(rmm::exec_policy(default_stream)->on(default_stream), adj_matrix_partition_offsets[i], @@ -176,19 +169,20 @@ graph_view_tget_number_of_vertices()); + CUGRAPH_EXPECTS( + partition.get_vertex_partition_range_last(comm_p_size - 1) == number_of_vertices, + "Invalid API parameter: vertex partition should cover [0, number_of_vertices)."); // FIXME: check for symmetricity may better be implemetned with transpose(). if (is_symmetric) {} From 4f6e5c3daa646f36995ba6dfa27cfec165f1c232 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 21:34:07 -0400 Subject: [PATCH 53/77] replace for loop with std::any_of --- cpp/src/experimental/graph.cu | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 07bfbb55339..6a9867b9ede 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -225,16 +225,18 @@ graph_tget_handle_ptr()->get_stream(); - for (size_t i = 0; i < edgelists.size(); ++i) { - CUGRAPH_EXPECTS( - (edgelists[i].p_src_vertices != nullptr) && (edgelists[i].p_dst_vertices != nullptr), - "Invalid API parameter: edgelists[].p_src_vertices and edgelists[].p_dst_vertices should " - "not be nullptr."); - CUGRAPH_EXPECTS((is_weighted && (edgelists[i].p_edge_weights != nullptr)) || - (!is_weighted && (edgelists[i].p_edge_weights == nullptr)), - "Invalid API parameter: edgelists[].p_edge_weights should not be nullptr (if " - "is_weighted is true) or should be nullptr (if is_weighted is false)."); - } + CUGRAPH_EXPECTS( + std::any_of(edgelists.begin(), + edgelists.end(), + [is_weighted](auto edgelist) { + return (edgelist.p_src_vertices == nullptr) || + (edgelist.p_dst_vertices == nullptr) || + (is_weighted && (edgelist.p_edge_weights == nullptr)) || + (!is_weighted && (edgelist.p_edge_weights != nullptr)); + }) == false, + "Invalid API parameter: edgelists[].p_src_vertices and edgelists[].p_dst_vertices should not " + "be nullptr and edgelists[].p_edge_weights should not be nullptr (if is_weighted is true) or " + "should be nullptr (if is_weighted is false)."); CUGRAPH_EXPECTS((partition.is_hypergraph_partitioned() && (edgelists.size() == static_cast(comm_p_row_size))) || From 5ea86a5180d9b74d56231cb8fdd75a36b6cb1754 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 22:47:19 -0400 Subject: [PATCH 54/77] misc updates --- cpp/include/experimental/graph.hpp | 6 ++---- cpp/include/experimental/graph_view.hpp | 3 +-- cpp/src/experimental/graph.cu | 3 +++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index d014125acea..7c870789267 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -74,10 +74,8 @@ class graph_t view() diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index 964cad024b6..d90c566e658 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -132,7 +132,7 @@ class partition_t { return vertex_partition_offsets_[vertex_partition_idx + 1]; } - std::tuple get_matrix_partition_major_range(int partition_idx) const + std::tuple get_matrix_partition_major_range(size_t partition_idx) const { auto major_first = hypergraph_partitioned_ @@ -261,7 +261,6 @@ class graph_view_tget_handle_ptr()->get_comms().get_rank(); return partition_.get_vertex_partition_range_last() - partition_.get_vertex_partition_range_first(); } diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 6a9867b9ede..e9e26f87449 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -426,7 +426,10 @@ graph_tget_number_of_vertices(), 0, this->get_number_of_vertices()}) == 0, "Invalid API parameter: edgelist have out-of-range values."); + // FIXME: check for symmetricity may better be implemetned with transpose(). if (is_symmetric) {} + // FIXME: check for duplicate edges may better be implemented after deciding whether to sort + // neighbor list or not. if (!is_multigraph) {} } From 322fcb2a04d89c52a3e231edb350783aaaa553b2 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Tue, 1 Sep 2020 23:27:25 -0400 Subject: [PATCH 55/77] factor out matrix market garph file read --- cpp/tests/experimental/graph_test.cpp | 77 +++++++++++--------------- cpp/tests/utilities/test_utilities.hpp | 60 +++++++++++++++++--- 2 files changed, 84 insertions(+), 53 deletions(-) diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp index fbd6b3674c6..536b5081d69 100644 --- a/cpp/tests/experimental/graph_test.cpp +++ b/cpp/tests/experimental/graph_test.cpp @@ -91,75 +91,64 @@ class Tests_Graph : public ::testing::TestWithParam { template void run_current_test(Graph_Usecase const& configuration) { - MM_typecode mc{}; - vertex_t m{}; - vertex_t k{}; - edge_t nnz{}; - - FILE* file = fopen(configuration.graph_file_full_path.c_str(), "r"); - ASSERT_NE(file, nullptr) << "fopen (" << configuration.graph_file_full_path << ") failure."; - - edge_t tmp_m{}; - edge_t tmp_k{}; - ASSERT_EQ(cugraph::test::mm_properties(file, 1, &mc, &tmp_m, &tmp_k, &nnz), 0) - << "could not read Matrix Market file properties\n"; - m = static_cast(tmp_m); - k = static_cast(tmp_k); - ASSERT_TRUE(mm_is_matrix(mc)); - ASSERT_TRUE(mm_is_coordinate(mc)); - ASSERT_FALSE(mm_is_complex(mc)); - ASSERT_FALSE(mm_is_skew(mc)); - - std::vector h_rows(nnz, vertex_t{0}); - std::vector h_cols(nnz, vertex_t{0}); - std::vector h_weights(nnz, weight_t{0.0}); - - ASSERT_EQ((cugraph::test::mm_to_coo( - file, 1, nnz, h_rows.data(), h_cols.data(), h_weights.data(), nullptr)), - 0) - << "could not read matrix data\n"; - ASSERT_EQ(fclose(file), 0); + auto mm_graph = + cugraph::test::read_edgelist_from_matrix_market_file( + configuration.graph_file_full_path); + edge_t number_of_edges = static_cast(mm_graph.h_rows.size()); std::vector h_reference_offsets{}; std::vector h_reference_indices{}; std::vector h_reference_weights{}; std::tie(h_reference_offsets, h_reference_indices, h_reference_weights) = - graph_reference(h_rows.data(), - h_cols.data(), - configuration.test_weighted ? h_weights.data() : nullptr, - m, - nnz); + graph_reference( + mm_graph.h_rows.data(), + mm_graph.h_cols.data(), + configuration.test_weighted ? mm_graph.h_weights.data() : nullptr, + mm_graph.number_of_vertices, + number_of_edges); raft::handle_t handle{}; - rmm::device_uvector d_rows(nnz, handle.get_stream()); - rmm::device_uvector d_cols(nnz, handle.get_stream()); - rmm::device_uvector d_weights(configuration.test_weighted ? nnz : 0, + rmm::device_uvector d_rows(number_of_edges, handle.get_stream()); + rmm::device_uvector d_cols(number_of_edges, handle.get_stream()); + rmm::device_uvector d_weights(configuration.test_weighted ? number_of_edges : 0, handle.get_stream()); - raft::update_device(d_rows.data(), h_rows.data(), h_rows.size(), handle.get_stream()); - raft::update_device(d_cols.data(), h_cols.data(), h_cols.size(), handle.get_stream()); + raft::update_device( + d_rows.data(), mm_graph.h_rows.data(), number_of_edges, handle.get_stream()); + raft::update_device( + d_cols.data(), mm_graph.h_cols.data(), number_of_edges, handle.get_stream()); if (configuration.test_weighted) { raft::update_device( - d_weights.data(), h_weights.data(), h_weights.size(), handle.get_stream()); + d_weights.data(), mm_graph.h_weights.data(), number_of_edges, handle.get_stream()); } CUDA_TRY(cudaStreamSynchronize(handle.get_stream())); cugraph::experimental::edgelist_t edgelist{ - d_rows.data(), d_cols.data(), configuration.test_weighted ? d_weights.data() : nullptr, nnz}; + d_rows.data(), + d_cols.data(), + configuration.test_weighted ? d_weights.data() : nullptr, + number_of_edges}; CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement auto graph = cugraph::experimental::graph_t( - handle, edgelist, m, mm_is_symmetric(mc), false, configuration.test_weighted, false, true); + handle, + edgelist, + mm_graph.number_of_vertices, + mm_graph.is_symmetric, + false, + configuration.test_weighted, + false, + true); auto graph_view = graph.view(); - ASSERT_EQ(graph_view.get_number_of_vertices(), m); - ASSERT_EQ(graph_view.get_number_of_edges(), nnz); + ASSERT_EQ(graph_view.get_number_of_vertices(), mm_graph.number_of_vertices); + ASSERT_EQ(graph_view.get_number_of_edges(), number_of_edges); CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement @@ -189,7 +178,7 @@ class Tests_Graph : public ::testing::TestWithParam { std::equal(h_reference_offsets.begin(), h_reference_offsets.end(), h_cugraph_offsets.begin())) << "Graph compressed sparse format offsets do not match with the reference values."; ASSERT_EQ(h_reference_weights.size(), h_cugraph_weights.size()); - for (vertex_t i = 0; i < m; ++i) { + for (vertex_t i = 0; i < mm_graph.number_of_vertices; ++i) { auto start = h_reference_offsets[i]; auto degree = h_reference_offsets[i + 1] - start; if (configuration.test_weighted) { diff --git a/cpp/tests/utilities/test_utilities.hpp b/cpp/tests/utilities/test_utilities.hpp index b9aace72f5d..65703e9541d 100644 --- a/cpp/tests/utilities/test_utilities.hpp +++ b/cpp/tests/utilities/test_utilities.hpp @@ -15,6 +15,8 @@ */ #pragma once +#include + #include #include @@ -25,15 +27,7 @@ extern "C" { #include #include - -#define MPICHECK(cmd) \ - { \ - int e = cmd; \ - if (e != MPI_SUCCESS) { \ - printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \ - FAIL(); \ - } \ - } +#include namespace cugraph { namespace test { @@ -334,5 +328,53 @@ static const std::string& get_rapids_dataset_root_dir() return rdrd; } +template +struct edgelist_from_market_matrix_file_t { + std::vector h_rows{}; + std::vector h_cols{}; + std::vector h_weights{}; + vertex_t number_of_vertices{}; + bool is_symmetric{}; +}; + +template +edgelist_from_market_matrix_file_t read_edgelist_from_matrix_market_file( + std::string const& graph_file_full_path) +{ + edgelist_from_market_matrix_file_t ret{}; + + MM_typecode mc{}; + vertex_t m{}; + vertex_t k{}; + edge_t nnz{}; + + FILE* file = fopen(graph_file_full_path.c_str(), "r"); + CUGRAPH_EXPECTS(file != nullptr, "fopen failure."); + + edge_t tmp_m{}; + edge_t tmp_k{}; + auto mm_ret = cugraph::test::mm_properties(file, 1, &mc, &tmp_m, &tmp_k, &nnz); + CUGRAPH_EXPECTS(mm_ret == 0, "could not read Matrix Market file properties."); + m = static_cast(tmp_m); + k = static_cast(tmp_k); + CUGRAPH_EXPECTS(mm_is_matrix(mc) && mm_is_coordinate(mc) && !mm_is_complex(mc) && !mm_is_skew(mc), + "invalid Matrix Market file properties."); + + ret.h_rows.assign(nnz, vertex_t{0}); + ret.h_cols.assign(nnz, vertex_t{0}); + ret.h_weights.assign(nnz, weight_t{0.0}); + ret.number_of_vertices = m; + ret.is_symmetric = mm_is_symmetric(mc); + + mm_ret = cugraph::test::mm_to_coo( + file, 1, nnz, ret.h_rows.data(), ret.h_cols.data(), ret.h_weights.data(), nullptr); + CUGRAPH_EXPECTS(mm_ret == 0, "could not read matrix data"); + + auto file_ret = fclose(file); + CUGRAPH_EXPECTS(file_ret == 0, "fclose failure."); + + return std::move(ret); +} + } // namespace test } // namespace cugraph From 41b8e9df5fd94a09a2ea1d045a4a41ed58803d71 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Wed, 2 Sep 2020 10:07:27 -0400 Subject: [PATCH 56/77] Added HITS and sorted algorithms alphabetically --- docs/source/api.rst | 51 ++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 84a01604d33..538cd8527f4 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -57,34 +57,34 @@ Katz Centrality Community ========= -Leiden +Ensemble clustering for graphs (ECG) +------------------------------------ + +.. automodule:: cugraph.community.ecg + :members: + :undoc-members: + +K-Truss ------- -.. automodule:: cugraph.community.leiden +.. automodule:: cugraph.community.ktruss_subgraph :members: :undoc-members: -Louvain +Leiden ------- -.. automodule:: cugraph.community.louvain +.. automodule:: cugraph.community.leiden :members: :undoc-members: -K-Truss +Louvain ------- -.. automodule:: cugraph.community.ktruss_subgraph +.. automodule:: cugraph.community.louvain :members: :undoc-members: -ECG ---- - -.. automodule:: cugraph.community.ecg - :members: - :undoc-members: - Spectral Clustering ------------------- @@ -117,9 +117,17 @@ Connected Components :members: :undoc-members: + Cores ===== +Core Number +----------- + +.. automodule:: cugraph.cores.core_number + :members: + :undoc-members: + K-Core ------ @@ -127,12 +135,6 @@ K-Core :members: :undoc-members: -Core Number ------------ - -.. automodule:: cugraph.cores.core_number - :members: - :undoc-members: Layout ====== @@ -144,9 +146,17 @@ Force Atlas 2 :members: :undoc-members: + Link Analysis ============= +HITS +--------- + +.. automodule:: cugraph.link_analysis.hits + :members: + :undoc-members: + Pagerank --------- @@ -154,6 +164,7 @@ Pagerank :members: :undoc-members: + Link Prediction =============== @@ -179,6 +190,7 @@ Overlap Coefficient :members: :undoc-members: + Traversal ========= @@ -196,6 +208,7 @@ Single-source-shortest-path :members: :undoc-members: + Utilities ========= From 9e4d81c9cb6c32addcfeee4531565ab18cd464ac Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Wed, 2 Sep 2020 10:40:33 -0400 Subject: [PATCH 57/77] fix doc issues and added additional docs --- python/cugraph/structure/graph.py | 34 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/python/cugraph/structure/graph.py b/python/cugraph/structure/graph.py index f579a32cab6..e7c015b321b 100644 --- a/python/cugraph/structure/graph.py +++ b/python/cugraph/structure/graph.py @@ -767,20 +767,12 @@ def get_two_hop_neighbors(self): Returns ------- df : cudf.DataFrame - df['first'] : cudf.Series + df[first] : cudf.Series the first vertex id of a pair, if an external vertex id is defined by only one column - df['second'] : cudf.Series + df[second] : cudf.Series the second vertex id of a pair, if an external vertex id is defined by only one column - df['*_first'] : cudf.Series - the first vertex id of a pair, column 0 of the external - vertex id will be represented as '0_first', column 1 as - '1_first', etc. - df['*_second'] : cudf.Series - the second vertex id of a pair, column 0 of the external - vertex id will be represented as '0_first', column 1 as - '1_first', etc. """ if self.distributed: raise Exception("Not supported for distributed graph") @@ -874,10 +866,11 @@ def in_degree(self, vertex_subset=None): vertices (vertex_subset) containing the in_degree. The ordering is relative to the adjacency list, or that given by the specified vertex_subset. - df['vertex'] : cudf.Series + + df[vertex] : cudf.Series The vertex IDs (will be identical to vertex_subset if specified). - df['degree'] : cudf.Series + df[degree] : cudf.Series The computed in-degree of the corresponding vertex. Examples @@ -912,10 +905,11 @@ def out_degree(self, vertex_subset=None): vertices (vertex_subset) containing the out_degree. The ordering is relative to the adjacency list, or that given by the specified vertex_subset. - df['vertex'] : cudf.Series + + df[vertex] : cudf.Series The vertex IDs (will be identical to vertex_subset if specified). - df['degree'] : cudf.Series + df[degree] : cudf.Series The computed out-degree of the corresponding vertex. Examples @@ -931,17 +925,19 @@ def out_degree(self, vertex_subset=None): raise Exception("Not supported for distributed graph") return self._degree(vertex_subset, x=2) + def degree(self, vertex_subset=None): """ - Compute vertex degree. By default, this method computes vertex + Compute vertex degree, which is the total number of edges incident + to a vertex (both in and out edges). By default, this method computes degrees for the entire set of vertices. If vertex_subset is provided, - this method optionally filters out all but those listed in + then this method optionally filters out all but those listed in vertex_subset. Parameters ---------- vertex_subset : cudf.Series or iterable container, optional - A container of vertices for displaying corresponding degree. If not + a container of vertices for displaying corresponding degree. If not set, degrees are computed for the entire set of vertices. Returns @@ -951,6 +947,7 @@ def degree(self, vertex_subset=None): vertices (vertex_subset) containing the degree. The ordering is relative to the adjacency list, or that given by the specified vertex_subset. + df['vertex'] : cudf.Series The vertex IDs (will be identical to vertex_subset if specified). @@ -963,7 +960,8 @@ def degree(self, vertex_subset=None): >>> dtype=['int32', 'int32', 'float32'], header=None) >>> G = cugraph.Graph() >>> G.from_cudf_edgelist(M, '0', '1') - >>> df = G.degree([0,9,12]) + >>> all_df = G.degree() + >>> subset_df = G.degree([0,9,12]) """ if self.distributed: From 03d7982f5ceebb875386fe4e23e29472af2b7f47 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Wed, 2 Sep 2020 10:41:50 -0400 Subject: [PATCH 58/77] dropped regular karate to reduce number of tests. Now only the douible karate set is processed --- python/cugraph/tests/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/cugraph/tests/utils.py b/python/cugraph/tests/utils.py index 3b823348bc8..7e33ce0a628 100644 --- a/python/cugraph/tests/utils.py +++ b/python/cugraph/tests/utils.py @@ -25,8 +25,7 @@ DATASETS_UNDIRECTED = ['../datasets/karate.csv', '../datasets/dolphins.csv'] DATASETS_UNRENUMBERED = ['../datasets/karate-disjoint.csv'] -DATASETS = ['../datasets/karate.csv', - '../datasets/karate-disjoint.csv', +DATASETS = ['../datasets/karate-disjoint.csv', '../datasets/dolphins.csv', '../datasets/polbooks.csv', '../datasets/netscience.csv', From 6975d8d4e40e4c62d0d25c2fcd2305c18f9cefe8 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Wed, 2 Sep 2020 10:54:36 -0400 Subject: [PATCH 59/77] removed Renumbering since it is now in Graph. Drop RMAT since it is not implemented --- docs/source/api.rst | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index 538cd8527f4..b194aa0e03c 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -14,12 +14,6 @@ Graph :members: :undoc-members: -Renumbering ------------ - -.. automodule:: cugraph.structure.renumber - :members: - :undoc-members: Symmetrize ---------- @@ -212,10 +206,5 @@ Single-source-shortest-path Utilities ========= -R-mat Graph Generation ----------------------- -.. automodule:: cugraph.utilities.grmat - :members: - :undoc-members: From 0bd7fa8810af1609e30f12ccb23b7db924662d1c Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Wed, 2 Sep 2020 10:54:45 -0400 Subject: [PATCH 60/77] fix docstring errors --- python/cugraph/community/ecg.py | 10 +++++----- python/cugraph/layout/force_atlas2.py | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/python/cugraph/community/ecg.py b/python/cugraph/community/ecg.py index 5030a2475b7..8389516d9af 100644 --- a/python/cugraph/community/ecg.py +++ b/python/cugraph/community/ecg.py @@ -46,21 +46,21 @@ def ecg(input_graph, min_weight=0.05, ensemble_size=16): parts : cudf.DataFrame GPU data frame of size V containing two columns, the vertex id and the partition id it is assigned to. - - df['vertex'] : cudf.Series + + df[vertex] : cudf.Series Contains the vertex identifiers - df['partition'] : cudf.Series + df[partition] : cudf.Series Contains the partition assigned to the vertices Examples -------- - >>> M = cudf.read_csv('datasets/karate.csv', - delimiter = ' ', + >>> M = cudf.read_csv('datasets/karate.csv', delimiter = ' ', dtype=['int32', 'int32', 'float32'], header=None) >>> G = cugraph.Graph() >>> G.from_cudf_edgelist(M, source='0', destination='1', edge_attr='2') >>> parts = cugraph.ecg(G) + """ parts = ecg_wrapper.ecg(input_graph, min_weight, ensemble_size) diff --git a/python/cugraph/layout/force_atlas2.py b/python/cugraph/layout/force_atlas2.py index 4a61e2a345b..5d648262024 100644 --- a/python/cugraph/layout/force_atlas2.py +++ b/python/cugraph/layout/force_atlas2.py @@ -83,6 +83,7 @@ def force_atlas2( callback: GraphBasedDimRedCallback An instance of GraphBasedDimRedCallback class to intercept the internal state of positions while they are being trained. + Example of callback usage: from cugraph.internals import GraphBasedDimRedCallback class CustomCallback(GraphBasedDimRedCallback): From 9274bfe550c706af7fac11043c71190c9d674957 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Wed, 2 Sep 2020 11:47:09 -0400 Subject: [PATCH 61/77] flake8 issues --- python/cugraph/community/ecg.py | 2 +- python/cugraph/layout/force_atlas2.py | 2 +- python/cugraph/structure/graph.py | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/cugraph/community/ecg.py b/python/cugraph/community/ecg.py index 8389516d9af..85d97b50a8e 100644 --- a/python/cugraph/community/ecg.py +++ b/python/cugraph/community/ecg.py @@ -46,7 +46,7 @@ def ecg(input_graph, min_weight=0.05, ensemble_size=16): parts : cudf.DataFrame GPU data frame of size V containing two columns, the vertex id and the partition id it is assigned to. - + df[vertex] : cudf.Series Contains the vertex identifiers df[partition] : cudf.Series diff --git a/python/cugraph/layout/force_atlas2.py b/python/cugraph/layout/force_atlas2.py index 5d648262024..047431b713a 100644 --- a/python/cugraph/layout/force_atlas2.py +++ b/python/cugraph/layout/force_atlas2.py @@ -83,7 +83,7 @@ def force_atlas2( callback: GraphBasedDimRedCallback An instance of GraphBasedDimRedCallback class to intercept the internal state of positions while they are being trained. - + Example of callback usage: from cugraph.internals import GraphBasedDimRedCallback class CustomCallback(GraphBasedDimRedCallback): diff --git a/python/cugraph/structure/graph.py b/python/cugraph/structure/graph.py index e7c015b321b..c918cd44ae2 100644 --- a/python/cugraph/structure/graph.py +++ b/python/cugraph/structure/graph.py @@ -925,7 +925,6 @@ def out_degree(self, vertex_subset=None): raise Exception("Not supported for distributed graph") return self._degree(vertex_subset, x=2) - def degree(self, vertex_subset=None): """ Compute vertex degree, which is the total number of edges incident @@ -947,7 +946,7 @@ def degree(self, vertex_subset=None): vertices (vertex_subset) containing the degree. The ordering is relative to the adjacency list, or that given by the specified vertex_subset. - + df['vertex'] : cudf.Series The vertex IDs (will be identical to vertex_subset if specified). From fe2883ff883ee5c77a3af1420f287a70d090110b Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Wed, 2 Sep 2020 15:38:25 -0400 Subject: [PATCH 62/77] add necessary stream syncs --- .../experimental/detail/graph_utils.cuh | 3 +++ cpp/src/experimental/graph.cu | 21 ++++++++++++++++--- cpp/src/experimental/graph_view.cu | 2 ++ cpp/tests/experimental/graph_test.cpp | 6 ++---- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index a3e9b95ed39..853c17eebb4 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -92,6 +92,9 @@ rmm::device_uvector compute_major_degree( handle.get_stream()); } + handle.sync_stream(handle.get_stream()); // this is neessary as local_degrees will become + // out-of-scope once this function returns. + return degrees; } diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index e9e26f87449..00239662465 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -188,6 +188,11 @@ std::vector segment_degree_sorted_vertex_partition(raft::handle_t cons d_segment_offsets.size(), handle.get_stream()); + CUDA_TRY(cudaStreamSynchronize( + handle.get_stream())); // this is necessary as d_segment_offsets will become out-of-scope once + // this functions and returning a host variable which can be used right + // after return. + return h_segment_offsets; } @@ -299,9 +304,9 @@ graph_t offsets(0, this->get_handle_ptr()->get_stream()); - rmm::device_uvector indices(0, this->get_handle_ptr()->get_stream()); - rmm::device_uvector weights(0, this->get_handle_ptr()->get_stream()); + rmm::device_uvector offsets(0, default_stream); + rmm::device_uvector indices(0, default_stream); + rmm::device_uvector weights(0, default_stream); std::tie(offsets, indices, weights) = edge_list_to_compressed_sparse( *(this->get_handle_ptr()), edgelists[i], major_first, major_last, minor_first, minor_last); adj_matrix_partition_offsets_.push_back(std::move(offsets)); @@ -361,6 +366,11 @@ graph_tget_number_of_edges(), "Invalid API parameter: the sum of local edges doe counts not match with " "number_of_local_edges."); diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp index 536b5081d69..fccc180a292 100644 --- a/cpp/tests/experimental/graph_test.cpp +++ b/cpp/tests/experimental/graph_test.cpp @@ -124,8 +124,6 @@ class Tests_Graph : public ::testing::TestWithParam { d_weights.data(), mm_graph.h_weights.data(), number_of_edges, handle.get_stream()); } - CUDA_TRY(cudaStreamSynchronize(handle.get_stream())); - cugraph::experimental::edgelist_t edgelist{ d_rows.data(), d_cols.data(), @@ -147,11 +145,11 @@ class Tests_Graph : public ::testing::TestWithParam { auto graph_view = graph.view(); + CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement + ASSERT_EQ(graph_view.get_number_of_vertices(), mm_graph.number_of_vertices); ASSERT_EQ(graph_view.get_number_of_edges(), number_of_edges); - CUDA_TRY(cudaDeviceSynchronize()); // for consistent performance measurement - std::vector h_cugraph_offsets(graph_view.get_number_of_vertices() + 1); std::vector h_cugraph_indices(graph_view.get_number_of_edges()); std::vector h_cugraph_weights( From 488f1ce93d437976c6fc4fb488ef25a7f8011cee Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Wed, 2 Sep 2020 15:54:57 -0400 Subject: [PATCH 63/77] fix compile error --- cpp/include/experimental/detail/graph_utils.cuh | 6 ++++-- cpp/src/experimental/graph.cu | 3 ++- cpp/src/experimental/graph_view.cu | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/include/experimental/detail/graph_utils.cuh b/cpp/include/experimental/detail/graph_utils.cuh index 853c17eebb4..fe092342f80 100644 --- a/cpp/include/experimental/detail/graph_utils.cuh +++ b/cpp/include/experimental/detail/graph_utils.cuh @@ -92,8 +92,10 @@ rmm::device_uvector compute_major_degree( handle.get_stream()); } - handle.sync_stream(handle.get_stream()); // this is neessary as local_degrees will become - // out-of-scope once this function returns. + auto status = handle.get_comms().sync_stream( + handle.get_stream()); // this is neessary as local_degrees will become out-of-scope once this + // function returns. + CUGRAPH_EXPECTS(status == raft::comms::status_t::SUCCESS, "sync_stream() failure."); return degrees; } diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 00239662465..0a0d1c39370 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -367,10 +367,11 @@ graph_tget_number_of_edges(), "Invalid API parameter: the sum of local edges doe counts not match with " "number_of_local_edges."); From 1318518a951e0dc56b42a38c89d46ba86306296f Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Wed, 2 Sep 2020 16:04:20 -0400 Subject: [PATCH 64/77] fixed test issues --- python/cugraph/tests/test_hits.py | 2 +- python/cugraph/tests/utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/cugraph/tests/test_hits.py b/python/cugraph/tests/test_hits.py index 5778fcebe72..c8a9274e078 100644 --- a/python/cugraph/tests/test_hits.py +++ b/python/cugraph/tests/test_hits.py @@ -93,7 +93,7 @@ def networkx_call(M, max_iter, tol): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize("graph_file", utils.DATASETS_SMALL) +@pytest.mark.parametrize("graph_file", utils.DATASETS_UNDIRECTED) @pytest.mark.parametrize("max_iter", MAX_ITERATIONS) @pytest.mark.parametrize("tol", TOLERANCE) def test_hits(graph_file, max_iter, tol): diff --git a/python/cugraph/tests/utils.py b/python/cugraph/tests/utils.py index 7e33ce0a628..280c458af10 100644 --- a/python/cugraph/tests/utils.py +++ b/python/cugraph/tests/utils.py @@ -27,7 +27,6 @@ DATASETS = ['../datasets/karate-disjoint.csv', '../datasets/dolphins.csv', - '../datasets/polbooks.csv', '../datasets/netscience.csv', '../datasets/email-Eu-core.csv'] From be2d4492dba9fd29dc95b615367c5b1897b0de9f Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 10:10:09 -0400 Subject: [PATCH 65/77] addressing dask issues --- .../tests/dask/test_mg_batch_betweenness_centrality.py | 2 +- .../tests/dask/test_mg_batch_edge_betweenness_centrality.py | 4 ++-- python/cugraph/tests/dask/test_mg_renumber.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py index 0a0c372d9c5..61368a26762 100644 --- a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py @@ -24,7 +24,7 @@ # ============================================================================= # Parameters # ============================================================================= -DATASETS = utils.DATASETS +DATASETS = ['../datasets/karate.csv'] MG_DEVICE_COUNT_OPTIONS = [1, 2, 3, 4] RESULT_DTYPE_OPTIONS = [np.float64] diff --git a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py index 31ca62bd5c6..054fc148eea 100644 --- a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py @@ -24,8 +24,8 @@ # ============================================================================= # Parameters # ============================================================================= -DATASETS = utils.DATASETS -MG_DEVICE_COUNT_OPTIONS = [1, 2, 3, 4] +DATASETS = ['../datasets/karate.csv'] +MG_DEVICE_COUNT_OPTIONS = [1, 2, 4] RESULT_DTYPE_OPTIONS = [np.float64] diff --git a/python/cugraph/tests/dask/test_mg_renumber.py b/python/cugraph/tests/dask/test_mg_renumber.py index cf6fada2def..ceeeeb77a5a 100644 --- a/python/cugraph/tests/dask/test_mg_renumber.py +++ b/python/cugraph/tests/dask/test_mg_renumber.py @@ -45,7 +45,7 @@ def client_connection(): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize("graph_file", utils.DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_UNRENUMBERED) def test_mg_renumber(graph_file, client_connection): gc.collect() @@ -85,7 +85,7 @@ def test_mg_renumber(graph_file, client_connection): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize("graph_file", utils.DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_UNRENUMBERED) def test_mg_renumber2(graph_file, client_connection): gc.collect() @@ -120,7 +120,7 @@ def test_mg_renumber2(graph_file, client_connection): # Test all combinations of default/managed and pooled/non-pooled allocation -@pytest.mark.parametrize("graph_file", utils.DATASETS) +@pytest.mark.parametrize("graph_file", utils.DATASETS_UNRENUMBERED) def test_mg_renumber3(graph_file, client_connection): gc.collect() From 20be1664b0313be63811dbce64513b698239e58d Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 10:14:19 -0400 Subject: [PATCH 66/77] flake8 unused fixes --- .../cugraph/tests/dask/test_mg_batch_betweenness_centrality.py | 2 +- .../tests/dask/test_mg_batch_edge_betweenness_centrality.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py index 61368a26762..c5f5f699017 100644 --- a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py @@ -1,5 +1,5 @@ import pytest -import cugraph.tests.utils as utils +# import cugraph.tests.utils as utils import numpy as np from cugraph.tests.dask.mg_context import (MGContext, diff --git a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py index 054fc148eea..ba446d00da8 100644 --- a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py @@ -1,5 +1,5 @@ import pytest -import cugraph.tests.utils as utils +# import cugraph.tests.utils as utils import numpy as np From 9048133a614df329a8a4a050fd81e0243ea65e72 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 10:16:43 -0400 Subject: [PATCH 67/77] added missing copyright --- python/cugraph/tests/dask/mg_context.py | 13 +++++++++++++ .../dask/test_mg_batch_betweenness_centrality.py | 14 +++++++++++++- .../test_mg_batch_edge_betweenness_centrality.py | 14 +++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/python/cugraph/tests/dask/mg_context.py b/python/cugraph/tests/dask/mg_context.py index e062cfd842b..9a7ea2ace67 100644 --- a/python/cugraph/tests/dask/mg_context.py +++ b/python/cugraph/tests/dask/mg_context.py @@ -1,3 +1,16 @@ +# Copyright (c) 2019-2020, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import time import os diff --git a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py index c5f5f699017..ccb0c94b020 100644 --- a/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_betweenness_centrality.py @@ -1,5 +1,17 @@ +# Copyright (c) 2019-2020, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest -# import cugraph.tests.utils as utils import numpy as np from cugraph.tests.dask.mg_context import (MGContext, diff --git a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py index ba446d00da8..01023839d06 100644 --- a/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py +++ b/python/cugraph/tests/dask/test_mg_batch_edge_betweenness_centrality.py @@ -1,5 +1,17 @@ +# Copyright (c) 2019-2020, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import pytest -# import cugraph.tests.utils as utils import numpy as np From 8f59b53f3410b8399556fb627c47318829450acb Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 11:04:25 -0400 Subject: [PATCH 68/77] added a version of Karate that contains a second disjoint version --- .gitignore | 1 + datasets/karate-disjoint.csv | 312 +++++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+) create mode 100644 datasets/karate-disjoint.csv diff --git a/.gitignore b/.gitignore index 30bcd5a845d..c2498b35cf1 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ datasets/* !datasets/cyber.csv !datasets/karate-data.csv !datasets/karate_undirected.csv +!datasets/karate-disjoint.csv !datasets/netscience.csv diff --git a/datasets/karate-disjoint.csv b/datasets/karate-disjoint.csv new file mode 100644 index 00000000000..d4f03bb35b9 --- /dev/null +++ b/datasets/karate-disjoint.csv @@ -0,0 +1,312 @@ +1 0 1.0 +2 0 1.0 +3 0 1.0 +4 0 1.0 +5 0 1.0 +6 0 1.0 +7 0 1.0 +8 0 1.0 +10 0 1.0 +11 0 1.0 +12 0 1.0 +13 0 1.0 +17 0 1.0 +19 0 1.0 +21 0 1.0 +31 0 1.0 +2 1 1.0 +3 1 1.0 +7 1 1.0 +13 1 1.0 +17 1 1.0 +19 1 1.0 +21 1 1.0 +30 1 1.0 +3 2 1.0 +7 2 1.0 +8 2 1.0 +9 2 1.0 +13 2 1.0 +27 2 1.0 +28 2 1.0 +32 2 1.0 +7 3 1.0 +12 3 1.0 +13 3 1.0 +6 4 1.0 +10 4 1.0 +6 5 1.0 +10 5 1.0 +16 5 1.0 +16 6 1.0 +30 8 1.0 +32 8 1.0 +33 8 1.0 +33 9 1.0 +33 13 1.0 +32 14 1.0 +33 14 1.0 +32 15 1.0 +33 15 1.0 +32 18 1.0 +33 18 1.0 +33 19 1.0 +32 20 1.0 +33 20 1.0 +32 22 1.0 +33 22 1.0 +25 23 1.0 +27 23 1.0 +29 23 1.0 +32 23 1.0 +33 23 1.0 +25 24 1.0 +27 24 1.0 +31 24 1.0 +31 25 1.0 +29 26 1.0 +33 26 1.0 +33 27 1.0 +31 28 1.0 +33 28 1.0 +32 29 1.0 +33 29 1.0 +32 30 1.0 +33 30 1.0 +32 31 1.0 +33 31 1.0 +33 32 1.0 +0 1 1.0 +0 2 1.0 +0 3 1.0 +0 4 1.0 +0 5 1.0 +0 6 1.0 +0 7 1.0 +0 8 1.0 +0 10 1.0 +0 11 1.0 +0 12 1.0 +0 13 1.0 +0 17 1.0 +0 19 1.0 +0 21 1.0 +0 31 1.0 +1 2 1.0 +1 3 1.0 +1 7 1.0 +1 13 1.0 +1 17 1.0 +1 19 1.0 +1 21 1.0 +1 30 1.0 +2 3 1.0 +2 7 1.0 +2 8 1.0 +2 9 1.0 +2 13 1.0 +2 27 1.0 +2 28 1.0 +2 32 1.0 +3 7 1.0 +3 12 1.0 +3 13 1.0 +4 6 1.0 +4 10 1.0 +5 6 1.0 +5 10 1.0 +5 16 1.0 +6 16 1.0 +8 30 1.0 +8 32 1.0 +8 33 1.0 +9 33 1.0 +13 33 1.0 +14 32 1.0 +14 33 1.0 +15 32 1.0 +15 33 1.0 +18 32 1.0 +18 33 1.0 +19 33 1.0 +20 32 1.0 +20 33 1.0 +22 32 1.0 +22 33 1.0 +23 25 1.0 +23 27 1.0 +23 29 1.0 +23 32 1.0 +23 33 1.0 +24 25 1.0 +24 27 1.0 +24 31 1.0 +25 31 1.0 +26 29 1.0 +26 33 1.0 +27 33 1.0 +28 31 1.0 +28 33 1.0 +29 32 1.0 +29 33 1.0 +30 32 1.0 +30 33 1.0 +31 32 1.0 +31 33 1.0 +32 33 1.0 +101 100 1.0 +102 100 1.0 +103 100 1.0 +104 100 1.0 +105 100 1.0 +106 100 1.0 +107 100 1.0 +108 100 1.0 +110 100 1.0 +111 100 1.0 +112 100 1.0 +113 100 1.0 +117 100 1.0 +119 100 1.0 +121 100 1.0 +131 100 1.0 +102 101 1.0 +103 101 1.0 +107 101 1.0 +113 101 1.0 +117 101 1.0 +119 101 1.0 +121 101 1.0 +130 101 1.0 +103 102 1.0 +107 102 1.0 +108 102 1.0 +109 102 1.0 +113 102 1.0 +127 102 1.0 +128 102 1.0 +132 102 1.0 +107 103 1.0 +112 103 1.0 +113 103 1.0 +106 104 1.0 +110 104 1.0 +106 105 1.0 +110 105 1.0 +116 105 1.0 +116 106 1.0 +130 108 1.0 +132 108 1.0 +133 108 1.0 +133 109 1.0 +133 113 1.0 +132 114 1.0 +133 114 1.0 +132 115 1.0 +133 115 1.0 +132 118 1.0 +133 118 1.0 +133 119 1.0 +132 120 1.0 +133 120 1.0 +132 122 1.0 +133 122 1.0 +125 123 1.0 +127 123 1.0 +129 123 1.0 +132 123 1.0 +133 123 1.0 +125 124 1.0 +127 124 1.0 +131 124 1.0 +131 125 1.0 +129 126 1.0 +133 126 1.0 +133 127 1.0 +131 128 1.0 +133 128 1.0 +132 129 1.0 +133 129 1.0 +132 130 1.0 +133 130 1.0 +132 131 1.0 +133 131 1.0 +133 132 1.0 +100 101 1.0 +100 102 1.0 +100 103 1.0 +100 104 1.0 +100 105 1.0 +100 106 1.0 +100 107 1.0 +100 108 1.0 +100 110 1.0 +100 111 1.0 +100 112 1.0 +100 113 1.0 +100 117 1.0 +100 119 1.0 +100 121 1.0 +100 131 1.0 +101 102 1.0 +101 103 1.0 +101 107 1.0 +101 113 1.0 +101 117 1.0 +101 119 1.0 +101 121 1.0 +101 130 1.0 +102 103 1.0 +102 107 1.0 +102 108 1.0 +102 109 1.0 +102 113 1.0 +102 127 1.0 +102 128 1.0 +102 132 1.0 +103 107 1.0 +103 112 1.0 +103 113 1.0 +104 106 1.0 +104 110 1.0 +105 106 1.0 +105 110 1.0 +105 116 1.0 +106 116 1.0 +108 130 1.0 +108 132 1.0 +108 133 1.0 +109 133 1.0 +113 133 1.0 +114 132 1.0 +114 133 1.0 +115 132 1.0 +115 133 1.0 +118 132 1.0 +118 133 1.0 +119 133 1.0 +120 132 1.0 +120 133 1.0 +122 132 1.0 +122 133 1.0 +123 125 1.0 +123 127 1.0 +123 129 1.0 +123 132 1.0 +123 133 1.0 +124 125 1.0 +124 127 1.0 +124 131 1.0 +125 131 1.0 +126 129 1.0 +126 133 1.0 +127 133 1.0 +128 131 1.0 +128 133 1.0 +129 132 1.0 +129 133 1.0 +130 132 1.0 +130 133 1.0 +131 132 1.0 +131 133 1.0 +132 133 1.0 From fc2ed41a2ef6d2c3034af636fc169e9257470e06 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 13:34:45 -0400 Subject: [PATCH 69/77] added additional blogs --- docs/source/cugraph_blogs.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/source/cugraph_blogs.rst b/docs/source/cugraph_blogs.rst index ec8c0f0e278..a9954aee5cb 100644 --- a/docs/source/cugraph_blogs.rst +++ b/docs/source/cugraph_blogs.rst @@ -31,7 +31,18 @@ Media Academic Papers =============== - * HPEC +* S. Kang, A. Fender, J. Eaton, B. Rees: Computing PageRank Scores of Web Crawl Data Using DGX A100 Clusters. In IEEE HPEC, Sep. 2020 + + +Other BLOGS +======================== +* `4 graph algorithms on steroids for data scientists with cugraph + `_ +* `Where should I walk `_ +* `Where really are the parking spots? `_ +* `Accelerating Single Cell Genomic Analysis using RAPIDS `_ + + Copyright (c) 2020, NVIDIA CORPORATION. From c2817f721de70ea5fbc3c2b5feb56373d3a1163e Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 14:35:19 -0400 Subject: [PATCH 70/77] looking at comms time-out issues --- python/cugraph/tests/dask/test_mg_replication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cugraph/tests/dask/test_mg_replication.py b/python/cugraph/tests/dask/test_mg_replication.py index 5248998d801..4236ec3d6dd 100644 --- a/python/cugraph/tests/dask/test_mg_replication.py +++ b/python/cugraph/tests/dask/test_mg_replication.py @@ -19,7 +19,7 @@ import cugraph.tests.utils as utils import pytest -DATASETS_OPTIONS = utils.DATASETS +DATASETS_OPTIONS = utils.DATASETS_SMALL DIRECTED_GRAPH_OPTIONS = [False, True] MG_DEVICE_COUNT_OPTIONS = [1, 2, 3, 4] From 56619b5b1ee881c4b442df8df21c9abde0ace393 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 15:07:07 -0400 Subject: [PATCH 71/77] addeed a call to gc.coolect() --- python/cugraph/tests/dask/test_mg_replication.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/cugraph/tests/dask/test_mg_replication.py b/python/cugraph/tests/dask/test_mg_replication.py index 4236ec3d6dd..d1a2d3c29b6 100644 --- a/python/cugraph/tests/dask/test_mg_replication.py +++ b/python/cugraph/tests/dask/test_mg_replication.py @@ -18,6 +18,7 @@ import cugraph.dask.structure.replication as replication import cugraph.tests.utils as utils import pytest +import gc DATASETS_OPTIONS = utils.DATASETS_SMALL DIRECTED_GRAPH_OPTIONS = [False, True] @@ -28,7 +29,9 @@ @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_replicate_cudf_dataframe_with_weights(input_data_path, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) + df = cudf.read_csv(input_data_path, delimiter=' ', names=['src', 'dst', 'value'], @@ -45,6 +48,7 @@ def test_replicate_cudf_dataframe_with_weights(input_data_path, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_replicate_cudf_dataframe_no_weights(input_data_path, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) df = cudf.read_csv(input_data_path, delimiter=' ', @@ -62,6 +66,7 @@ def test_replicate_cudf_dataframe_no_weights(input_data_path, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_replicate_cudf_series(input_data_path, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) df = cudf.read_csv(input_data_path, delimiter=' ', @@ -85,6 +90,7 @@ def test_replicate_cudf_series(input_data_path, @pytest.mark.parametrize("directed", DIRECTED_GRAPH_OPTIONS) @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_no_context(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) G = utils.generate_cugraph_graph_from_file(graph_file, directed) assert G.batch_enabled is False, "Internal property should be False" @@ -97,6 +103,7 @@ def test_enable_batch_no_context(graph_file, directed, mg_device_count): @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_no_context_view_adj(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) G = utils.generate_cugraph_graph_from_file(graph_file, directed) assert G.batch_enabled is False, "Internal property should be False" @@ -108,6 +115,7 @@ def test_enable_batch_no_context_view_adj(graph_file, directed, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_context_then_views(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) G = utils.generate_cugraph_graph_from_file(graph_file, directed) with MGContext(mg_device_count): @@ -131,6 +139,7 @@ def test_enable_batch_context_then_views(graph_file, directed, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_view_then_context(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) G = utils.generate_cugraph_graph_from_file(graph_file, directed) @@ -158,6 +167,7 @@ def test_enable_batch_view_then_context(graph_file, directed, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_context_no_context_views(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) G = utils.generate_cugraph_graph_from_file(graph_file, directed) with MGContext(mg_device_count): @@ -177,6 +187,7 @@ def test_enable_batch_context_no_context_views(graph_file, directed, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_edgelist_replication(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) G = utils.generate_cugraph_graph_from_file(graph_file, directed) with MGContext(mg_device_count): @@ -192,6 +203,7 @@ def test_enable_batch_edgelist_replication(graph_file, directed, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_adjlist_replication_weights(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) df = cudf.read_csv(graph_file, delimiter=' ', @@ -224,6 +236,7 @@ def test_enable_batch_adjlist_replication_weights(graph_file, directed, @pytest.mark.parametrize("mg_device_count", MG_DEVICE_COUNT_OPTIONS) def test_enable_batch_adjlist_replication_no_weights(graph_file, directed, mg_device_count): + gc.collect() skip_if_not_enough_devices(mg_device_count) df = cudf.read_csv(graph_file, delimiter=' ', From 53c54b23dc9817651ea90dc4f780db140e1fc933 Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 15:27:08 -0400 Subject: [PATCH 72/77] flake8 issue --- python/cugraph/tests/dask/test_mg_replication.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cugraph/tests/dask/test_mg_replication.py b/python/cugraph/tests/dask/test_mg_replication.py index d1a2d3c29b6..061bcf83f20 100644 --- a/python/cugraph/tests/dask/test_mg_replication.py +++ b/python/cugraph/tests/dask/test_mg_replication.py @@ -31,7 +31,6 @@ def test_replicate_cudf_dataframe_with_weights(input_data_path, mg_device_count): gc.collect() skip_if_not_enough_devices(mg_device_count) - df = cudf.read_csv(input_data_path, delimiter=' ', names=['src', 'dst', 'value'], From 9f645f4d07267833841780988e439ac3dd410f5e Mon Sep 17 00:00:00 2001 From: BradReesWork Date: Thu, 3 Sep 2020 17:34:13 -0400 Subject: [PATCH 73/77] updated dataset list again --- python/cugraph/tests/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cugraph/tests/utils.py b/python/cugraph/tests/utils.py index 280c458af10..e68f934c619 100644 --- a/python/cugraph/tests/utils.py +++ b/python/cugraph/tests/utils.py @@ -27,8 +27,8 @@ DATASETS = ['../datasets/karate-disjoint.csv', '../datasets/dolphins.csv', - '../datasets/netscience.csv', - '../datasets/email-Eu-core.csv'] + '../datasets/netscience.csv'] +# '../datasets/email-Eu-core.csv'] STRONGDATASETS = ['../datasets/dolphins.csv', '../datasets/netscience.csv', From a91c7730b01b432486f45f94755b329265e8e202 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Thu, 3 Sep 2020 21:47:36 -0400 Subject: [PATCH 74/77] remove NCC_TEST --- cpp/tests/CMakeLists.txt | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 0624e44bf09..eb10790f328 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -253,21 +253,6 @@ set(FIND_MATCHES_TEST_SRC ConfigureTest(FIND_MATCHES_TEST "${FIND_MATCHES_TEST_SRC}" "") -################################################################################################### -#-NCCL tests --------------------------------------------------------------------- - -if (BUILD_MPI) - set(NCCL_TEST_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/nccl/nccl_test.cu") - - ConfigureTest(NCCL_TEST "${NCCL_TEST_SRC}" "") - - set(NCCL_DEGREE_TEST_SRC - "${CMAKE_CURRENT_SOURCE_DIR}/nccl/degree_test.cu") - - ConfigureTest(NCCL_DEGREE_TEST "${NCCL_DEGREE_TEST_SRC}" "") -endif(BUILD_MPI) - ################################################################################################### # - Experimental Graph tests ---------------------------------------------------------------------- From 3637aa19d7141580517d9a022e47a07edf3f065b Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Thu, 3 Sep 2020 22:22:21 -0400 Subject: [PATCH 75/77] update graph_t & graph_view_t to take properties_t object --- cpp/include/experimental/graph.hpp | 16 +++--------- cpp/include/experimental/graph_view.hpp | 29 +++++++++------------- cpp/src/experimental/graph.cu | 33 +++++++++++-------------- cpp/src/experimental/graph_view.cu | 26 +++++++++---------- cpp/tests/experimental/graph_test.cpp | 4 +-- 5 files changed, 42 insertions(+), 66 deletions(-) diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index 7c870789267..4b322fc25c5 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -66,9 +66,7 @@ class graph_t const &partition, vertex_t number_of_vertices, edge_t number_of_edges, - bool is_symmetric, - bool is_multigraph, - bool is_weighted, + graph_properties_t properties, bool sorted_by_global_degree_within_vertex_partition, bool do_expensive_check = false); @@ -98,9 +96,7 @@ class graph_tget_number_of_vertices(), this->get_number_of_edges(), - this->is_symmetric(), - this->is_multigraph(), - this->is_weighted(), + this->get_graph_properties(), vertex_partition_segment_offsets_.size() > 0, false); } @@ -136,9 +132,7 @@ class graph_t const &edge_list, vertex_t number_of_vertices, - bool is_symmetric, - bool is_multigraph, - bool is_weighted, + graph_properties_t properties, bool sorted_by_degree, bool do_expensive_check = false); @@ -154,9 +148,7 @@ class graph_tget_number_of_vertices(), this->get_number_of_edges(), - this->is_symmetric(), - this->is_multigraph(), - this->is_weighted(), + this->get_graph_properties(), segment_offsets_.size() > 0, false); } diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index d90c566e658..d1e7799891a 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -165,7 +165,13 @@ class partition_t { int comm_p_col_size_{0}; int comm_p_row_rank_{0}; int comm_p_col_rank_{0}; -}; // namespace experimental +}; + +struct graph_properties_t { + bool is_symmetric{false}; + bool is_multigraph{false}; + bool is_weighted{false}; +}; namespace detail { @@ -174,12 +180,6 @@ size_t constexpr low_degree_threshold{raft::warp_size()}; size_t constexpr mid_degree_threshold{1024}; size_t constexpr num_segments_per_vertex_partition{3}; -struct graph_properties_t { - bool is_symmetric{false}; - bool is_multigraph{false}; - bool is_weighted{false}; -}; - // Common for both graph_view_t & graph_t and both single-GPU & multi-GPU versions template class graph_base_t { @@ -187,13 +187,11 @@ class graph_base_t { graph_base_t(raft::handle_t const& handle, vertex_t number_of_vertices, edge_t number_of_edges, - bool is_symmetric, - bool is_multigraph, - bool is_weighted) + graph_properties_t properties) : handle_ptr_(&handle), number_of_vertices_(number_of_vertices), number_of_edges_(number_of_edges), - properties_({is_symmetric, is_multigraph, is_weighted}){}; + properties_(properties){}; vertex_t get_number_of_vertices() const { return number_of_vertices_; } edge_t get_number_of_edges() const { return number_of_edges_; } @@ -204,6 +202,7 @@ class graph_base_t { protected: raft::handle_t const* get_handle_ptr() const { return handle_ptr_; }; + graph_properties_t get_graph_properties() const { return properties_; } private: raft::handle_t const* handle_ptr_{nullptr}; @@ -253,9 +252,7 @@ class graph_view_t const& partition, vertex_t number_of_vertices, edge_t number_of_edges, - bool is_symmetric, - bool is_multigraph, - bool is_weighted, + graph_properties_t properties, bool sorted_by_global_degree_within_vertex_partition, bool do_expensive_check = false); @@ -334,9 +331,7 @@ class graph_view_t const& segment_offsets, vertex_t number_of_vertices, edge_t number_of_edges, - bool is_symmetric, - bool is_multigraph, - bool is_weighted, + graph_properties_t properties, bool sorted_by_degree, bool do_expensive_check = false); diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 0a0d1c39370..635c904679d 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -209,13 +209,11 @@ graph_t const &partition, vertex_t number_of_vertices, edge_t number_of_edges, - bool is_symmetric, - bool is_multigraph, - bool is_weighted, + graph_properties_t properties, bool sorted_by_global_degree_within_vertex_partition, bool do_expensive_check) : detail::graph_base_t( - handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + handle, number_of_vertices, number_of_edges, properties), partition_(partition) { // cheap error checks @@ -230,6 +228,7 @@ graph_tget_handle_ptr()->get_stream(); + bool is_weighted = this->is_weighted(); CUGRAPH_EXPECTS( std::any_of(edgelists.begin(), edgelists.end(), @@ -378,10 +377,10 @@ graph_tis_symmetric()) {} // FIXME: check for duplicate edges may better be implemented after deciding whether to sort // neighbor list or not. - if (!is_multigraph) {} + if (!this->is_multigraph()) {} } } @@ -394,17 +393,13 @@ graph_t const &edgelist, vertex_t number_of_vertices, - bool is_symmetric, - bool is_multigraph, - bool is_weighted, + graph_properties_t properties, bool sorted_by_degree, bool do_expensive_check) : detail::graph_base_t(handle, number_of_vertices, edgelist.number_of_edges, - is_symmetric, - is_multigraph, - is_weighted), + properties), offsets_(rmm::device_uvector(0, handle.get_stream())), indices_(rmm::device_uvector(0, handle.get_stream())), weights_(rmm::device_uvector(0, handle.get_stream())) @@ -417,8 +412,8 @@ graph_tis_weighted() && (edgelist.p_edge_weights != nullptr)) || + (!this->is_weighted() && (edgelist.p_edge_weights == nullptr)), "Invalid API parameter: edgelist.p_edge_weights should not be nullptr (if " "is_weighted is true) or should be nullptr (if is_weighted is false)."); @@ -438,16 +433,16 @@ graph_tis_symmetric()) {} // FIXME: check for duplicate edges may better be implemented after deciding whether to sort // neighbor list or not. - if (!is_multigraph) {} + if (!this->is_multigraph()) {} } // convert edge list (COO) to compressed sparse format (CSR or CSC) CUGRAPH_EXPECTS( - (is_weighted == false) || (edgelist.p_edge_weights != nullptr), + (this->is_weighted() == false) || (edgelist.p_edge_weights != nullptr), "Invalid API parameter, edgelist.p_edge_weights shoud not be nullptr if is_weighted == true"); std::tie(offsets_, indices_, weights_) = @@ -513,10 +508,10 @@ graph_tis_symmetric()) {} // FIXME: check for duplicate edges may better be implemented after deciding whether to sort // neighbor list or not. - if (!is_multigraph) {} + if (!this->is_multigraph()) {} } } diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index 768381b5b5c..e8301c94c68 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -63,13 +63,11 @@ graph_view_t const& partition, vertex_t number_of_vertices, edge_t number_of_edges, - bool is_symmetric, - bool is_multigraph, - bool is_weighted, + graph_properties_t properties, bool sorted_by_global_degree_within_vertex_partition, bool do_expensive_check) : detail::graph_base_t( - handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + handle, number_of_vertices, number_of_edges, properties), adj_matrix_partition_offsets_(adj_matrix_partition_offsets), adj_matrix_partition_indices_(adj_matrix_partition_indices), adj_matrix_partition_weights_(adj_matrix_partition_weights), @@ -86,8 +84,8 @@ graph_view_tis_weighted() && (adj_matrix_partition_weights.size() == adj_matrix_partition_offsets.size())) || + (!this->is_weighted() && (adj_matrix_partition_weights.size() == 0)), "Invalid API parameter: adj_matrix_partition_weights.size() should coincide with " "adj_matrix_partition_offsets.size() (if is_weighted is true) or 0 (if is_weighted is false)."); @@ -188,10 +186,10 @@ graph_view_tis_symmetric()) {} // FIXME: check for duplicate edges may better be implemented after deciding whether to sort // neighbor list or not. - if (!is_multigraph) {} + if (!this->is_multigraph()) {} } } @@ -213,13 +211,11 @@ graph_view_t( - handle, number_of_vertices, number_of_edges, is_symmetric, is_multigraph, is_weighted), + handle, number_of_vertices, number_of_edges, properties), offsets_(offsets), indices_(indices), weights_(weights), @@ -227,7 +223,7 @@ graph_view_tis_weighted() && (weights != nullptr)) || (!this->is_weighted() && (weights == nullptr)), "Invalid API parameter: weights shouldn't be nullptr if is_weighted is true and " "should be nullptr if is_weighted is false."); @@ -274,10 +270,10 @@ graph_view_tis_symmetric()) {} // FIXME: check for duplicate edges may better be implemented after deciding whether to sort // neighbor list or not. - if (!is_multigraph) {} + if (!this->is_multigraph()) {} } } diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp index fccc180a292..792dab6d29a 100644 --- a/cpp/tests/experimental/graph_test.cpp +++ b/cpp/tests/experimental/graph_test.cpp @@ -137,9 +137,7 @@ class Tests_Graph : public ::testing::TestWithParam { handle, edgelist, mm_graph.number_of_vertices, - mm_graph.is_symmetric, - false, - configuration.test_weighted, + cugraph::experimental::graph_properties_t{mm_graph.is_symmetric, false, configuration.test_weighted}, false, true); From f4ac4f13e69ccbdf469d03ae008deb878566a921 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Thu, 3 Sep 2020 22:58:00 -0400 Subject: [PATCH 76/77] uint32_t & uint64_t to int32_t & int64_t in expliciit template instantiation --- cpp/include/experimental/graph.hpp | 2 +- cpp/include/experimental/graph_view.hpp | 4 +- cpp/src/experimental/graph.cu | 76 +++++++++++-------------- cpp/src/experimental/graph_view.cu | 60 +++++++++---------- cpp/tests/experimental/graph_test.cpp | 26 ++++----- 5 files changed, 76 insertions(+), 92 deletions(-) diff --git a/cpp/include/experimental/graph.hpp b/cpp/include/experimental/graph.hpp index 4b322fc25c5..ea4a7882363 100644 --- a/cpp/include/experimental/graph.hpp +++ b/cpp/include/experimental/graph.hpp @@ -84,7 +84,7 @@ class graph_tis_weighted()) { weights[i] = adj_matrix_partition_weights_[i].data(); } + if (weights.size() > 0) { weights[i] = adj_matrix_partition_weights_[i].data(); } } return graph_view_t( diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index d1e7799891a..c6a39e3305f 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -170,7 +170,6 @@ class partition_t { struct graph_properties_t { bool is_symmetric{false}; bool is_multigraph{false}; - bool is_weighted{false}; }; namespace detail { @@ -198,7 +197,6 @@ class graph_base_t { bool is_symmetric() const { return properties_.is_symmetric; } bool is_multigraph() const { return properties_.is_multigraph; } - bool is_weighted() const { return properties_.is_weighted; } protected: raft::handle_t const* get_handle_ptr() const { return handle_ptr_; }; @@ -288,7 +286,7 @@ class graph_view_t 0 ? adj_matrix_partition_weights_[adj_matrix_partition_idx] : static_cast(nullptr); } private: diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 635c904679d..04d711edf78 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -228,9 +228,12 @@ graph_tget_handle_ptr()->get_stream(); - bool is_weighted = this->is_weighted(); + CUGRAPH_EXPECTS(edgelists.size() > 0, "Invalid API parameter: edgelists.size() should be non-zero."); + + bool is_weighted = edgelists[0].p_edge_weights != nullptr; + CUGRAPH_EXPECTS( - std::any_of(edgelists.begin(), + std::any_of(edgelists.begin() + 1, edgelists.end(), [is_weighted](auto edgelist) { return (edgelist.p_src_vertices == nullptr) || @@ -239,8 +242,7 @@ graph_t(comm_p_row_size))) || @@ -290,12 +292,8 @@ graph_tis_weighted() ? edgelists.size() : 0); + adj_matrix_partition_weights_.reserve(is_weighted ? edgelists.size() : 0); for (size_t i = 0; i < edgelists.size(); ++i) { - CUGRAPH_EXPECTS((is_weighted == false) || (edgelists[i].p_edge_weights != nullptr), - "Invalid API parameter, edgelists[i].p_edge_weights shoud not be nullptr if " - "is_weighted == true"); - vertex_t major_first{}; vertex_t major_last{}; vertex_t minor_first{}; @@ -310,7 +308,7 @@ graph_tget_handle_ptr()), edgelists[i], major_first, major_last, minor_first, minor_last); adj_matrix_partition_offsets_.push_back(std::move(offsets)); adj_matrix_partition_indices_.push_back(std::move(indices)); - if (this->is_weighted()) { adj_matrix_partition_weights_.push_back(std::move(weights)); } + if (adj_matrix_partition_weights_.size() > 0) { adj_matrix_partition_weights_.push_back(std::move(weights)); } } // update degree-based segment offsets (to be used for graph analytics kernel optimization) @@ -412,10 +410,6 @@ graph_tis_weighted() && (edgelist.p_edge_weights != nullptr)) || - (!this->is_weighted() && (edgelist.p_edge_weights == nullptr)), - "Invalid API parameter: edgelist.p_edge_weights should not be nullptr (if " - "is_weighted is true) or should be nullptr (if is_weighted is false)."); // optional expensive checks (part 1/2) @@ -441,10 +435,6 @@ graph_tis_weighted() == false) || (edgelist.p_edge_weights != nullptr), - "Invalid API parameter, edgelist.p_edge_weights shoud not be nullptr if is_weighted == true"); - std::tie(offsets_, indices_, weights_) = edge_list_to_compressed_sparse(*(this->get_handle_ptr()), edgelist, @@ -517,31 +507,31 @@ graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; - -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; -template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; + +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; +template class graph_t; } // namespace experimental } // namespace cugraph diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index e8301c94c68..2feb954f0e4 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -84,10 +84,10 @@ graph_view_tis_weighted() && (adj_matrix_partition_weights.size() == adj_matrix_partition_offsets.size())) || - (!this->is_weighted() && (adj_matrix_partition_weights.size() == 0)), + (adj_matrix_partition_weights.size() == adj_matrix_partition_offsets.size()) || + (adj_matrix_partition_weights.size() == 0), "Invalid API parameter: adj_matrix_partition_weights.size() should coincide with " - "adj_matrix_partition_offsets.size() (if is_weighted is true) or 0 (if is_weighted is false)."); + "adj_matrix_partition_offsets.size() (if weighted) or 0 (if unweighted)."); CUGRAPH_EXPECTS( (partition.is_hypergraph_partitioned() && @@ -223,10 +223,6 @@ graph_view_tis_weighted() && (weights != nullptr)) || (!this->is_weighted() && (weights == nullptr)), - "Invalid API parameter: weights shouldn't be nullptr if is_weighted is true and " - "should be nullptr if is_weighted is false."); - CUGRAPH_EXPECTS( (sorted_by_degree && (segment_offsets.size() == (detail::num_segments_per_vertex_partition + 1))) || @@ -279,31 +275,31 @@ graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; - -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; -template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; + +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; +template class graph_view_t; } // namespace experimental } // namespace cugraph diff --git a/cpp/tests/experimental/graph_test.cpp b/cpp/tests/experimental/graph_test.cpp index 792dab6d29a..8225290b696 100644 --- a/cpp/tests/experimental/graph_test.cpp +++ b/cpp/tests/experimental/graph_test.cpp @@ -137,7 +137,7 @@ class Tests_Graph : public ::testing::TestWithParam { handle, edgelist, mm_graph.number_of_vertices, - cugraph::experimental::graph_properties_t{mm_graph.is_symmetric, false, configuration.test_weighted}, + cugraph::experimental::graph_properties_t{mm_graph.is_symmetric, false}, false, true); @@ -211,23 +211,23 @@ class Tests_Graph : public ::testing::TestWithParam { // FIXME: add tests for type combinations TEST_P(Tests_Graph, CheckStoreTransposedFalse) { - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); } // FIXME: add tests for type combinations TEST_P(Tests_Graph, CheckStoreTransposedTrue) { - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); - run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); + run_current_test(GetParam()); } INSTANTIATE_TEST_CASE_P(simple_test, From bc01ce3779c907c0c2aabc4633cf0553760b0439 Mon Sep 17 00:00:00 2001 From: Seunghwa Kang Date: Fri, 4 Sep 2020 10:04:45 -0400 Subject: [PATCH 77/77] clang-format --- cpp/include/experimental/graph_view.hpp | 4 +++- cpp/src/experimental/graph.cu | 16 +++++++++------- cpp/src/experimental/graph_view.cu | 9 ++++----- cpp/tests/experimental/graph_test.cpp | 2 -- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cpp/include/experimental/graph_view.hpp b/cpp/include/experimental/graph_view.hpp index c6a39e3305f..b3b899a5068 100644 --- a/cpp/include/experimental/graph_view.hpp +++ b/cpp/include/experimental/graph_view.hpp @@ -286,7 +286,9 @@ class graph_view_t 0 ? adj_matrix_partition_weights_[adj_matrix_partition_idx] : static_cast(nullptr); + return adj_matrix_partition_weights_.size() > 0 + ? adj_matrix_partition_weights_[adj_matrix_partition_idx] + : static_cast(nullptr); } private: diff --git a/cpp/src/experimental/graph.cu b/cpp/src/experimental/graph.cu index 04d711edf78..7b7625fd911 100644 --- a/cpp/src/experimental/graph.cu +++ b/cpp/src/experimental/graph.cu @@ -228,7 +228,8 @@ graph_tget_handle_ptr()->get_stream(); - CUGRAPH_EXPECTS(edgelists.size() > 0, "Invalid API parameter: edgelists.size() should be non-zero."); + CUGRAPH_EXPECTS(edgelists.size() > 0, + "Invalid API parameter: edgelists.size() should be non-zero."); bool is_weighted = edgelists[0].p_edge_weights != nullptr; @@ -242,7 +243,8 @@ graph_t(comm_p_row_size))) || @@ -308,7 +310,9 @@ graph_tget_handle_ptr()), edgelists[i], major_first, major_last, minor_first, minor_last); adj_matrix_partition_offsets_.push_back(std::move(offsets)); adj_matrix_partition_indices_.push_back(std::move(indices)); - if (adj_matrix_partition_weights_.size() > 0) { adj_matrix_partition_weights_.push_back(std::move(weights)); } + if (adj_matrix_partition_weights_.size() > 0) { + adj_matrix_partition_weights_.push_back(std::move(weights)); + } } // update degree-based segment offsets (to be used for graph analytics kernel optimization) @@ -394,10 +398,8 @@ graph_t(handle, - number_of_vertices, - edgelist.number_of_edges, - properties), + : detail::graph_base_t( + handle, number_of_vertices, edgelist.number_of_edges, properties), offsets_(rmm::device_uvector(0, handle.get_stream())), indices_(rmm::device_uvector(0, handle.get_stream())), weights_(rmm::device_uvector(0, handle.get_stream())) diff --git a/cpp/src/experimental/graph_view.cu b/cpp/src/experimental/graph_view.cu index 2feb954f0e4..b297a825a01 100644 --- a/cpp/src/experimental/graph_view.cu +++ b/cpp/src/experimental/graph_view.cu @@ -83,11 +83,10 @@ graph_view_t { } }; -// FIXME: add tests for type combinations TEST_P(Tests_Graph, CheckStoreTransposedFalse) { run_current_test(GetParam()); @@ -219,7 +218,6 @@ TEST_P(Tests_Graph, CheckStoreTransposedFalse) run_current_test(GetParam()); } -// FIXME: add tests for type combinations TEST_P(Tests_Graph, CheckStoreTransposedTrue) { run_current_test(GetParam());