merge latest 0.17

CHANGELOG.md

@@ -1,13 +1,20 @@
 # cuGraph 0.17.0 (Date TBD)
 
 ## New Features
+- PR #1245 Add functions to add pandas and numpy compatibility
 
 ## Improvements
+- PR #1227 Pin cmake policies to cmake 3.17 version
 
 ## Bug Fixes
+- PR #1242 Calling gunrock cmake using explicit -D options, re-enabling C++ tests
+- PR #1246 Use latest Gunrock, update HITS implementation
+- PR #1250 Updated cuco commit hash to latest as of 2020-10-30 and removed unneeded GIT_SHALLOW param
+- PR #1251 Changed the MG context testing class to use updated parameters passed in from the individual tests
+- PR #1253 MG test fixes: updated additional comms.initialize() calls, fixed dask DataFrame comparisons
 
 
-# cuGraph 0.16.0 (Date TBD)
+# cuGraph 0.16.0 (21 Oct 2020)
 
 ## New Features
 - PR #1098 Add new graph classes to support 2D partitioning
@@ -45,7 +52,7 @@
 - PR #1222 Added min CUDA version check to MG Louvain
 - PR #1217 NetworkX Transition doc
 - PR #1223 Update mnmg docs
-
+- PR #1230 Improve gpuCI scripts
 
 ## Bug Fixes
 - PR #1131 Show style checker errors with set +e
@@ -62,8 +69,10 @@
 - PR #1196 Move subcomms init outside of individual algorithm functions
 - PR #1198 Remove deprecated call to from_gpu_matrix
 - PR #1174 Fix bugs in MNMG pattern accelerators and pattern accelerator based implementations of MNMG PageRank, BFS, and SSSP
-
-
+- PR #1233 Temporarily disabling C++ tests for 0.16
+- PR #1240 Require `ucx-proc=*=gpu`
+- PR #1241 Fix a bug in personalized PageRank with the new graph primitives API.
+- PR #1249 Fix upload script syntax
 
 # cuGraph 0.15.0 (26 Aug 2020)
 

ci/benchmark/build.sh

@@ -9,36 +9,40 @@ set -o pipefail
 NUMARGS=$#
 ARGS=$*
 
-function logger {
-  echo -e "\n>>>> $@\n"
-}
-
 function hasArg {
     (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
 function cleanup {
-  logger "Removing datasets and temp files..."
+  gpuci_logger "Removing datasets and temp files"
   rm -rf $WORKSPACE/datasets/test
   rm -rf $WORKSPACE/datasets/benchmark
   rm -f testoutput.txt
 }
 
 # Set cleanup trap for Jenkins
 if [ ! -z "$JENKINS_HOME" ] ; then
-  logger "Jenkins environment detected, setting cleanup trap..."
+  gpuci_logger "Jenkins environment detected, setting cleanup trap"
   trap cleanup EXIT
 fi
 
 # Set path, build parallel level, and CUDA version
 cd $WORKSPACE
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 export CUDA_REL=${CUDA_VERSION%.*}
+
+# Set home
 export HOME=$WORKSPACE
+
+# Parse git describe
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 # Set Benchmark Vars
 export DATASETS_DIR=${WORKSPACE}/datasets
 export BENCHMARKS_DIR=${WORKSPACE}/benchmarks
@@ -49,56 +53,58 @@ export BENCHMARKS_DIR=${WORKSPACE}/benchmarks
 
 # TODO: Delete build section when artifacts are available
 
-logger "Check environment..."
+gpuci_logger "Check environment"
 env
 
-logger "Check GPU usage..."
+gpuci_logger "Check GPU usage"
 nvidia-smi
 
-logger "Activate conda env..."
-source activate rapids
-
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
 # Enter dependencies to be shown in ASV tooltips.
 CUGRAPH_DEPS=(cudf rmm)
 LIBCUGRAPH_DEPS=(cudf rmm)
 
-logger "conda install required packages"
-conda install -c nvidia -c rapidsai -c rapidsai-nightly -c conda-forge -c defaults \
+gpuci_logger "Install required packages"
+gpuci_conda_retry install -c nvidia -c rapidsai -c rapidsai-nightly -c conda-forge -c defaults \
       "cudf=${MINOR_VERSION}" \
       "rmm=${MINOR_VERSION}" \
       "cudatoolkit=$CUDA_REL" \
       "dask-cudf=${MINOR_VERSION}" \
       "dask-cuda=${MINOR_VERSION}" \
       "ucx-py=${MINOR_VERSION}" \
+      "ucx-proc=*=gpu" \
       "rapids-build-env=${MINOR_VERSION}" \
       rapids-pytest-benchmark
 
-# Install the master version of dask and distributed
-logger "pip install git+https://github.com/dask/distributed.git --upgrade --no-deps"
+gpuci_logger "Install the master version of dask and distributed"
 pip install "git+https://github.com/dask/distributed.git" --upgrade --no-deps
-
-logger "pip install git+https://github.com/dask/dask.git --upgrade --no-deps"
 pip install "git+https://github.com/dask/dask.git" --upgrade --no-deps
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
 $CC --version
 $CXX --version
-conda list
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 ##########################################
 # Build cuGraph                          #
 ##########################################
 
-logger "Build libcugraph..."
+gpuci_logger "Build libcugraph"
 $WORKSPACE/build.sh clean libcugraph cugraph
 
 ##########################################
 # Run Benchmarks                         #
 ##########################################
 
-logger "Downloading Datasets for Benchmarks..."
+gpuci_logger "Downloading Datasets for Benchmarks"
 cd $DATASETS_DIR
 bash ./get_test_data.sh --benchmark
 ERRORCODE=$((ERRORCODE | $?))
@@ -148,7 +154,7 @@ BENCHMARK_META=$(jq -n \
 echo "Benchmark meta:"
 echo "${BENCHMARK_META}" | jq "."
 
-logger "Running Benchmarks..."
+gpuci_logger "Running Benchmarks"
 cd $BENCHMARKS_DIR
 set +e
 time pytest -v -m "small and managedmem_on and poolallocator_on" \

ci/cpu/build.sh

@@ -5,14 +5,9 @@
 #########################################
 set -e
 
-# Logger function for build status output
-function logger() {
-  echo -e "\n>>>> $@\n"
-}
-
 # Set path and build parallel level
-export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
-export PARALLEL_LEVEL=4
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
 
 # Set home to the job's workspace
 export HOME=$WORKSPACE
@@ -25,41 +20,59 @@ if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then
   export VERSION_SUFFIX=`date +%y%m%d`
 fi
 
+# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts)
+export GPUCI_CONDA_RETRY_MAX=1
+export GPUCI_CONDA_RETRY_SLEEP=30
+
 ################################################################################
 # SETUP - Check environment
 ################################################################################
 
-logger "Get env..."
+gpuci_logger "Check environment variables"
 env
 
-logger "Activate conda env..."
-source activate gdf
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate rapids
 
-logger "Check versions..."
+gpuci_logger "Check versions"
 python --version
-gcc --version
-g++ --version
-conda list
+$CC --version
+$CXX --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
 
 # FIX Added to deal with Anancoda SSL verification issues during conda builds
 conda config --set ssl_verify False
 
 ###############################################################################
-# BUILD - Conda package builds (conda deps: libcugraph <- cugraph)
-################################################################################
+# BUILD - Conda package builds
+###############################################################################
 
-logger "Build conda pkg for libcugraph..."
-source ci/cpu/libcugraph/build_libcugraph.sh
+gpuci_logger "Build conda pkg for libcugraph"
+if [ "$BUILD_LIBCUGRAPH" == '1' ]; then
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    conda build conda/recipes/libcugraph
+  else
+    conda build --dirty --no-remove-work-dir conda/recipes/libcugraph
+  fi
+fi
 
-logger "Build conda pkg for cugraph..."
-source ci/cpu/cugraph/build_cugraph.sh
+gpuci_logger "Build conda pkg for cugraph"
+if [ "$BUILD_CUGRAPH" == "1" ]; then
+  if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    conda build conda/recipes/cugraph --python=$PYTHON
+  else
+    conda build conda/recipes/cugraph -c ci/artifacts/cugraph/cpu/conda-bld/ --dirty --no-remove-work-dir --python=$PYTHON
+  fi
+fi
 
 ################################################################################
 # UPLOAD - Conda packages
 ################################################################################
 
-logger "Upload libcugraph conda pkg..."
-source ci/cpu/libcugraph/upload-anaconda.sh
-
-logger "Upload cugraph conda pkg..."
-source ci/cpu/cugraph/upload-anaconda.sh
+gpuci_logger "Upload conda packages"
+source ci/cpu/upload.sh