From 1cdf993e329f68a105987a613ca0b2907a68acc9 Mon Sep 17 00:00:00 2001 From: ayush Date: Mon, 25 Jun 2018 20:10:22 -0700 Subject: [PATCH 01/30] First commit for gridded method --- package/MDAnalysis/lib/c_gridsearch.pyx | 747 ++++++++++++++++++++++++ package/setup.py | 9 +- 2 files changed, 755 insertions(+), 1 deletion(-) create mode 100644 package/MDAnalysis/lib/c_gridsearch.pyx diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx new file mode 100644 index 00000000000..82d7192544a --- /dev/null +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -0,0 +1,747 @@ +# -*- coding: utf-8; Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 +# +# Copyright (C) 2013-2016 Sébastien Buchoux +# +# This file is part of FATSLiM. +# +# FATSLiM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FATSLiM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FATSLiM. If not, see . +#cython: cdivision=True +#cython: boundscheck=False + +# Preprocessor DEFs +DEF DIM = 3 +DEF XX = 0 +DEF YY = 1 +DEF ZZ = 2 +DEF RET_OK = 1 +DEF RET_ERROR = 0 +DEF EPSILON = 1e-5 +DEF NEIGHBORHOOD_ALLOCATION_INCREMENT = 50 +DEF GRID_ALLOCATION_INCREMENT = 50 + +DEF BOX_MARGIN=1.0010 +DEF MAX_NTRICVEC=12 + + +# Cython C imports (no Python here!) +from cython.parallel cimport prange +from libc.stdlib cimport malloc, realloc, free, abort +from libc.stdio cimport fprintf, stderr +from libc.math cimport sqrt +from libc.math cimport abs as real_abs + +cimport openmp + + +# Python imports +import numpy as np +cimport numpy as np + +# Ctypes +ctypedef np.int_t ns_int +ctypedef np.float32_t real +ctypedef real rvec[DIM] +ctypedef real matrix[DIM][DIM] + +cdef struct ns_grid: + ns_int size + ns_int[DIM] ncells + real[DIM] cellsize + ns_int *nbeads + ns_int **beadids + +cdef struct ns_neighborhood: + real cutoff + ns_int allocated_size + ns_int size + ns_int *beadids + real *beaddist + ### +cdef struct ns_neighborhood_holder: + ns_int size + ns_neighborhood **neighborhoods + +# Useful stuff + +cdef real rvec_norm2(const rvec a) nogil: + return a[XX]*a[XX]+a[YY]*a[YY]+a[ZZ]*a[ZZ] + +cdef void rvec_clear(rvec a) nogil: + a[XX]=0.0 + a[YY]=0.0 + a[ZZ]=0.0 + +cdef struct cPBCBox_t: + matrix box + rvec fbox_diag + rvec hbox_diag + rvec mhbox_diag + real max_cutoff2 + ns_int ntric_vec + ns_int[DIM] tric_shift[MAX_NTRICVEC] + real[DIM] tric_vec[MAX_NTRICVEC] + +# noinspection PyNoneFunctionAssignment +cdef class PBCBox(object): + cdef cPBCBox_t c_pbcbox + cdef rvec center + cdef rvec bbox_center + + def __init__(self, real[:,::1] box): + self.update(box) + + cdef void fast_update(self, real[:,::1] box) nogil: + cdef ns_int i, j, k, d, jc, kc, shift + cdef real d2old, d2new, d2new_c + cdef rvec trial, pos + cdef ns_int ii, jj ,kk + cdef ns_int *order = [0, -1, 1, -2, 2] + cdef bint use + cdef real min_hv2, min_ss, tmp + + rvec_clear(self.center) + # Update matrix + for i in range(DIM): + for j in range(DIM): + self.c_pbcbox.box[i][j] = box[i, j] + self.center[j] += 0.5 * box[i, j] + self.bbox_center[i] = 0.5 * box[i, i] + + # Update diagonals + for i in range(DIM): + self.c_pbcbox.fbox_diag[i] = box[i, i] + self.c_pbcbox.hbox_diag[i] = self.c_pbcbox.fbox_diag[i] * 0.5 + self.c_pbcbox.mhbox_diag[i] = - self.c_pbcbox.hbox_diag[i] + + # Update maximum cutoff + + # Physical limitation of the cut-off + # by half the length of the shortest box vector. + min_hv2 = min(0.25 * rvec_norm2(&box[XX, XX]), 0.25 * rvec_norm2(&box[YY, XX])) + min_hv2 = min(min_hv2, 0.25 * rvec_norm2(&box[ZZ, XX])) + + # Limitation to the smallest diagonal element due to optimizations: + # checking only linear combinations of single box-vectors (2 in x) + # in the grid search and pbc_dx is a lot faster + # than checking all possible combinations. + tmp = box[YY, YY] + if box[ZZ, YY] < 0: + tmp -= box[ZZ, YY] + else: + tmp += box[ZZ, YY] + + min_ss = min(box[XX, XX], min(tmp, box[ZZ, ZZ])) + + self.c_pbcbox.max_cutoff2 = min(min_hv2, min_ss * min_ss) + + # Update shift vectors + self.c_pbcbox.ntric_vec = 0 + # We will only use single shifts, but we will check a few + # more shifts to see if there is a limiting distance + # above which we can not be sure of the correct distance. + for kk in range(5): + k = order[kk] + + for jj in range(5): + j = order[jj] + + for ii in range(5): + i = order[ii] + + # A shift is only useful when it is trilinic + if j != 0 or k != 0: + d2old = 0 + d2new = 0 + + for d in range(DIM): + trial[d] = i*box[XX, d] + j*box[YY, d] + k*box[ZZ, d] + + # Choose the vector within the brick around 0,0,0 that + # will become the shortest due to shift try. + + if d == DIM: + trial[d] = 0 + pos[d] = 0 + else: + if trial[d] < 0: + pos[d] = min(self.c_pbcbox.hbox_diag[d], -trial[d]) + else: + pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) + + d2old += sqrt(pos[d]) + d2new += sqrt(pos[d] + trial[d]) + + if BOX_MARGIN*d2new < d2old: + if not (j < -1 or j > 1 or k < -1 or k > 1): + use = True + + for dd in range(DIM): + if dd == 0: + shift = i + elif dd == 1: + shift = j + else: + shift = k + + if shift: + d2new_c = 0 + + for d in range(DIM): + d2new_c += sqrt(pos[d] + trial[d] - shift*box[dd, d]) + + if d2new_c <= BOX_MARGIN*d2new: + use = False + + if use: # Accept this shift vector. + if self.c_pbcbox.ntric_vec >= MAX_NTRICVEC: + with gil: + print("\nWARNING: Found more than %d triclinic " + "correction vectors, ignoring some." + % MAX_NTRICVEC) + print(" There is probably something wrong with " + "your box.") + print(box) + else: + for d in range(DIM): + self.c_pbcbox.tric_vec[self.c_pbcbox.ntric_vec][d] = \ + trial[d] + self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][XX] = i + self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][YY] = j + self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][ZZ] = k + self.c_pbcbox.ntric_vec += 1 + + + def update(self, real[:,::1] box): + if box.shape[0] != DIM or box.shape[1] != DIM: + raise ValueError("Box must be a %i x %i matrix. (shape: %i x %i)" % + (DIM, DIM, box.shape[0], box.shape[1])) + if (box[XX, XX] == 0) or (box[YY, YY] == 0) or (box[ZZ, ZZ] == 0): + raise ValueError("Box does not correspond to PBC=xyz") + self.fast_update(box) + + cdef void fast_pbc_dx(self, rvec ref, rvec other, rvec dx) nogil: + cdef ns_int i, j + cdef rvec dx_start, trial + + for i in range(DIM): + dx[i] = other[i] - ref[i] + + for i in range (DIM-1, -1, -1): + while dx[i] > self.c_pbcbox.hbox_diag[i]: + for j in range (i, -1, -1): + dx[j] -= self.c_pbcbox.box[i][j] + + while dx[i] <= self.c_pbcbox.mhbox_diag[i]: + for j in range (i, -1, -1): + dx[j] += self.c_pbcbox.box[i][j] + + cdef real[:, ::1]fast_put_atoms_in_bbox(self, real[:,::1] coords) nogil: + cdef ns_int i, m, d, natoms, wd = 0 + cdef real[:,::1] bbox_coords + + natoms = coords.shape[0] + with gil: + if natoms == 0: + bbox_coords = np.empty((0, DIM)) + else: + bbox_coords = coords.copy() + + for i in range(natoms): + for m in range(DIM - 1, -1, -1): + while bbox_coords[i, m] < 0: + for d in range(m+1): + bbox_coords[i, d] += self.c_pbcbox.box[m][d] + while bbox_coords[i, m] >= self.c_pbcbox.box[m][m]: + for d in range(m+1): + bbox_coords[i, d] -= self.c_pbcbox.box[m][d] + return bbox_coords + + def put_atoms_in_bbox(self, real[:,::1] coords): + return np.asarray(self.fast_put_atoms_in_bbox(coords)) + + + +######################################################################################################################## +# +# Neighbor Search Stuff +# +######################################################################################################################## +cdef struct ns_grid: + ns_int size + ns_int[DIM] ncells + real[DIM] cellsize + ns_int *nbeads + ns_int **beadids + +cdef ns_grid initialize_nsgrid(matrix box, + float cutoff) nogil: + cdef ns_grid grid + cdef ns_int i + + for i in range(DIM): + grid.ncells[i] = (box[i][i] / cutoff) + if grid.ncells[i] == 0: + grid.ncells[i] = 1 + grid.cellsize[i] = box[i][i] / grid.ncells[i] + + grid.size = grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ] + return grid + +cdef ns_int populate_grid(ns_grid *grid, + real[:,::1] coords) nogil: + cdef ns_int ncoords = coords.shape[0] + cdef bint ret_val + + ret_val = populate_grid_array(grid, + &coords[0, 0], + ncoords) + + return ret_val + +cdef ns_int populate_grid_array(ns_grid *grid, + rvec *coords, + ns_int ncoords) nogil: + cdef ns_int i, cellindex = -1 + cdef ns_int grid_size = grid.size + cdef ns_int *allocated_size = NULL + + if grid_size != grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ]: # Grid not initialized + return RET_ERROR + + + # Allocate memory + grid.nbeads = malloc(sizeof(ns_int) * grid_size) + if grid.nbeads == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS grid.nbeads (requested: %i bytes)\n", + sizeof(ns_int) * grid_size) + abort() + + allocated_size = malloc(sizeof(ns_int) * grid_size) + if allocated_size == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS allocated_size (requested: %i bytes)\n", + sizeof(ns_int) * grid_size) + abort() + + for i in range(grid_size): + grid.nbeads[i] = 0 + allocated_size[i] = GRID_ALLOCATION_INCREMENT + + grid.beadids = malloc(sizeof(ns_int *) * grid_size) + if grid.beadids == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids (requested: %i bytes)\n", + sizeof(ns_int *) * grid_size) + abort() + + for i in range(grid_size): + grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) + if grid.beadids[i] == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids[i] (requested: %i bytes)\n", + sizeof(ns_int) * allocated_size[i]) + abort() + + # Get cell indices for coords + for i in range(ncoords): + cellindex = (coords[i][ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ + (coords[i][YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ + (coords[i][XX] / grid.cellsize[XX]) + + grid.beadids[cellindex][grid.nbeads[cellindex]] = i + grid.nbeads[cellindex] += 1 + + if grid.nbeads[cellindex] >= allocated_size[cellindex]: + allocated_size[cellindex] += GRID_ALLOCATION_INCREMENT + grid.beadids[cellindex] = realloc( grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) + free(allocated_size) + return RET_OK + +cdef void destroy_nsgrid(ns_grid *grid) nogil: + cdef ns_int i + if grid.nbeads != NULL: + free(grid.nbeads) + + for i in range(grid.size): + if grid.beadids[i] != NULL: + free(grid.beadids[i]) + free(grid.beadids) + + +cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: + cdef ns_neighborhood_holder *holder + + holder = malloc(sizeof(ns_neighborhood_holder)) + + return holder + +cdef void free_neighborhood_holder(ns_neighborhood_holder *holder) nogil: + cdef ns_int i + + if holder == NULL: + return + + for i in range(holder.size): + if holder.neighborhoods[i].beadids != NULL: + free(holder.neighborhoods[i].beadids) + free(holder.neighborhoods[i]) + free(holder.neighborhoods) + free(holder) + +cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]neighborcoords, ns_grid *grid, PBCBox box, real cutoff2) nogil: + cdef ns_int d, m + cdef ns_int xi, yi, zi, bid + cdef real d2 + cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords + + cdef bint already_checked[27] + cdef bint skip + cdef ns_int nchecked = 0, icheck + cdef ns_int cell_index + + cdef ns_neighborhood *neighborhood = malloc(sizeof(ns_neighborhood)) + if neighborhood == NULL: + abort() + + neighborhood.size = 0 + neighborhood.allocated_size = NEIGHBORHOOD_ALLOCATION_INCREMENT + neighborhood.beadids = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(ns_int)) + ###Modified here + neighborhood.beaddist = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(real)) + ### + if neighborhood.beadids == NULL: + abort() + + for zi in range(3): + for yi in range(3): + for xi in range(3): + # Calculate and/or reinitialize shifted coordinates + shifted_coord[XX] = current_coords[XX] + (xi - 1) * grid.cellsize[XX] + shifted_coord[YY] = current_coords[YY] + (yi - 1) * grid.cellsize[YY] + shifted_coord[ZZ] = current_coords[ZZ] + (zi - 1) * grid.cellsize[ZZ] + + # Make sure the shifted coordinates is inside the brick-shaped box + for m in range(DIM - 1, -1, -1): + + while shifted_coord[m] < 0: + for d in range(m+1): + shifted_coord[d] += box.c_pbcbox.box[m][d] + + + while shifted_coord[m] >= box.c_pbcbox.box[m][m]: + for d in range(m+1): + shifted_coord[d] -= box.c_pbcbox.box[m][d] + + # Get the cell index corresponding to the coord + cell_index = (shifted_coord[ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ + (shifted_coord[YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ + (shifted_coord[XX] / grid.cellsize[XX]) + + # Just a safeguard + if cell_index >= grid.size: + continue + + # Check the cell index was not already selected + skip = False + for icheck in range(nchecked): + if already_checked[icheck] == cell_index: + skip = True + break + if skip: + continue + + # Search for neighbors inside this cell + for i_bead in range(grid.nbeads[cell_index]): + bid = grid.beadids[cell_index][i_bead] + + box.fast_pbc_dx(current_coords, &neighborcoords[bid, XX], dx) + + d2 = rvec_norm2(dx) + + if d2 < cutoff2: + if d2 < EPSILON: # Don't add the current bead as its own neighbor! + continue + + # Update neighbor lists + neighborhood.beadids[neighborhood.size] = bid + ### Modified here + neighborhood.beaddist[neighborhood.size] = d2 + ### + neighborhood.size += 1 + + if neighborhood.size >= neighborhood.allocated_size: + neighborhood.allocated_size += NEIGHBORHOOD_ALLOCATION_INCREMENT + neighborhood.beadids = realloc( neighborhood.beadids, neighborhood.allocated_size * sizeof(ns_int)) + ###Modified here + neighborhood.beaddist = realloc( neighborhood.beaddist, neighborhood.allocated_size * sizeof(real)) + ### + if neighborhood.beadids == NULL: + abort() + ###Modified + if neighborhood.beaddist == NULL: + abort() + ### + # Register the cell as checked + already_checked[nchecked] = cell_index + nchecked += 1 + + return neighborhood + + +cdef ns_neighborhood_holder *ns_core_parallel(real[:, ::1] refcoords, + real[:, ::1] neighborcoords, + ns_grid *grid, + PBCBox box, + real cutoff, + int nthreads=-1) nogil: + cdef ns_int coordid, i, j + cdef ns_int ncoords = refcoords.shape[0] + cdef ns_int ncoords_neighbors = neighborcoords.shape[0] + cdef real cutoff2 = cutoff * cutoff + cdef ns_neighborhood_holder *holder + + cdef ns_int *neighbor_buf + cdef ns_int buf_size, ibuf + + if nthreads < 0: + nthreads = openmp.omp_get_num_threads() + + holder = create_neighborhood_holder() + if holder == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", + sizeof(ns_int) * ncoords) + abort() + + holder.size = ncoords + holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) + if holder.neighborhoods == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", + sizeof(ns_neighborhood) * ncoords) + abort() + + # Here starts the real core and the iteration over coordinates + for coordid in prange(ncoords, schedule='dynamic', num_threads=nthreads): + holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], + neighborcoords, + grid, + box, + cutoff2) + holder.neighborhoods[coordid].cutoff = cutoff + + return holder + +cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, + real[:, ::1] neighborcoords, + ns_grid *grid, + PBCBox box, + real cutoff) nogil: + cdef ns_int coordid, i, j + cdef ns_int ncoords = refcoords.shape[0] + cdef ns_int ncoords_neighbors = neighborcoords.shape[0] + cdef real cutoff2 = cutoff * cutoff + cdef ns_neighborhood_holder *holder + + cdef ns_int *neighbor_buf + cdef ns_int buf_size, ibuf + + holder = create_neighborhood_holder() + if holder == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", + sizeof(ns_int) * ncoords) + abort() + + holder.size = ncoords + holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) + if holder.neighborhoods == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", + sizeof(ns_neighborhood) * ncoords) + abort() + + # Here starts the real core and the iteration over coordinates + for coordid in range(ncoords): + holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], + neighborcoords, + grid, + box, + cutoff2) + holder.neighborhoods[coordid].cutoff = cutoff + + return holder + + +# Python interface +cdef class FastNS(object): + cdef PBCBox box + cdef readonly int nthreads + cdef readonly real[:, ::1] coords + cdef real[:, ::1] coords_bbox + cdef readonly real cutoff + cdef bint prepared + cdef ns_grid *grid + + + def __init__(self, box): + if box.shape != (3, 3): + raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") + + self.box = PBCBox(box) + + self.nthreads = 1 + + self.coords = None + self.coords_bbox = None + + self.cutoff = -1 + + self.prepared = False + + self.grid = malloc(sizeof(ns_grid)) + + + def __dealloc__(self): + #destroy_nsgrid(self.grid) + self.grid.size = 0 + + #free(self.grid) + + def set_nthreads(self, nthreads, silent=False): + import multiprocessing + + if nthreads > multiprocessing.cpu_count(): + print("Warning: the number of threads requested if greater than the number of cores available. Performances may not be optimal!") + + if not silent: + print("Number of threads for NS adjusted to {}.".format(nthreads)) + + self.nthreads = nthreads + + + def set_coords(self, real[:, ::1] coords): + self.coords = coords + + # Make sure atoms are inside the brick-shaped box + self.coords_bbox = self.box.fast_put_atoms_in_bbox(coords) + + self.prepared = False + + + def set_cutoff(self, real cutoff): + self.cutoff = cutoff + + self.prepared = False + + + def prepare(self, force=False): + cdef ns_int i + cdef bint initialization_ok + + if self.prepared and not force: + print("NS already prepared, nothing to do!") + + if self.coords is None: + raise ValueError("Coordinates must be set before NS preparation!") + + if self.cutoff < 0: + raise ValueError("Cutoff must be set before NS preparation!") + + with nogil: + initialization_ok = False + + # Initializing grid + for i in range(DIM): + self.grid.ncells[i] = (self.box.c_pbcbox.box[i][i] / self.cutoff) + if self.grid.ncells[i] == 0: + self.grid.ncells[i] = 1 + self.grid.cellsize[i] = self.box.c_pbcbox.box[i][i] / self.grid.ncells[i] + + self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] + + # Populating grid + if populate_grid(self.grid, self.coords_bbox) == RET_OK: + initialization_ok = True + + + if initialization_ok: + self.prepared = True + else: + raise RuntimeError("Could not initialize NS grid") + + + def search(self, real[:, ::1]search_coords, return_ids=False): + cdef real[:, ::1] search_coords_bbox + cdef ns_int nid, i, j + cdef ns_neighborhood_holder *holder + cdef ns_neighborhood *neighborhood + + if not self.prepared: + self.prepare() + + + # Make sure atoms are inside the brick-shaped box + search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) + + + with nogil: + # Retrieve neighbors from grid + if self.nthreads == 1: + holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) + else: + holder = ns_core_parallel(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff, self.nthreads) + + + + neighbors = [] + ###Modify for distance + sqdist = [] + indx = [] + ### + for nid in range(holder.size): + neighborhood = holder.neighborhoods[nid] + + if return_ids: + neighborhood_py = np.empty(neighborhood.size, dtype=np.int64) + ###Modify for distance + neighborhood_dis = np.empty(neighborhood.size, dtype=np.float32) + neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) + ### + for i in range(neighborhood.size): + neighborhood_py[i] = neighborhood.beadids[i] + ###Modify for distance + neighborhood_dis[i] = neighborhood.beaddist[i] + ### + else: + neighborhood_py = np.empty((neighborhood.size, DIM), dtype=np.float32) + ###Modify for distance + neighborhood_dis = np.empty((neighborhood.size), dtype=np.float32) + neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) + ### + for i in range(neighborhood.size): + ###Modify for distance + neighborhood_dis[i] = neighborhood.beaddist[i] + neighborhood_indx[i] = neighborhood.beadids[i] + ### + + for j in range(DIM): + neighborhood_py[i,j] = self.coords[neighborhood.beadids[i], j] + + neighbors.append(neighborhood_py) + sqdist.append(neighborhood_dis) + indx.append(neighborhood_indx) + + # Free Memory + free_neighborhood_holder(holder) + + return neighbors, sqdist, indx + + + +__version__ = "26" \ No newline at end of file diff --git a/package/setup.py b/package/setup.py index 38420dd2b3c..83a0052ac2a 100755 --- a/package/setup.py +++ b/package/setup.py @@ -359,9 +359,16 @@ def extensions(config): libraries=mathlib, define_macros=define_macros, extra_compile_args=extra_compile_args) + grid = MDAExtension('MDAnalysis.lib.grid', + ['MDAnalysis/lib/c_gridsearch' + source_suffix], + include_dirs=include_dirs, + libraries=mathlib + parallel_libraries, + define_macros=define_macros + parallel_macros, + extra_compile_args=extra_compile_args + parallel_args, + extra_link_args=parallel_args) pre_exts = [libdcd, distances, distances_omp, qcprot, transformation, libmdaxdr, util, encore_utils, - ap_clustering, spe_dimred] + ap_clustering, spe_dimred, grid] cython_generated = [] if use_cython: From b8fea35a523bce12d4abcd4f9b33ea06ddcb5d3f Mon Sep 17 00:00:00 2001 From: ayush Date: Mon, 25 Jun 2018 22:06:44 -0700 Subject: [PATCH 02/30] removed the parallel handler to remove hard dependency on omp.h, can be added again for conditional compilation --- package/MDAnalysis/lib/c_gridsearch.pyx | 104 +----------------------- 1 file changed, 2 insertions(+), 102 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 82d7192544a..0a8089b9ef1 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -1,22 +1,3 @@ -# -*- coding: utf-8; Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 -# -# Copyright (C) 2013-2016 Sébastien Buchoux -# -# This file is part of FATSLiM. -# -# FATSLiM is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# FATSLiM is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with FATSLiM. If not, see . #cython: cdivision=True #cython: boundscheck=False @@ -34,22 +15,14 @@ DEF GRID_ALLOCATION_INCREMENT = 50 DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 - -# Cython C imports (no Python here!) -from cython.parallel cimport prange from libc.stdlib cimport malloc, realloc, free, abort from libc.stdio cimport fprintf, stderr from libc.math cimport sqrt from libc.math cimport abs as real_abs -cimport openmp - - -# Python imports import numpy as np cimport numpy as np -# Ctypes ctypedef np.int_t ns_int ctypedef np.float32_t real ctypedef real rvec[DIM] @@ -93,7 +66,6 @@ cdef struct cPBCBox_t: ns_int[DIM] tric_shift[MAX_NTRICVEC] real[DIM] tric_vec[MAX_NTRICVEC] -# noinspection PyNoneFunctionAssignment cdef class PBCBox(object): cdef cPBCBox_t c_pbcbox cdef rvec center @@ -271,8 +243,6 @@ cdef class PBCBox(object): def put_atoms_in_bbox(self, real[:,::1] coords): return np.asarray(self.fast_put_atoms_in_bbox(coords)) - - ######################################################################################################################## # # Neighbor Search Stuff @@ -495,50 +465,6 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei nchecked += 1 return neighborhood - - -cdef ns_neighborhood_holder *ns_core_parallel(real[:, ::1] refcoords, - real[:, ::1] neighborcoords, - ns_grid *grid, - PBCBox box, - real cutoff, - int nthreads=-1) nogil: - cdef ns_int coordid, i, j - cdef ns_int ncoords = refcoords.shape[0] - cdef ns_int ncoords_neighbors = neighborcoords.shape[0] - cdef real cutoff2 = cutoff * cutoff - cdef ns_neighborhood_holder *holder - - cdef ns_int *neighbor_buf - cdef ns_int buf_size, ibuf - - if nthreads < 0: - nthreads = openmp.omp_get_num_threads() - - holder = create_neighborhood_holder() - if holder == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", - sizeof(ns_int) * ncoords) - abort() - - holder.size = ncoords - holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) - if holder.neighborhoods == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", - sizeof(ns_neighborhood) * ncoords) - abort() - - # Here starts the real core and the iteration over coordinates - for coordid in prange(ncoords, schedule='dynamic', num_threads=nthreads): - holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], - neighborcoords, - grid, - box, - cutoff2) - holder.neighborhoods[coordid].cutoff = cutoff - - return holder - cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, real[:, ::1] neighborcoords, ns_grid *grid, @@ -577,7 +503,6 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, return holder - # Python interface cdef class FastNS(object): cdef PBCBox box @@ -610,21 +535,7 @@ cdef class FastNS(object): def __dealloc__(self): #destroy_nsgrid(self.grid) self.grid.size = 0 - - #free(self.grid) - - def set_nthreads(self, nthreads, silent=False): - import multiprocessing - - if nthreads > multiprocessing.cpu_count(): - print("Warning: the number of threads requested if greater than the number of cores available. Performances may not be optimal!") - - if not silent: - print("Number of threads for NS adjusted to {}.".format(nthreads)) - - self.nthreads = nthreads - - + def set_coords(self, real[:, ::1] coords): self.coords = coords @@ -689,15 +600,8 @@ cdef class FastNS(object): # Make sure atoms are inside the brick-shaped box search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) - with nogil: - # Retrieve neighbors from grid - if self.nthreads == 1: - holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) - else: - holder = ns_core_parallel(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff, self.nthreads) - - + holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) neighbors = [] ###Modify for distance @@ -741,7 +645,3 @@ cdef class FastNS(object): free_neighborhood_holder(holder) return neighbors, sqdist, indx - - - -__version__ = "26" \ No newline at end of file From 30ff05cbf5c38fab1cd7aa5ce7cf35e101f8aa20 Mon Sep 17 00:00:00 2001 From: ayush Date: Sat, 30 Jun 2018 01:23:12 -0700 Subject: [PATCH 03/30] added the license header --- package/MDAnalysis/lib/c_gridsearch.pyx | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 0a8089b9ef1..a4e8745aec4 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -1,3 +1,22 @@ +# -*- coding: utf-8; Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 +# +# Copyright (C) 2013-2016 Sébastien Buchoux +# +# This file is part of FATSLiM. +# +# FATSLiM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FATSLiM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FATSLiM. If not, see . #cython: cdivision=True #cython: boundscheck=False From b152ae748387ed30782712729311dc93f305d71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 1 Jul 2018 14:06:40 +0200 Subject: [PATCH 04/30] Added MDAnalysis header file to c_gridsearch.pyx --- package/MDAnalysis/lib/c_gridsearch.pyx | 32 +++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index a4e8745aec4..ac73c97a5a2 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -1,22 +1,28 @@ -# -*- coding: utf-8; Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- +# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 # -# Copyright (C) 2013-2016 Sébastien Buchoux +# MDAnalysis --- https://www.mdanalysis.org # -# This file is part of FATSLiM. +# Copyright (C) 2013-2018 Sébastien Buchoux +# Copyright (c) 2018 The MDAnalysis Development Team and contributors +# (see the file AUTHORS for the full list of names) # -# FATSLiM is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# Released under the GNU Public Licence, v3 or any higher version # -# FATSLiM is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# Please cite your use of MDAnalysis in published work: # -# You should have received a copy of the GNU General Public License -# along with FATSLiM. If not, see . +# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler, +# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. +# MDAnalysis: A Python package for the rapid analysis of molecular dynamics +# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th +# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. +# +# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. +# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. +# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 +# +# + #cython: cdivision=True #cython: boundscheck=False From 919bdbaf2a9d9639496da4b3606429fa2942b688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 1 Jul 2018 16:53:22 +0200 Subject: [PATCH 05/30] Corrected C (de)allocation in c_gridsearch.pyx. Should fix memory leak --- package/MDAnalysis/lib/c_gridsearch.pyx | 32 +++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index ac73c97a5a2..2a7af4b33b7 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -145,16 +145,15 @@ cdef class PBCBox(object): # Update shift vectors self.c_pbcbox.ntric_vec = 0 - # We will only use single shifts, but we will check a few - # more shifts to see if there is a limiting distance - # above which we can not be sure of the correct distance. - for kk in range(5): + + # We will only use single shifts + for kk in range(3): k = order[kk] - for jj in range(5): + for jj in range(3): j = order[jj] - for ii in range(5): + for ii in range(3): i = order[ii] # A shift is only useful when it is trilinic @@ -177,8 +176,8 @@ cdef class PBCBox(object): else: pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) - d2old += sqrt(pos[d]) - d2new += sqrt(pos[d] + trial[d]) + d2old += pos[d]**2 + d2new += (pos[d] + trial[d])**2 if BOX_MARGIN*d2new < d2old: if not (j < -1 or j > 1 or k < -1 or k > 1): @@ -196,7 +195,7 @@ cdef class PBCBox(object): d2new_c = 0 for d in range(DIM): - d2new_c += sqrt(pos[d] + trial[d] - shift*box[dd, d]) + d2new_c += (pos[d] + trial[d] - shift*box[dd, d])**2 if d2new_c <= BOX_MARGIN*d2new: use = False @@ -209,7 +208,13 @@ cdef class PBCBox(object): % MAX_NTRICVEC) print(" There is probably something wrong with " "your box.") - print(box) + print(np.array(box)) + + for i in range(self.c_pbcbox.ntric_vec): + print(" -> shift #{}: [{}, {}, {}]".format(i+1, + self.c_pbcbox.tric_shift[i][XX], + self.c_pbcbox.tric_shift[i][YY], + self.c_pbcbox.tric_shift[i][ZZ])) else: for d in range(DIM): self.c_pbcbox.tric_vec[self.c_pbcbox.ntric_vec][d] = \ @@ -539,6 +544,9 @@ cdef class FastNS(object): cdef ns_grid *grid + def __cinit__(self): + self.grid = malloc(sizeof(ns_grid)) + def __init__(self, box): if box.shape != (3, 3): raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") @@ -554,11 +562,9 @@ cdef class FastNS(object): self.prepared = False - self.grid = malloc(sizeof(ns_grid)) - def __dealloc__(self): - #destroy_nsgrid(self.grid) + destroy_nsgrid(self.grid) self.grid.size = 0 def set_coords(self, real[:, ::1] coords): From 11f50d229f2b5b4a04c9c0f9348c555a288edac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 1 Jul 2018 16:54:21 +0200 Subject: [PATCH 06/30] Added Tests for grid search. Added grid in MDAnalysis.lib.__init__ --- package/MDAnalysis/lib/__init__.py | 3 +- .../MDAnalysisTests/lib/test_gridsearch.py | 81 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 testsuite/MDAnalysisTests/lib/test_gridsearch.py diff --git a/package/MDAnalysis/lib/__init__.py b/package/MDAnalysis/lib/__init__.py index 51c148dde65..a740bd8d5bf 100644 --- a/package/MDAnalysis/lib/__init__.py +++ b/package/MDAnalysis/lib/__init__.py @@ -29,7 +29,7 @@ from __future__ import absolute_import __all__ = ['log', 'transformations', 'util', 'mdamath', 'distances', - 'NeighborSearch', 'formats', 'pkdtree'] + 'NeighborSearch', 'formats', 'pkdtree', 'grid'] from . import log from . import transformations @@ -39,3 +39,4 @@ from . import NeighborSearch from . import formats from . import pkdtree +from . import grid \ No newline at end of file diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py new file mode 100644 index 00000000000..cab23f3f372 --- /dev/null +++ b/testsuite/MDAnalysisTests/lib/test_gridsearch.py @@ -0,0 +1,81 @@ +# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8 +# +# MDAnalysis --- https://www.mdanalysis.org +# Copyright (c) 2006-2018 The MDAnalysis Development Team and contributors +# (see the file AUTHORS for the full list of names) +# +# Released under the GNU Public Licence, v2 or any higher version +# +# Please cite your use of MDAnalysis in published work: +# +# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler, +# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. +# MDAnalysis: A Python package for the rapid analysis of molecular dynamics +# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th +# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. +# +# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. +# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. +# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 +# + +from __future__ import print_function, absolute_import + +import pytest +from numpy.testing import assert_equal +import numpy as np + +import MDAnalysis as mda +from MDAnalysis.lib import grid +from MDAnalysis.lib.pkdtree import PeriodicKDTree +from MDAnalysis.lib.mdamath import triclinic_vectors + +from MDAnalysisTests.datafiles import GRO + +@pytest.fixture +def universe(): + u = mda.Universe(GRO) + return u + +def run_search(universe, ref_id): + cutoff = 3 + + coords = universe.atoms.positions + ref_pos = coords[ref_id] + triclinic_box = triclinic_vectors(universe.dimensions) + + # Run pkdtree search + pkdt = PeriodicKDTree(universe.atoms.dimensions, bucket_size=10) + pkdt.set_coords(coords) + pkdt.search(ref_pos, cutoff) + + results_pkdtree = pkdt.get_indices() + results_pkdtree.remove(ref_id) + results_pkdtree = np.array(results_pkdtree) + results_pkdtree.sort() + + # Run grid search + searcher = grid.FastNS(triclinic_box) + searcher.set_cutoff(cutoff) + searcher.set_coords(coords) + searcher.prepare() + + results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0][0] + results_grid.sort() + + return results_pkdtree, results_grid + +def test_gridsearch(universe): + """Check that pkdtree and grid search return the same results (No PBC needed)""" + + ref_id = 0 + results_pkdtree, results_grid = run_search(universe, ref_id) + assert_equal(results_pkdtree, results_grid) + +def test_gridsearch_PBC(universe): + """Check that pkdtree and grid search return the same results (PBC needed)""" + + ref_id = 13937 + results_pkdtree, results_grid = run_search(universe, ref_id) + assert_equal(results_pkdtree, results_grid) From 9b8cacdd85b284f0eabeb9ff50ef4710b901d0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Wed, 4 Jul 2018 16:17:11 +0200 Subject: [PATCH 07/30] Removed pointer to nsgrid structure to avoid need to free it --- package/MDAnalysis/lib/c_gridsearch.pyx | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 2a7af4b33b7..7a6de80389d 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -42,8 +42,7 @@ DEF MAX_NTRICVEC=12 from libc.stdlib cimport malloc, realloc, free, abort from libc.stdio cimport fprintf, stderr -from libc.math cimport sqrt -from libc.math cimport abs as real_abs + import numpy as np cimport numpy as np @@ -541,11 +540,7 @@ cdef class FastNS(object): cdef real[:, ::1] coords_bbox cdef readonly real cutoff cdef bint prepared - cdef ns_grid *grid - - - def __cinit__(self): - self.grid = malloc(sizeof(ns_grid)) + cdef ns_grid grid def __init__(self, box): if box.shape != (3, 3): @@ -562,9 +557,11 @@ cdef class FastNS(object): self.prepared = False + self.grid.size = 0 + def __dealloc__(self): - destroy_nsgrid(self.grid) + destroy_nsgrid(&self.grid) self.grid.size = 0 def set_coords(self, real[:, ::1] coords): @@ -608,7 +605,7 @@ cdef class FastNS(object): self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] # Populating grid - if populate_grid(self.grid, self.coords_bbox) == RET_OK: + if populate_grid(&self.grid, self.coords_bbox) == RET_OK: initialization_ok = True @@ -632,7 +629,7 @@ cdef class FastNS(object): search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) with nogil: - holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) + holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) neighbors = [] ###Modify for distance From 1bc126c4de53b615a40c29b0e36156665d29a122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Wed, 4 Jul 2018 18:13:41 +0200 Subject: [PATCH 08/30] Removed abort() from c_gridsearch.pyx --- package/MDAnalysis/lib/c_gridsearch.pyx | 131 +++++++----------- .../MDAnalysisTests/lib/test_gridsearch.py | 2 +- 2 files changed, 48 insertions(+), 85 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 7a6de80389d..6008f0387dd 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -31,18 +31,19 @@ DEF DIM = 3 DEF XX = 0 DEF YY = 1 DEF ZZ = 2 + +DEF RET_ALLOCATIONERROR = 2 DEF RET_OK = 1 DEF RET_ERROR = 0 DEF EPSILON = 1e-5 + DEF NEIGHBORHOOD_ALLOCATION_INCREMENT = 50 DEF GRID_ALLOCATION_INCREMENT = 50 DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 -from libc.stdlib cimport malloc, realloc, free, abort -from libc.stdio cimport fprintf, stderr - +from libc.stdlib cimport malloc, realloc, free import numpy as np cimport numpy as np @@ -64,8 +65,7 @@ cdef struct ns_neighborhood: ns_int allocated_size ns_int size ns_int *beadids - real *beaddist - ### + cdef struct ns_neighborhood_holder: ns_int size ns_neighborhood **neighborhoods @@ -80,6 +80,7 @@ cdef void rvec_clear(rvec a) nogil: a[YY]=0.0 a[ZZ]=0.0 + cdef struct cPBCBox_t: matrix box rvec fbox_diag @@ -90,6 +91,8 @@ cdef struct cPBCBox_t: ns_int[DIM] tric_shift[MAX_NTRICVEC] real[DIM] tric_vec[MAX_NTRICVEC] + +# Class to handle PBC calculations cdef class PBCBox(object): cdef cPBCBox_t c_pbcbox cdef rvec center @@ -232,6 +235,7 @@ cdef class PBCBox(object): raise ValueError("Box does not correspond to PBC=xyz") self.fast_update(box) + cdef void fast_pbc_dx(self, rvec ref, rvec other, rvec dx) nogil: cdef ns_int i, j cdef rvec dx_start, trial @@ -284,20 +288,6 @@ cdef struct ns_grid: ns_int *nbeads ns_int **beadids -cdef ns_grid initialize_nsgrid(matrix box, - float cutoff) nogil: - cdef ns_grid grid - cdef ns_int i - - for i in range(DIM): - grid.ncells[i] = (box[i][i] / cutoff) - if grid.ncells[i] == 0: - grid.ncells[i] = 1 - grid.cellsize[i] = box[i][i] / grid.ncells[i] - - grid.size = grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ] - return grid - cdef ns_int populate_grid(ns_grid *grid, real[:,::1] coords) nogil: cdef ns_int ncoords = coords.shape[0] @@ -323,15 +313,12 @@ cdef ns_int populate_grid_array(ns_grid *grid, # Allocate memory grid.nbeads = malloc(sizeof(ns_int) * grid_size) if grid.nbeads == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS grid.nbeads (requested: %i bytes)\n", - sizeof(ns_int) * grid_size) - abort() + return RET_ALLOCATIONERROR allocated_size = malloc(sizeof(ns_int) * grid_size) if allocated_size == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS allocated_size (requested: %i bytes)\n", - sizeof(ns_int) * grid_size) - abort() + # No need to free grid.nbeads as it will be freed by destroy_nsgrid + return RET_ALLOCATIONERROR for i in range(grid_size): grid.nbeads[i] = 0 @@ -339,16 +326,12 @@ cdef ns_int populate_grid_array(ns_grid *grid, grid.beadids = malloc(sizeof(ns_int *) * grid_size) if grid.beadids == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids (requested: %i bytes)\n", - sizeof(ns_int *) * grid_size) - abort() + return RET_ALLOCATIONERROR for i in range(grid_size): grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) if grid.beadids[i] == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids[i] (requested: %i bytes)\n", - sizeof(ns_int) * allocated_size[i]) - abort() + return RET_ALLOCATIONERROR # Get cell indices for coords for i in range(ncoords): @@ -380,6 +363,8 @@ cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: cdef ns_neighborhood_holder *holder holder = malloc(sizeof(ns_neighborhood_holder)) + holder.size = 0 + holder.neighborhoods = NULL return holder @@ -393,7 +378,9 @@ cdef void free_neighborhood_holder(ns_neighborhood_holder *holder) nogil: if holder.neighborhoods[i].beadids != NULL: free(holder.neighborhoods[i].beadids) free(holder.neighborhoods[i]) - free(holder.neighborhoods) + + if holder.neighborhoods != NULL: + free(holder.neighborhoods) free(holder) cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]neighborcoords, ns_grid *grid, PBCBox box, real cutoff2) nogil: @@ -409,16 +396,15 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei cdef ns_neighborhood *neighborhood = malloc(sizeof(ns_neighborhood)) if neighborhood == NULL: - abort() + return NULL neighborhood.size = 0 neighborhood.allocated_size = NEIGHBORHOOD_ALLOCATION_INCREMENT neighborhood.beadids = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(ns_int)) - ###Modified here - neighborhood.beaddist = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(real)) - ### + if neighborhood.beadids == NULL: - abort() + free(neighborhood) + return NULL for zi in range(3): for yi in range(3): @@ -472,28 +458,23 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei # Update neighbor lists neighborhood.beadids[neighborhood.size] = bid - ### Modified here - neighborhood.beaddist[neighborhood.size] = d2 - ### neighborhood.size += 1 if neighborhood.size >= neighborhood.allocated_size: neighborhood.allocated_size += NEIGHBORHOOD_ALLOCATION_INCREMENT neighborhood.beadids = realloc( neighborhood.beadids, neighborhood.allocated_size * sizeof(ns_int)) - ###Modified here - neighborhood.beaddist = realloc( neighborhood.beaddist, neighborhood.allocated_size * sizeof(real)) - ### + if neighborhood.beadids == NULL: - abort() - ###Modified - if neighborhood.beaddist == NULL: - abort() - ### + free(neighborhood) + return NULL + # Register the cell as checked already_checked[nchecked] = cell_index nchecked += 1 return neighborhood + + cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, real[:, ::1] neighborcoords, ns_grid *grid, @@ -510,16 +491,12 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, holder = create_neighborhood_holder() if holder == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", - sizeof(ns_int) * ncoords) - abort() + return NULL - holder.size = ncoords holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) if holder.neighborhoods == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", - sizeof(ns_neighborhood) * ncoords) - abort() + free_neighborhood_holder(holder) + return NULL # Here starts the real core and the iteration over coordinates for coordid in range(ncoords): @@ -528,7 +505,12 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, grid, box, cutoff2) + if holder.neighborhoods[coordid] == NULL: + free_neighborhood_holder(holder) + return NULL + holder.neighborhoods[coordid].cutoff = cutoff + holder.size += 1 return holder @@ -580,7 +562,7 @@ cdef class FastNS(object): def prepare(self, force=False): - cdef ns_int i + cdef ns_int i, retcode cdef bint initialization_ok if self.prepared and not force: @@ -605,12 +587,11 @@ cdef class FastNS(object): self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] # Populating grid - if populate_grid(&self.grid, self.coords_bbox) == RET_OK: - initialization_ok = True - - - if initialization_ok: + retcode = populate_grid(&self.grid, self.coords_bbox) + if retcode == RET_OK: self.prepared = True + elif retcode == RET_ALLOCATIONERROR: + raise MemoryError("Could not allocate memory to initialize NS grid") else: raise RuntimeError("Could not initialize NS grid") @@ -631,45 +612,27 @@ cdef class FastNS(object): with nogil: holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) + if holder == NULL: + raise MemoryError("Could not allocate memory to run NS core") + neighbors = [] - ###Modify for distance - sqdist = [] - indx = [] - ### for nid in range(holder.size): neighborhood = holder.neighborhoods[nid] if return_ids: neighborhood_py = np.empty(neighborhood.size, dtype=np.int64) - ###Modify for distance - neighborhood_dis = np.empty(neighborhood.size, dtype=np.float32) - neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) - ### + for i in range(neighborhood.size): neighborhood_py[i] = neighborhood.beadids[i] - ###Modify for distance - neighborhood_dis[i] = neighborhood.beaddist[i] - ### else: neighborhood_py = np.empty((neighborhood.size, DIM), dtype=np.float32) - ###Modify for distance - neighborhood_dis = np.empty((neighborhood.size), dtype=np.float32) - neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) - ### for i in range(neighborhood.size): - ###Modify for distance - neighborhood_dis[i] = neighborhood.beaddist[i] - neighborhood_indx[i] = neighborhood.beadids[i] - ### - for j in range(DIM): neighborhood_py[i,j] = self.coords[neighborhood.beadids[i], j] neighbors.append(neighborhood_py) - sqdist.append(neighborhood_dis) - indx.append(neighborhood_indx) # Free Memory free_neighborhood_holder(holder) - return neighbors, sqdist, indx + return neighbors diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py index cab23f3f372..c8d526e9074 100644 --- a/testsuite/MDAnalysisTests/lib/test_gridsearch.py +++ b/testsuite/MDAnalysisTests/lib/test_gridsearch.py @@ -61,7 +61,7 @@ def run_search(universe, ref_id): searcher.set_coords(coords) searcher.prepare() - results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0][0] + results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0] results_grid.sort() return results_pkdtree, results_grid From 0a7843cb830d36c63400774ca008b6081861dd38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sat, 7 Jul 2018 10:30:10 +0200 Subject: [PATCH 09/30] grid allocation moved to FastNS method --- package/MDAnalysis/lib/c_gridsearch.pyx | 158 ++++++++++-------------- 1 file changed, 67 insertions(+), 91 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 6008f0387dd..98d97eded65 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -32,13 +32,9 @@ DEF XX = 0 DEF YY = 1 DEF ZZ = 2 -DEF RET_ALLOCATIONERROR = 2 -DEF RET_OK = 1 -DEF RET_ERROR = 0 DEF EPSILON = 1e-5 DEF NEIGHBORHOOD_ALLOCATION_INCREMENT = 50 -DEF GRID_ALLOCATION_INCREMENT = 50 DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 @@ -288,77 +284,6 @@ cdef struct ns_grid: ns_int *nbeads ns_int **beadids -cdef ns_int populate_grid(ns_grid *grid, - real[:,::1] coords) nogil: - cdef ns_int ncoords = coords.shape[0] - cdef bint ret_val - - ret_val = populate_grid_array(grid, - &coords[0, 0], - ncoords) - - return ret_val - -cdef ns_int populate_grid_array(ns_grid *grid, - rvec *coords, - ns_int ncoords) nogil: - cdef ns_int i, cellindex = -1 - cdef ns_int grid_size = grid.size - cdef ns_int *allocated_size = NULL - - if grid_size != grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ]: # Grid not initialized - return RET_ERROR - - - # Allocate memory - grid.nbeads = malloc(sizeof(ns_int) * grid_size) - if grid.nbeads == NULL: - return RET_ALLOCATIONERROR - - allocated_size = malloc(sizeof(ns_int) * grid_size) - if allocated_size == NULL: - # No need to free grid.nbeads as it will be freed by destroy_nsgrid - return RET_ALLOCATIONERROR - - for i in range(grid_size): - grid.nbeads[i] = 0 - allocated_size[i] = GRID_ALLOCATION_INCREMENT - - grid.beadids = malloc(sizeof(ns_int *) * grid_size) - if grid.beadids == NULL: - return RET_ALLOCATIONERROR - - for i in range(grid_size): - grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) - if grid.beadids[i] == NULL: - return RET_ALLOCATIONERROR - - # Get cell indices for coords - for i in range(ncoords): - cellindex = (coords[i][ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ - (coords[i][YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ - (coords[i][XX] / grid.cellsize[XX]) - - grid.beadids[cellindex][grid.nbeads[cellindex]] = i - grid.nbeads[cellindex] += 1 - - if grid.nbeads[cellindex] >= allocated_size[cellindex]: - allocated_size[cellindex] += GRID_ALLOCATION_INCREMENT - grid.beadids[cellindex] = realloc( grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) - free(allocated_size) - return RET_OK - -cdef void destroy_nsgrid(ns_grid *grid) nogil: - cdef ns_int i - if grid.nbeads != NULL: - free(grid.nbeads) - - for i in range(grid.size): - if grid.beadids[i] != NULL: - free(grid.beadids[i]) - free(grid.beadids) - - cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: cdef ns_neighborhood_holder *holder @@ -517,7 +442,6 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, # Python interface cdef class FastNS(object): cdef PBCBox box - cdef readonly int nthreads cdef readonly real[:, ::1] coords cdef real[:, ::1] coords_bbox cdef readonly real cutoff @@ -530,8 +454,6 @@ cdef class FastNS(object): self.box = PBCBox(box) - self.nthreads = 1 - self.coords = None self.coords_bbox = None @@ -543,7 +465,15 @@ cdef class FastNS(object): def __dealloc__(self): - destroy_nsgrid(&self.grid) + cdef ns_int i + if self.grid.nbeads != NULL: + free(self.grid.nbeads) + + for i in range(self.grid.size): + if self.grid.beadids[i] != NULL: + free(self.grid.beadids[i]) + free(self.grid.beadids) + self.grid.size = 0 def set_coords(self, real[:, ::1] coords): @@ -562,8 +492,11 @@ cdef class FastNS(object): def prepare(self, force=False): - cdef ns_int i, retcode - cdef bint initialization_ok + cdef ns_int i, cellindex = -1 + cdef ns_int *allocated_size = NULL + cdef ns_int ncoords = self.coords.shape[0] + cdef ns_int allocation_guess + cdef rvec *coords = &self.coords_bbox[0, 0] if self.prepared and not force: print("NS already prepared, nothing to do!") @@ -575,7 +508,6 @@ cdef class FastNS(object): raise ValueError("Cutoff must be set before NS preparation!") with nogil: - initialization_ok = False # Initializing grid for i in range(DIM): @@ -583,17 +515,61 @@ cdef class FastNS(object): if self.grid.ncells[i] == 0: self.grid.ncells[i] = 1 self.grid.cellsize[i] = self.box.c_pbcbox.box[i][i] / self.grid.ncells[i] - self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] - # Populating grid - retcode = populate_grid(&self.grid, self.coords_bbox) - if retcode == RET_OK: - self.prepared = True - elif retcode == RET_ALLOCATIONERROR: - raise MemoryError("Could not allocate memory to initialize NS grid") - else: - raise RuntimeError("Could not initialize NS grid") + # This is just a guess on how much memory we might for each grid cell: + # we just assume an average bead density and we take four times this density just to be safe + allocation_guess = (4 * (ncoords / self.grid.size + 1)) + + # Allocate memory for the grid + self.grid.nbeads = malloc(sizeof(ns_int) * self.grid.size) + if self.grid.nbeads == NULL: + with gil: + raise MemoryError("Could not allocate memory for NS grid") + + # Allocate memory from temporary allocation counter + allocated_size = malloc(sizeof(ns_int) * self.grid.size) + if allocated_size == NULL: + # No need to free grid.nbeads as it will be freed by destroy_nsgrid called by __dealloc___ + with gil: + raise MemoryError("Could not allocate memory for allocation buffer") + + # Pre-allocate some memory for grid cells + for i in range(self.grid.size): + self.grid.nbeads[i] = 0 + allocated_size[i] = allocation_guess + + self.grid.beadids = malloc(sizeof(ns_int *) * self.grid.size) + if self.grid.beadids == NULL: + with gil: + raise MemoryError("Could not allocate memory for grid cells") + + for i in range(self.grid.size): + self.grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) + if self.grid.beadids[i] == NULL: + with gil: + raise MemoryError("Could not allocate memory for grid cell") + + # Populate grid cells using the coordinates (ie do the heavy work) + for i in range(ncoords): + cellindex = (coords[i][ZZ] / self.grid.cellsize[ZZ]) * (self.grid.ncells[XX] * self.grid.ncells[YY]) +\ + (coords[i][YY] / self.grid.cellsize[YY]) * self.grid.ncells[XX] + \ + (coords[i][XX] / self.grid.cellsize[XX]) + + self.grid.beadids[cellindex][self.grid.nbeads[cellindex]] = i + self.grid.nbeads[cellindex] += 1 + + # We need to allocate more memory (simply double the amount of memory as + # 1. it should barely be needed + # 2. the size should stay fairly reasonable + if self.grid.nbeads[cellindex] >= allocated_size[cellindex]: + allocated_size[cellindex] *= 2 + self.grid.beadids[cellindex] = realloc( self.grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) + + # Now we can free the allocation buffer + free(allocated_size) + + self.prepared = True def search(self, real[:, ::1]search_coords, return_ids=False): From e07d64fb18e8d7c0a444893d86b7b5cb9a49af5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 8 Jul 2018 10:35:17 +0200 Subject: [PATCH 10/30] NSResults object added to store results from NS. Started documentation --- package/MDAnalysis/lib/c_gridsearch.pyx | 235 ++++++++++++++++-- .../source/documentation_pages/lib/grid.rst | 2 + .../documentation_pages/lib_modules.rst | 2 + .../MDAnalysisTests/lib/test_gridsearch.py | 79 +++++- 4 files changed, 282 insertions(+), 36 deletions(-) create mode 100644 package/doc/sphinx/source/documentation_pages/lib/grid.rst diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 98d97eded65..9c50a5b1777 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -25,6 +25,15 @@ #cython: cdivision=True #cython: boundscheck=False +#cython: initializedcheck=False + +""" +Neighbor search library --- :mod:`MDAnalysis.lib.grid` +====================================================== + +This Neighbor search library is a serialized Cython port of the NS grid search implemented in GROMACS. +""" + # Preprocessor DEFs DEF DIM = 3 @@ -40,6 +49,7 @@ DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 from libc.stdlib cimport malloc, realloc, free +from libc.math cimport sqrt import numpy as np cimport numpy as np @@ -248,16 +258,21 @@ cdef class PBCBox(object): for j in range (i, -1, -1): dx[j] += self.c_pbcbox.box[i][j] + cdef real fast_distance2(self, rvec a, rvec b) nogil: + cdef rvec dx + self.fast_pbc_dx(a, b, dx) + return rvec_norm2(dx) + + cdef real fast_distance(self, rvec a, rvec b) nogil: + return sqrt(self.fast_distance2(a,b)) + cdef real[:, ::1]fast_put_atoms_in_bbox(self, real[:,::1] coords) nogil: cdef ns_int i, m, d, natoms, wd = 0 cdef real[:,::1] bbox_coords natoms = coords.shape[0] with gil: - if natoms == 0: - bbox_coords = np.empty((0, DIM)) - else: - bbox_coords = coords.copy() + bbox_coords = coords.copy() for i in range(natoms): for m in range(DIM - 1, -1, -1): @@ -270,6 +285,8 @@ cdef class PBCBox(object): return bbox_coords def put_atoms_in_bbox(self, real[:,::1] coords): + if coords.shape[0] == 0: + return np.zeros((0, DIM), dtype=np.float32) return np.asarray(self.fast_put_atoms_in_bbox(coords)) ######################################################################################################################## @@ -439,6 +456,167 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, return holder +cdef class NSResults(object): + """ + Class used to store results returned by `MDAnalysis.lib.grid.FastNS.search` + """ + cdef PBCBox box + cdef readonly real cutoff + cdef real[:, ::1] grid_coords + cdef real[:, ::1] ref_coords + cdef ns_int **nids + cdef ns_int *nsizes + cdef ns_int size + cdef list indices + cdef list coordinates + cdef list distances + + def __init__(self, PBCBox box, real cutoff): + self.box = box + self.cutoff = cutoff + + self.size = 0 + self.nids = NULL + self.nsizes = NULL + + self.grid_coords = None + self.ref_coords = None + + self.indices = None + self.coordinates = None + self.distances = None + + + cdef populate(self, ns_neighborhood_holder *holder, grid_coords, ref_coords): + cdef ns_int nid, i + cdef ns_neighborhood *neighborhood + + self.grid_coords = grid_coords.copy() + self.ref_coords = ref_coords.copy() + + # Allocate memory + self.nsizes = malloc(sizeof(ns_int) * holder.size) + if self.nsizes == NULL: + raise MemoryError("Could not allocate memory for NSResults") + + self.nids = malloc(sizeof(ns_int *) * holder.size) + if self.nids == NULL: + raise MemoryError("Could not allocate memory for NSResults") + + for nid in range(holder.size): + neighborhood = holder.neighborhoods[nid] + + self.nsizes[nid] = neighborhood.size + + self.nids[nid] = malloc(sizeof(ns_int *) * neighborhood.size) + if self.nids[nid] == NULL: + raise MemoryError("Could not allocate memory for NSResults") + + with nogil: + for nid in range(holder.size): + neighborhood = holder.neighborhoods[nid] + + for i in range(neighborhood.size): + self.nids[nid][i] = neighborhood.beadids[i] + + self.size = holder.size + + def __dealloc__(self): + if self.nids != NULL: + for i in range(self.size): + if self.nids[i] != NULL: + free(self.nids[i]) + free(self.nids) + + if self.nsizes != NULL: + free(self.nsizes) + + + def get_indices(self): + """ + Return Neighbors indices. + + :return: list of indices + """ + cdef ns_int i, nid, size + + if self.indices is None: + indices = [] + + for nid in range(self.size): + size = self.nsizes[nid] + + tmp_incides = np.empty((size), dtype=np.int) + + for i in range(size): + tmp_incides[i] = self.nids[nid][i] + + indices.append(tmp_incides) + + self.indices = indices + + return self.indices + + + def get_coordinates(self): + """ + Return coordinates of neighbors. + + :return: list of coordinates + """ + cdef ns_int i, nid, size, beadid + + if self.coordinates is None: + coordinates = [] + + for nid in range(self.size): + size = self.nsizes[nid] + + tmp_values = np.empty((size, DIM), dtype=np.float32) + + for i in range(size): + beadid = self.nids[nid][i] + tmp_values[i] = self.grid_coords[beadid] + + coordinates.append(tmp_values) + + self.coordinates = coordinates + + return self.coordinates + + + def get_distances(self): + """ + Return coordinates of neighbors. + + :return: list of distances + """ + cdef ns_int i, nid, size, j, beadid + cdef rvec ref, other, dx + cdef real dist + + if self.distances is None: + distances = [] + + for nid in range(self.size): + size = self.nsizes[nid] + + tmp_values = np.empty((size), dtype=np.float32) + ref = &self.ref_coords[nid, 0] + + for i in range(size): + beadid = self.nids[nid][i] + other = &self.grid_coords[beadid, 0] + + tmp_values[i] = self.box.fast_distance(ref, other) + + distances.append(tmp_values) + + self.distances = distances + + return self.distances + + # Python interface cdef class FastNS(object): cdef PBCBox box @@ -448,7 +626,14 @@ cdef class FastNS(object): cdef bint prepared cdef ns_grid grid - def __init__(self, box): + def __init__(self, u): + import MDAnalysis as mda + from MDAnalysis.lib.mdamath import triclinic_vectors + + if not isinstance(u, mda.Universe): + raise TypeError("FastNS class must be initialized with a valid MDAnalysis.Universe instance") + box = triclinic_vectors(u.dimensions) + if box.shape != (3, 3): raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") @@ -466,6 +651,7 @@ cdef class FastNS(object): def __dealloc__(self): cdef ns_int i + # Deallocate NS grid if self.grid.nbeads != NULL: free(self.grid.nbeads) @@ -475,7 +661,8 @@ cdef class FastNS(object): free(self.grid.beadids) self.grid.size = 0 - + + def set_coords(self, real[:, ::1] coords): self.coords = coords @@ -568,12 +755,12 @@ cdef class FastNS(object): # Now we can free the allocation buffer free(allocated_size) - self.prepared = True - def search(self, real[:, ::1]search_coords, return_ids=False): + def search(self, search_coords): cdef real[:, ::1] search_coords_bbox + cdef real[:, ::1] search_coords_view cdef ns_int nid, i, j cdef ns_neighborhood_holder *holder cdef ns_neighborhood *neighborhood @@ -581,9 +768,18 @@ cdef class FastNS(object): if not self.prepared: self.prepare() + # Check the shape of search_coords as a array of 3D coords if needed + shape = search_coords.shape + if len(shape) == 1: + if not shape[0] == 3: + raise ValueError("Coordinates must be 3D") + else: + search_coords_view = np.array([search_coords,], dtype=np.float32) + else: + search_coords_view = search_coords # Make sure atoms are inside the brick-shaped box - search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) + search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords_view) with nogil: holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) @@ -591,24 +787,11 @@ cdef class FastNS(object): if holder == NULL: raise MemoryError("Could not allocate memory to run NS core") - neighbors = [] - for nid in range(holder.size): - neighborhood = holder.neighborhoods[nid] - if return_ids: - neighborhood_py = np.empty(neighborhood.size, dtype=np.int64) - - for i in range(neighborhood.size): - neighborhood_py[i] = neighborhood.beadids[i] - else: - neighborhood_py = np.empty((neighborhood.size, DIM), dtype=np.float32) - for i in range(neighborhood.size): - for j in range(DIM): - neighborhood_py[i,j] = self.coords[neighborhood.beadids[i], j] - - neighbors.append(neighborhood_py) + results = NSResults(self.box, self.cutoff) + results.populate(holder, self.coords, search_coords_view) - # Free Memory + # Free memory allocated to holder free_neighborhood_holder(holder) - return neighbors + return results diff --git a/package/doc/sphinx/source/documentation_pages/lib/grid.rst b/package/doc/sphinx/source/documentation_pages/lib/grid.rst new file mode 100644 index 00000000000..a6143e9ca38 --- /dev/null +++ b/package/doc/sphinx/source/documentation_pages/lib/grid.rst @@ -0,0 +1,2 @@ +.. automodule:: MDAnalysis.lib.grid + :members: \ No newline at end of file diff --git a/package/doc/sphinx/source/documentation_pages/lib_modules.rst b/package/doc/sphinx/source/documentation_pages/lib_modules.rst index 5bbe3d041ed..83d7f81cfd9 100644 --- a/package/doc/sphinx/source/documentation_pages/lib_modules.rst +++ b/package/doc/sphinx/source/documentation_pages/lib_modules.rst @@ -36,6 +36,7 @@ functions whereas mathematical functions are to be found in :mod:`MDAnalysis.lib.NeighborSearch` contains classes to do neighbor searches with MDAnalysis objects. +:mod:`MDAnalysis.lib.grid` contains a fast implementation of grid neighbor search. List of modules --------------- @@ -50,6 +51,7 @@ List of modules ./lib/transformations ./lib/qcprot ./lib/util + ./lib/grid Low level file formats ---------------------- diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py index c8d526e9074..359d32544b7 100644 --- a/testsuite/MDAnalysisTests/lib/test_gridsearch.py +++ b/testsuite/MDAnalysisTests/lib/test_gridsearch.py @@ -23,27 +23,46 @@ from __future__ import print_function, absolute_import import pytest -from numpy.testing import assert_equal +from numpy.testing import assert_equal, assert_allclose import numpy as np import MDAnalysis as mda from MDAnalysis.lib import grid from MDAnalysis.lib.pkdtree import PeriodicKDTree -from MDAnalysis.lib.mdamath import triclinic_vectors from MDAnalysisTests.datafiles import GRO + @pytest.fixture def universe(): u = mda.Universe(GRO) return u + +@pytest.fixture +def grid_results(): + u = mda.Universe(GRO) + cutoff = 2 + ref_pos = u.atoms.positions[13937] + return run_grid_search(u, ref_pos, cutoff) + + +def run_grid_search(u, ref_pos, cutoff): + coords = u.atoms.positions + + # Run grid search + searcher = grid.FastNS(u) + searcher.set_cutoff(cutoff) + searcher.set_coords(coords) + searcher.prepare() + + return searcher.search(ref_pos) + + def run_search(universe, ref_id): cutoff = 3 - coords = universe.atoms.positions ref_pos = coords[ref_id] - triclinic_box = triclinic_vectors(universe.dimensions) # Run pkdtree search pkdt = PeriodicKDTree(universe.atoms.dimensions, bucket_size=10) @@ -56,16 +75,13 @@ def run_search(universe, ref_id): results_pkdtree.sort() # Run grid search - searcher = grid.FastNS(triclinic_box) - searcher.set_cutoff(cutoff) - searcher.set_coords(coords) - searcher.prepare() - - results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0] + results_grid = run_grid_search(universe, ref_pos, cutoff) + results_grid = results_grid.get_indices()[0] results_grid.sort() return results_pkdtree, results_grid + def test_gridsearch(universe): """Check that pkdtree and grid search return the same results (No PBC needed)""" @@ -73,9 +89,52 @@ def test_gridsearch(universe): results_pkdtree, results_grid = run_search(universe, ref_id) assert_equal(results_pkdtree, results_grid) + def test_gridsearch_PBC(universe): """Check that pkdtree and grid search return the same results (PBC needed)""" ref_id = 13937 results_pkdtree, results_grid = run_search(universe, ref_id) assert_equal(results_pkdtree, results_grid) + + +def test_gridsearch_arraycoord(universe): + """Check the NS routine accepts a single bead coordinate as well as array of coordinates""" + cutoff = 2 + ref_pos = universe.atoms.positions[:5] + + results = [ + np.array([2, 1, 4, 3]), + np.array([2, 0, 3]), + np.array([0, 1, 3]), + np.array([ 2, 0, 1, 38341]), + np.array([ 6, 0, 5, 17]) + ] + + results_grid = run_grid_search(universe, ref_pos, cutoff).get_indices() + + assert_equal(results_grid, results) + + +def test_gridsearch_search_coordinates(grid_results): + """Check the NS routine can return coordinates instead of ids""" + + results = np.array( + [ + [40.32, 34.25, 55.9], + [0.61, 76.33, -0.56], + [0.48999998, 75.9, 0.19999999], + [-0.11, 76.19, 0.77] + ]) + + assert_allclose(grid_results.get_coordinates()[0], results) + + +def test_gridsearch_search_distances(grid_results): + """Check the NS routine can return PBC distances from neighbors""" + results = np.array([0.096, 0.096, 0.015, 0.179]) * 10 # These distances were obtained using gmx distance + results.sort() + + rounded_results = np.round(grid_results.get_distances()[0], 2) + + assert_allclose(sorted(rounded_results), results) \ No newline at end of file From 6e0ceda1901e1d768ea1ed457370fb5ea1033ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 8 Jul 2018 20:10:33 +0200 Subject: [PATCH 11/30] Corrected Memory Leak --- package/MDAnalysis/lib/c_gridsearch.pyx | 36 ++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 9c50a5b1777..1ece4c5a1d6 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -462,8 +462,8 @@ cdef class NSResults(object): """ cdef PBCBox box cdef readonly real cutoff - cdef real[:, ::1] grid_coords - cdef real[:, ::1] ref_coords + cdef np.ndarray grid_coords + cdef np.ndarray ref_coords cdef ns_int **nids cdef ns_int *nsizes cdef ns_int size @@ -471,6 +471,16 @@ cdef class NSResults(object): cdef list coordinates cdef list distances + def __dealloc__(self): + if self.nids != NULL: + for i in range(self.size): + if self.nids[i] != NULL: + free(self.nids[i]) + free(self.nids) + + if self.nsizes != NULL: + free(self.nsizes) + def __init__(self, PBCBox box, real cutoff): self.box = box self.cutoff = cutoff @@ -491,8 +501,8 @@ cdef class NSResults(object): cdef ns_int nid, i cdef ns_neighborhood *neighborhood - self.grid_coords = grid_coords.copy() - self.ref_coords = ref_coords.copy() + self.grid_coords = np.asarray(grid_coords) + self.ref_coords = np.asarray(ref_coords) # Allocate memory self.nsizes = malloc(sizeof(ns_int) * holder.size) @@ -521,17 +531,6 @@ cdef class NSResults(object): self.size = holder.size - def __dealloc__(self): - if self.nids != NULL: - for i in range(self.size): - if self.nids[i] != NULL: - free(self.nids[i]) - free(self.nids) - - if self.nsizes != NULL: - free(self.nsizes) - - def get_indices(self): """ Return Neighbors indices. @@ -594,6 +593,8 @@ cdef class NSResults(object): cdef ns_int i, nid, size, j, beadid cdef rvec ref, other, dx cdef real dist + cdef real[:, ::1] ref_coords = self.ref_coords + cdef real[:, ::1] grid_coords = self.grid_coords if self.distances is None: distances = [] @@ -602,11 +603,11 @@ cdef class NSResults(object): size = self.nsizes[nid] tmp_values = np.empty((size), dtype=np.float32) - ref = &self.ref_coords[nid, 0] + ref = &ref_coords[nid, 0] for i in range(size): beadid = self.nids[nid][i] - other = &self.grid_coords[beadid, 0] + other = &grid_coords[beadid, 0] tmp_values[i] = self.box.fast_distance(ref, other) @@ -787,7 +788,6 @@ cdef class FastNS(object): if holder == NULL: raise MemoryError("Could not allocate memory to run NS core") - results = NSResults(self.box, self.cutoff) results.populate(holder, self.coords, search_coords_view) From fe780bc44d0828f60d94f9bc1b5e5f1d4d08ccdd Mon Sep 17 00:00:00 2001 From: ayush Date: Mon, 25 Jun 2018 20:10:22 -0700 Subject: [PATCH 12/30] Added MDAnalysis/lib/c_search.pyx for Grid NS search --- package/MDAnalysis/lib/c_gridsearch.pyx | 747 ++++++++++++++++++++++++ package/setup.py | 7 + 2 files changed, 754 insertions(+) create mode 100644 package/MDAnalysis/lib/c_gridsearch.pyx diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx new file mode 100644 index 00000000000..82d7192544a --- /dev/null +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -0,0 +1,747 @@ +# -*- coding: utf-8; Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 +# +# Copyright (C) 2013-2016 Sébastien Buchoux +# +# This file is part of FATSLiM. +# +# FATSLiM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FATSLiM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FATSLiM. If not, see . +#cython: cdivision=True +#cython: boundscheck=False + +# Preprocessor DEFs +DEF DIM = 3 +DEF XX = 0 +DEF YY = 1 +DEF ZZ = 2 +DEF RET_OK = 1 +DEF RET_ERROR = 0 +DEF EPSILON = 1e-5 +DEF NEIGHBORHOOD_ALLOCATION_INCREMENT = 50 +DEF GRID_ALLOCATION_INCREMENT = 50 + +DEF BOX_MARGIN=1.0010 +DEF MAX_NTRICVEC=12 + + +# Cython C imports (no Python here!) +from cython.parallel cimport prange +from libc.stdlib cimport malloc, realloc, free, abort +from libc.stdio cimport fprintf, stderr +from libc.math cimport sqrt +from libc.math cimport abs as real_abs + +cimport openmp + + +# Python imports +import numpy as np +cimport numpy as np + +# Ctypes +ctypedef np.int_t ns_int +ctypedef np.float32_t real +ctypedef real rvec[DIM] +ctypedef real matrix[DIM][DIM] + +cdef struct ns_grid: + ns_int size + ns_int[DIM] ncells + real[DIM] cellsize + ns_int *nbeads + ns_int **beadids + +cdef struct ns_neighborhood: + real cutoff + ns_int allocated_size + ns_int size + ns_int *beadids + real *beaddist + ### +cdef struct ns_neighborhood_holder: + ns_int size + ns_neighborhood **neighborhoods + +# Useful stuff + +cdef real rvec_norm2(const rvec a) nogil: + return a[XX]*a[XX]+a[YY]*a[YY]+a[ZZ]*a[ZZ] + +cdef void rvec_clear(rvec a) nogil: + a[XX]=0.0 + a[YY]=0.0 + a[ZZ]=0.0 + +cdef struct cPBCBox_t: + matrix box + rvec fbox_diag + rvec hbox_diag + rvec mhbox_diag + real max_cutoff2 + ns_int ntric_vec + ns_int[DIM] tric_shift[MAX_NTRICVEC] + real[DIM] tric_vec[MAX_NTRICVEC] + +# noinspection PyNoneFunctionAssignment +cdef class PBCBox(object): + cdef cPBCBox_t c_pbcbox + cdef rvec center + cdef rvec bbox_center + + def __init__(self, real[:,::1] box): + self.update(box) + + cdef void fast_update(self, real[:,::1] box) nogil: + cdef ns_int i, j, k, d, jc, kc, shift + cdef real d2old, d2new, d2new_c + cdef rvec trial, pos + cdef ns_int ii, jj ,kk + cdef ns_int *order = [0, -1, 1, -2, 2] + cdef bint use + cdef real min_hv2, min_ss, tmp + + rvec_clear(self.center) + # Update matrix + for i in range(DIM): + for j in range(DIM): + self.c_pbcbox.box[i][j] = box[i, j] + self.center[j] += 0.5 * box[i, j] + self.bbox_center[i] = 0.5 * box[i, i] + + # Update diagonals + for i in range(DIM): + self.c_pbcbox.fbox_diag[i] = box[i, i] + self.c_pbcbox.hbox_diag[i] = self.c_pbcbox.fbox_diag[i] * 0.5 + self.c_pbcbox.mhbox_diag[i] = - self.c_pbcbox.hbox_diag[i] + + # Update maximum cutoff + + # Physical limitation of the cut-off + # by half the length of the shortest box vector. + min_hv2 = min(0.25 * rvec_norm2(&box[XX, XX]), 0.25 * rvec_norm2(&box[YY, XX])) + min_hv2 = min(min_hv2, 0.25 * rvec_norm2(&box[ZZ, XX])) + + # Limitation to the smallest diagonal element due to optimizations: + # checking only linear combinations of single box-vectors (2 in x) + # in the grid search and pbc_dx is a lot faster + # than checking all possible combinations. + tmp = box[YY, YY] + if box[ZZ, YY] < 0: + tmp -= box[ZZ, YY] + else: + tmp += box[ZZ, YY] + + min_ss = min(box[XX, XX], min(tmp, box[ZZ, ZZ])) + + self.c_pbcbox.max_cutoff2 = min(min_hv2, min_ss * min_ss) + + # Update shift vectors + self.c_pbcbox.ntric_vec = 0 + # We will only use single shifts, but we will check a few + # more shifts to see if there is a limiting distance + # above which we can not be sure of the correct distance. + for kk in range(5): + k = order[kk] + + for jj in range(5): + j = order[jj] + + for ii in range(5): + i = order[ii] + + # A shift is only useful when it is trilinic + if j != 0 or k != 0: + d2old = 0 + d2new = 0 + + for d in range(DIM): + trial[d] = i*box[XX, d] + j*box[YY, d] + k*box[ZZ, d] + + # Choose the vector within the brick around 0,0,0 that + # will become the shortest due to shift try. + + if d == DIM: + trial[d] = 0 + pos[d] = 0 + else: + if trial[d] < 0: + pos[d] = min(self.c_pbcbox.hbox_diag[d], -trial[d]) + else: + pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) + + d2old += sqrt(pos[d]) + d2new += sqrt(pos[d] + trial[d]) + + if BOX_MARGIN*d2new < d2old: + if not (j < -1 or j > 1 or k < -1 or k > 1): + use = True + + for dd in range(DIM): + if dd == 0: + shift = i + elif dd == 1: + shift = j + else: + shift = k + + if shift: + d2new_c = 0 + + for d in range(DIM): + d2new_c += sqrt(pos[d] + trial[d] - shift*box[dd, d]) + + if d2new_c <= BOX_MARGIN*d2new: + use = False + + if use: # Accept this shift vector. + if self.c_pbcbox.ntric_vec >= MAX_NTRICVEC: + with gil: + print("\nWARNING: Found more than %d triclinic " + "correction vectors, ignoring some." + % MAX_NTRICVEC) + print(" There is probably something wrong with " + "your box.") + print(box) + else: + for d in range(DIM): + self.c_pbcbox.tric_vec[self.c_pbcbox.ntric_vec][d] = \ + trial[d] + self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][XX] = i + self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][YY] = j + self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][ZZ] = k + self.c_pbcbox.ntric_vec += 1 + + + def update(self, real[:,::1] box): + if box.shape[0] != DIM or box.shape[1] != DIM: + raise ValueError("Box must be a %i x %i matrix. (shape: %i x %i)" % + (DIM, DIM, box.shape[0], box.shape[1])) + if (box[XX, XX] == 0) or (box[YY, YY] == 0) or (box[ZZ, ZZ] == 0): + raise ValueError("Box does not correspond to PBC=xyz") + self.fast_update(box) + + cdef void fast_pbc_dx(self, rvec ref, rvec other, rvec dx) nogil: + cdef ns_int i, j + cdef rvec dx_start, trial + + for i in range(DIM): + dx[i] = other[i] - ref[i] + + for i in range (DIM-1, -1, -1): + while dx[i] > self.c_pbcbox.hbox_diag[i]: + for j in range (i, -1, -1): + dx[j] -= self.c_pbcbox.box[i][j] + + while dx[i] <= self.c_pbcbox.mhbox_diag[i]: + for j in range (i, -1, -1): + dx[j] += self.c_pbcbox.box[i][j] + + cdef real[:, ::1]fast_put_atoms_in_bbox(self, real[:,::1] coords) nogil: + cdef ns_int i, m, d, natoms, wd = 0 + cdef real[:,::1] bbox_coords + + natoms = coords.shape[0] + with gil: + if natoms == 0: + bbox_coords = np.empty((0, DIM)) + else: + bbox_coords = coords.copy() + + for i in range(natoms): + for m in range(DIM - 1, -1, -1): + while bbox_coords[i, m] < 0: + for d in range(m+1): + bbox_coords[i, d] += self.c_pbcbox.box[m][d] + while bbox_coords[i, m] >= self.c_pbcbox.box[m][m]: + for d in range(m+1): + bbox_coords[i, d] -= self.c_pbcbox.box[m][d] + return bbox_coords + + def put_atoms_in_bbox(self, real[:,::1] coords): + return np.asarray(self.fast_put_atoms_in_bbox(coords)) + + + +######################################################################################################################## +# +# Neighbor Search Stuff +# +######################################################################################################################## +cdef struct ns_grid: + ns_int size + ns_int[DIM] ncells + real[DIM] cellsize + ns_int *nbeads + ns_int **beadids + +cdef ns_grid initialize_nsgrid(matrix box, + float cutoff) nogil: + cdef ns_grid grid + cdef ns_int i + + for i in range(DIM): + grid.ncells[i] = (box[i][i] / cutoff) + if grid.ncells[i] == 0: + grid.ncells[i] = 1 + grid.cellsize[i] = box[i][i] / grid.ncells[i] + + grid.size = grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ] + return grid + +cdef ns_int populate_grid(ns_grid *grid, + real[:,::1] coords) nogil: + cdef ns_int ncoords = coords.shape[0] + cdef bint ret_val + + ret_val = populate_grid_array(grid, + &coords[0, 0], + ncoords) + + return ret_val + +cdef ns_int populate_grid_array(ns_grid *grid, + rvec *coords, + ns_int ncoords) nogil: + cdef ns_int i, cellindex = -1 + cdef ns_int grid_size = grid.size + cdef ns_int *allocated_size = NULL + + if grid_size != grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ]: # Grid not initialized + return RET_ERROR + + + # Allocate memory + grid.nbeads = malloc(sizeof(ns_int) * grid_size) + if grid.nbeads == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS grid.nbeads (requested: %i bytes)\n", + sizeof(ns_int) * grid_size) + abort() + + allocated_size = malloc(sizeof(ns_int) * grid_size) + if allocated_size == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS allocated_size (requested: %i bytes)\n", + sizeof(ns_int) * grid_size) + abort() + + for i in range(grid_size): + grid.nbeads[i] = 0 + allocated_size[i] = GRID_ALLOCATION_INCREMENT + + grid.beadids = malloc(sizeof(ns_int *) * grid_size) + if grid.beadids == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids (requested: %i bytes)\n", + sizeof(ns_int *) * grid_size) + abort() + + for i in range(grid_size): + grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) + if grid.beadids[i] == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids[i] (requested: %i bytes)\n", + sizeof(ns_int) * allocated_size[i]) + abort() + + # Get cell indices for coords + for i in range(ncoords): + cellindex = (coords[i][ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ + (coords[i][YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ + (coords[i][XX] / grid.cellsize[XX]) + + grid.beadids[cellindex][grid.nbeads[cellindex]] = i + grid.nbeads[cellindex] += 1 + + if grid.nbeads[cellindex] >= allocated_size[cellindex]: + allocated_size[cellindex] += GRID_ALLOCATION_INCREMENT + grid.beadids[cellindex] = realloc( grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) + free(allocated_size) + return RET_OK + +cdef void destroy_nsgrid(ns_grid *grid) nogil: + cdef ns_int i + if grid.nbeads != NULL: + free(grid.nbeads) + + for i in range(grid.size): + if grid.beadids[i] != NULL: + free(grid.beadids[i]) + free(grid.beadids) + + +cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: + cdef ns_neighborhood_holder *holder + + holder = malloc(sizeof(ns_neighborhood_holder)) + + return holder + +cdef void free_neighborhood_holder(ns_neighborhood_holder *holder) nogil: + cdef ns_int i + + if holder == NULL: + return + + for i in range(holder.size): + if holder.neighborhoods[i].beadids != NULL: + free(holder.neighborhoods[i].beadids) + free(holder.neighborhoods[i]) + free(holder.neighborhoods) + free(holder) + +cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]neighborcoords, ns_grid *grid, PBCBox box, real cutoff2) nogil: + cdef ns_int d, m + cdef ns_int xi, yi, zi, bid + cdef real d2 + cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords + + cdef bint already_checked[27] + cdef bint skip + cdef ns_int nchecked = 0, icheck + cdef ns_int cell_index + + cdef ns_neighborhood *neighborhood = malloc(sizeof(ns_neighborhood)) + if neighborhood == NULL: + abort() + + neighborhood.size = 0 + neighborhood.allocated_size = NEIGHBORHOOD_ALLOCATION_INCREMENT + neighborhood.beadids = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(ns_int)) + ###Modified here + neighborhood.beaddist = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(real)) + ### + if neighborhood.beadids == NULL: + abort() + + for zi in range(3): + for yi in range(3): + for xi in range(3): + # Calculate and/or reinitialize shifted coordinates + shifted_coord[XX] = current_coords[XX] + (xi - 1) * grid.cellsize[XX] + shifted_coord[YY] = current_coords[YY] + (yi - 1) * grid.cellsize[YY] + shifted_coord[ZZ] = current_coords[ZZ] + (zi - 1) * grid.cellsize[ZZ] + + # Make sure the shifted coordinates is inside the brick-shaped box + for m in range(DIM - 1, -1, -1): + + while shifted_coord[m] < 0: + for d in range(m+1): + shifted_coord[d] += box.c_pbcbox.box[m][d] + + + while shifted_coord[m] >= box.c_pbcbox.box[m][m]: + for d in range(m+1): + shifted_coord[d] -= box.c_pbcbox.box[m][d] + + # Get the cell index corresponding to the coord + cell_index = (shifted_coord[ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ + (shifted_coord[YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ + (shifted_coord[XX] / grid.cellsize[XX]) + + # Just a safeguard + if cell_index >= grid.size: + continue + + # Check the cell index was not already selected + skip = False + for icheck in range(nchecked): + if already_checked[icheck] == cell_index: + skip = True + break + if skip: + continue + + # Search for neighbors inside this cell + for i_bead in range(grid.nbeads[cell_index]): + bid = grid.beadids[cell_index][i_bead] + + box.fast_pbc_dx(current_coords, &neighborcoords[bid, XX], dx) + + d2 = rvec_norm2(dx) + + if d2 < cutoff2: + if d2 < EPSILON: # Don't add the current bead as its own neighbor! + continue + + # Update neighbor lists + neighborhood.beadids[neighborhood.size] = bid + ### Modified here + neighborhood.beaddist[neighborhood.size] = d2 + ### + neighborhood.size += 1 + + if neighborhood.size >= neighborhood.allocated_size: + neighborhood.allocated_size += NEIGHBORHOOD_ALLOCATION_INCREMENT + neighborhood.beadids = realloc( neighborhood.beadids, neighborhood.allocated_size * sizeof(ns_int)) + ###Modified here + neighborhood.beaddist = realloc( neighborhood.beaddist, neighborhood.allocated_size * sizeof(real)) + ### + if neighborhood.beadids == NULL: + abort() + ###Modified + if neighborhood.beaddist == NULL: + abort() + ### + # Register the cell as checked + already_checked[nchecked] = cell_index + nchecked += 1 + + return neighborhood + + +cdef ns_neighborhood_holder *ns_core_parallel(real[:, ::1] refcoords, + real[:, ::1] neighborcoords, + ns_grid *grid, + PBCBox box, + real cutoff, + int nthreads=-1) nogil: + cdef ns_int coordid, i, j + cdef ns_int ncoords = refcoords.shape[0] + cdef ns_int ncoords_neighbors = neighborcoords.shape[0] + cdef real cutoff2 = cutoff * cutoff + cdef ns_neighborhood_holder *holder + + cdef ns_int *neighbor_buf + cdef ns_int buf_size, ibuf + + if nthreads < 0: + nthreads = openmp.omp_get_num_threads() + + holder = create_neighborhood_holder() + if holder == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", + sizeof(ns_int) * ncoords) + abort() + + holder.size = ncoords + holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) + if holder.neighborhoods == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", + sizeof(ns_neighborhood) * ncoords) + abort() + + # Here starts the real core and the iteration over coordinates + for coordid in prange(ncoords, schedule='dynamic', num_threads=nthreads): + holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], + neighborcoords, + grid, + box, + cutoff2) + holder.neighborhoods[coordid].cutoff = cutoff + + return holder + +cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, + real[:, ::1] neighborcoords, + ns_grid *grid, + PBCBox box, + real cutoff) nogil: + cdef ns_int coordid, i, j + cdef ns_int ncoords = refcoords.shape[0] + cdef ns_int ncoords_neighbors = neighborcoords.shape[0] + cdef real cutoff2 = cutoff * cutoff + cdef ns_neighborhood_holder *holder + + cdef ns_int *neighbor_buf + cdef ns_int buf_size, ibuf + + holder = create_neighborhood_holder() + if holder == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", + sizeof(ns_int) * ncoords) + abort() + + holder.size = ncoords + holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) + if holder.neighborhoods == NULL: + fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", + sizeof(ns_neighborhood) * ncoords) + abort() + + # Here starts the real core and the iteration over coordinates + for coordid in range(ncoords): + holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], + neighborcoords, + grid, + box, + cutoff2) + holder.neighborhoods[coordid].cutoff = cutoff + + return holder + + +# Python interface +cdef class FastNS(object): + cdef PBCBox box + cdef readonly int nthreads + cdef readonly real[:, ::1] coords + cdef real[:, ::1] coords_bbox + cdef readonly real cutoff + cdef bint prepared + cdef ns_grid *grid + + + def __init__(self, box): + if box.shape != (3, 3): + raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") + + self.box = PBCBox(box) + + self.nthreads = 1 + + self.coords = None + self.coords_bbox = None + + self.cutoff = -1 + + self.prepared = False + + self.grid = malloc(sizeof(ns_grid)) + + + def __dealloc__(self): + #destroy_nsgrid(self.grid) + self.grid.size = 0 + + #free(self.grid) + + def set_nthreads(self, nthreads, silent=False): + import multiprocessing + + if nthreads > multiprocessing.cpu_count(): + print("Warning: the number of threads requested if greater than the number of cores available. Performances may not be optimal!") + + if not silent: + print("Number of threads for NS adjusted to {}.".format(nthreads)) + + self.nthreads = nthreads + + + def set_coords(self, real[:, ::1] coords): + self.coords = coords + + # Make sure atoms are inside the brick-shaped box + self.coords_bbox = self.box.fast_put_atoms_in_bbox(coords) + + self.prepared = False + + + def set_cutoff(self, real cutoff): + self.cutoff = cutoff + + self.prepared = False + + + def prepare(self, force=False): + cdef ns_int i + cdef bint initialization_ok + + if self.prepared and not force: + print("NS already prepared, nothing to do!") + + if self.coords is None: + raise ValueError("Coordinates must be set before NS preparation!") + + if self.cutoff < 0: + raise ValueError("Cutoff must be set before NS preparation!") + + with nogil: + initialization_ok = False + + # Initializing grid + for i in range(DIM): + self.grid.ncells[i] = (self.box.c_pbcbox.box[i][i] / self.cutoff) + if self.grid.ncells[i] == 0: + self.grid.ncells[i] = 1 + self.grid.cellsize[i] = self.box.c_pbcbox.box[i][i] / self.grid.ncells[i] + + self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] + + # Populating grid + if populate_grid(self.grid, self.coords_bbox) == RET_OK: + initialization_ok = True + + + if initialization_ok: + self.prepared = True + else: + raise RuntimeError("Could not initialize NS grid") + + + def search(self, real[:, ::1]search_coords, return_ids=False): + cdef real[:, ::1] search_coords_bbox + cdef ns_int nid, i, j + cdef ns_neighborhood_holder *holder + cdef ns_neighborhood *neighborhood + + if not self.prepared: + self.prepare() + + + # Make sure atoms are inside the brick-shaped box + search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) + + + with nogil: + # Retrieve neighbors from grid + if self.nthreads == 1: + holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) + else: + holder = ns_core_parallel(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff, self.nthreads) + + + + neighbors = [] + ###Modify for distance + sqdist = [] + indx = [] + ### + for nid in range(holder.size): + neighborhood = holder.neighborhoods[nid] + + if return_ids: + neighborhood_py = np.empty(neighborhood.size, dtype=np.int64) + ###Modify for distance + neighborhood_dis = np.empty(neighborhood.size, dtype=np.float32) + neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) + ### + for i in range(neighborhood.size): + neighborhood_py[i] = neighborhood.beadids[i] + ###Modify for distance + neighborhood_dis[i] = neighborhood.beaddist[i] + ### + else: + neighborhood_py = np.empty((neighborhood.size, DIM), dtype=np.float32) + ###Modify for distance + neighborhood_dis = np.empty((neighborhood.size), dtype=np.float32) + neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) + ### + for i in range(neighborhood.size): + ###Modify for distance + neighborhood_dis[i] = neighborhood.beaddist[i] + neighborhood_indx[i] = neighborhood.beadids[i] + ### + + for j in range(DIM): + neighborhood_py[i,j] = self.coords[neighborhood.beadids[i], j] + + neighbors.append(neighborhood_py) + sqdist.append(neighborhood_dis) + indx.append(neighborhood_indx) + + # Free Memory + free_neighborhood_holder(holder) + + return neighbors, sqdist, indx + + + +__version__ = "26" \ No newline at end of file diff --git a/package/setup.py b/package/setup.py index 4712e674d89..388b5e72354 100755 --- a/package/setup.py +++ b/package/setup.py @@ -374,6 +374,13 @@ def extensions(config): libraries=mathlib, define_macros=define_macros, extra_compile_args=extra_compile_args) + grid = MDAExtension('MDAnalysis.lib.grid', + ['MDAnalysis/lib/c_gridsearch' + source_suffix], + include_dirs=include_dirs, + libraries=mathlib + parallel_libraries, + define_macros=define_macros + parallel_macros, + extra_compile_args=extra_compile_args + parallel_args, + extra_link_args=parallel_args) pre_exts = [libdcd, distances, distances_omp, qcprot, transformation, libmdaxdr, util, encore_utils, ap_clustering, spe_dimred, cutil] From f894970deed0f0d0da014fa7f5cfd68cb78f27a7 Mon Sep 17 00:00:00 2001 From: ayush Date: Mon, 25 Jun 2018 22:06:44 -0700 Subject: [PATCH 13/30] removed the parallel handler to remove hard dependency on omp.h, can be added again for conditional compilation --- package/MDAnalysis/lib/c_gridsearch.pyx | 85 +------------------------ 1 file changed, 2 insertions(+), 83 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 82d7192544a..a4e8745aec4 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -34,22 +34,14 @@ DEF GRID_ALLOCATION_INCREMENT = 50 DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 - -# Cython C imports (no Python here!) -from cython.parallel cimport prange from libc.stdlib cimport malloc, realloc, free, abort from libc.stdio cimport fprintf, stderr from libc.math cimport sqrt from libc.math cimport abs as real_abs -cimport openmp - - -# Python imports import numpy as np cimport numpy as np -# Ctypes ctypedef np.int_t ns_int ctypedef np.float32_t real ctypedef real rvec[DIM] @@ -93,7 +85,6 @@ cdef struct cPBCBox_t: ns_int[DIM] tric_shift[MAX_NTRICVEC] real[DIM] tric_vec[MAX_NTRICVEC] -# noinspection PyNoneFunctionAssignment cdef class PBCBox(object): cdef cPBCBox_t c_pbcbox cdef rvec center @@ -271,8 +262,6 @@ cdef class PBCBox(object): def put_atoms_in_bbox(self, real[:,::1] coords): return np.asarray(self.fast_put_atoms_in_bbox(coords)) - - ######################################################################################################################## # # Neighbor Search Stuff @@ -495,50 +484,6 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei nchecked += 1 return neighborhood - - -cdef ns_neighborhood_holder *ns_core_parallel(real[:, ::1] refcoords, - real[:, ::1] neighborcoords, - ns_grid *grid, - PBCBox box, - real cutoff, - int nthreads=-1) nogil: - cdef ns_int coordid, i, j - cdef ns_int ncoords = refcoords.shape[0] - cdef ns_int ncoords_neighbors = neighborcoords.shape[0] - cdef real cutoff2 = cutoff * cutoff - cdef ns_neighborhood_holder *holder - - cdef ns_int *neighbor_buf - cdef ns_int buf_size, ibuf - - if nthreads < 0: - nthreads = openmp.omp_get_num_threads() - - holder = create_neighborhood_holder() - if holder == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", - sizeof(ns_int) * ncoords) - abort() - - holder.size = ncoords - holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) - if holder.neighborhoods == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", - sizeof(ns_neighborhood) * ncoords) - abort() - - # Here starts the real core and the iteration over coordinates - for coordid in prange(ncoords, schedule='dynamic', num_threads=nthreads): - holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], - neighborcoords, - grid, - box, - cutoff2) - holder.neighborhoods[coordid].cutoff = cutoff - - return holder - cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, real[:, ::1] neighborcoords, ns_grid *grid, @@ -577,7 +522,6 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, return holder - # Python interface cdef class FastNS(object): cdef PBCBox box @@ -610,21 +554,7 @@ cdef class FastNS(object): def __dealloc__(self): #destroy_nsgrid(self.grid) self.grid.size = 0 - - #free(self.grid) - - def set_nthreads(self, nthreads, silent=False): - import multiprocessing - - if nthreads > multiprocessing.cpu_count(): - print("Warning: the number of threads requested if greater than the number of cores available. Performances may not be optimal!") - - if not silent: - print("Number of threads for NS adjusted to {}.".format(nthreads)) - - self.nthreads = nthreads - - + def set_coords(self, real[:, ::1] coords): self.coords = coords @@ -689,15 +619,8 @@ cdef class FastNS(object): # Make sure atoms are inside the brick-shaped box search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) - with nogil: - # Retrieve neighbors from grid - if self.nthreads == 1: - holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) - else: - holder = ns_core_parallel(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff, self.nthreads) - - + holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) neighbors = [] ###Modify for distance @@ -741,7 +664,3 @@ cdef class FastNS(object): free_neighborhood_holder(holder) return neighbors, sqdist, indx - - - -__version__ = "26" \ No newline at end of file From 2f0b0fbf6b47211374280835ef4b0cf0f44a25c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 1 Jul 2018 14:06:40 +0200 Subject: [PATCH 14/30] Added MDAnalysis header file to c_gridsearch.pyx --- package/MDAnalysis/lib/c_gridsearch.pyx | 32 +++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index a4e8745aec4..ac73c97a5a2 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -1,22 +1,28 @@ -# -*- coding: utf-8; Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- +# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 # -# Copyright (C) 2013-2016 Sébastien Buchoux +# MDAnalysis --- https://www.mdanalysis.org # -# This file is part of FATSLiM. +# Copyright (C) 2013-2018 Sébastien Buchoux +# Copyright (c) 2018 The MDAnalysis Development Team and contributors +# (see the file AUTHORS for the full list of names) # -# FATSLiM is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# Released under the GNU Public Licence, v3 or any higher version # -# FATSLiM is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. +# Please cite your use of MDAnalysis in published work: # -# You should have received a copy of the GNU General Public License -# along with FATSLiM. If not, see . +# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler, +# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. +# MDAnalysis: A Python package for the rapid analysis of molecular dynamics +# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th +# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. +# +# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. +# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. +# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 +# +# + #cython: cdivision=True #cython: boundscheck=False From 1bb569bbdab34386c65231108d26f3455f3ec921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 1 Jul 2018 16:53:22 +0200 Subject: [PATCH 15/30] Corrected C (de)allocation in c_gridsearch.pyx. Should fix memory leak --- package/MDAnalysis/lib/c_gridsearch.pyx | 32 +++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index ac73c97a5a2..2a7af4b33b7 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -145,16 +145,15 @@ cdef class PBCBox(object): # Update shift vectors self.c_pbcbox.ntric_vec = 0 - # We will only use single shifts, but we will check a few - # more shifts to see if there is a limiting distance - # above which we can not be sure of the correct distance. - for kk in range(5): + + # We will only use single shifts + for kk in range(3): k = order[kk] - for jj in range(5): + for jj in range(3): j = order[jj] - for ii in range(5): + for ii in range(3): i = order[ii] # A shift is only useful when it is trilinic @@ -177,8 +176,8 @@ cdef class PBCBox(object): else: pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) - d2old += sqrt(pos[d]) - d2new += sqrt(pos[d] + trial[d]) + d2old += pos[d]**2 + d2new += (pos[d] + trial[d])**2 if BOX_MARGIN*d2new < d2old: if not (j < -1 or j > 1 or k < -1 or k > 1): @@ -196,7 +195,7 @@ cdef class PBCBox(object): d2new_c = 0 for d in range(DIM): - d2new_c += sqrt(pos[d] + trial[d] - shift*box[dd, d]) + d2new_c += (pos[d] + trial[d] - shift*box[dd, d])**2 if d2new_c <= BOX_MARGIN*d2new: use = False @@ -209,7 +208,13 @@ cdef class PBCBox(object): % MAX_NTRICVEC) print(" There is probably something wrong with " "your box.") - print(box) + print(np.array(box)) + + for i in range(self.c_pbcbox.ntric_vec): + print(" -> shift #{}: [{}, {}, {}]".format(i+1, + self.c_pbcbox.tric_shift[i][XX], + self.c_pbcbox.tric_shift[i][YY], + self.c_pbcbox.tric_shift[i][ZZ])) else: for d in range(DIM): self.c_pbcbox.tric_vec[self.c_pbcbox.ntric_vec][d] = \ @@ -539,6 +544,9 @@ cdef class FastNS(object): cdef ns_grid *grid + def __cinit__(self): + self.grid = malloc(sizeof(ns_grid)) + def __init__(self, box): if box.shape != (3, 3): raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") @@ -554,11 +562,9 @@ cdef class FastNS(object): self.prepared = False - self.grid = malloc(sizeof(ns_grid)) - def __dealloc__(self): - #destroy_nsgrid(self.grid) + destroy_nsgrid(self.grid) self.grid.size = 0 def set_coords(self, real[:, ::1] coords): From 0884555f3d479b1616f2db1af2af533af7fd94e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 1 Jul 2018 16:54:21 +0200 Subject: [PATCH 16/30] Removed pointer to nsgrid structure to avoid need to free it --- package/MDAnalysis/lib/__init__.py | 3 +- package/MDAnalysis/lib/c_gridsearch.pyx | 17 ++-- .../MDAnalysisTests/lib/test_gridsearch.py | 81 +++++++++++++++++++ 3 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 testsuite/MDAnalysisTests/lib/test_gridsearch.py diff --git a/package/MDAnalysis/lib/__init__.py b/package/MDAnalysis/lib/__init__.py index 51c148dde65..a740bd8d5bf 100644 --- a/package/MDAnalysis/lib/__init__.py +++ b/package/MDAnalysis/lib/__init__.py @@ -29,7 +29,7 @@ from __future__ import absolute_import __all__ = ['log', 'transformations', 'util', 'mdamath', 'distances', - 'NeighborSearch', 'formats', 'pkdtree'] + 'NeighborSearch', 'formats', 'pkdtree', 'grid'] from . import log from . import transformations @@ -39,3 +39,4 @@ from . import NeighborSearch from . import formats from . import pkdtree +from . import grid \ No newline at end of file diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 2a7af4b33b7..7a6de80389d 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -42,8 +42,7 @@ DEF MAX_NTRICVEC=12 from libc.stdlib cimport malloc, realloc, free, abort from libc.stdio cimport fprintf, stderr -from libc.math cimport sqrt -from libc.math cimport abs as real_abs + import numpy as np cimport numpy as np @@ -541,11 +540,7 @@ cdef class FastNS(object): cdef real[:, ::1] coords_bbox cdef readonly real cutoff cdef bint prepared - cdef ns_grid *grid - - - def __cinit__(self): - self.grid = malloc(sizeof(ns_grid)) + cdef ns_grid grid def __init__(self, box): if box.shape != (3, 3): @@ -562,9 +557,11 @@ cdef class FastNS(object): self.prepared = False + self.grid.size = 0 + def __dealloc__(self): - destroy_nsgrid(self.grid) + destroy_nsgrid(&self.grid) self.grid.size = 0 def set_coords(self, real[:, ::1] coords): @@ -608,7 +605,7 @@ cdef class FastNS(object): self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] # Populating grid - if populate_grid(self.grid, self.coords_bbox) == RET_OK: + if populate_grid(&self.grid, self.coords_bbox) == RET_OK: initialization_ok = True @@ -632,7 +629,7 @@ cdef class FastNS(object): search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) with nogil: - holder = ns_core(search_coords_bbox, self.coords_bbox, self.grid, self.box, self.cutoff) + holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) neighbors = [] ###Modify for distance diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py new file mode 100644 index 00000000000..cab23f3f372 --- /dev/null +++ b/testsuite/MDAnalysisTests/lib/test_gridsearch.py @@ -0,0 +1,81 @@ +# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*- +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8 +# +# MDAnalysis --- https://www.mdanalysis.org +# Copyright (c) 2006-2018 The MDAnalysis Development Team and contributors +# (see the file AUTHORS for the full list of names) +# +# Released under the GNU Public Licence, v2 or any higher version +# +# Please cite your use of MDAnalysis in published work: +# +# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler, +# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. +# MDAnalysis: A Python package for the rapid analysis of molecular dynamics +# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th +# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. +# +# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. +# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. +# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 +# + +from __future__ import print_function, absolute_import + +import pytest +from numpy.testing import assert_equal +import numpy as np + +import MDAnalysis as mda +from MDAnalysis.lib import grid +from MDAnalysis.lib.pkdtree import PeriodicKDTree +from MDAnalysis.lib.mdamath import triclinic_vectors + +from MDAnalysisTests.datafiles import GRO + +@pytest.fixture +def universe(): + u = mda.Universe(GRO) + return u + +def run_search(universe, ref_id): + cutoff = 3 + + coords = universe.atoms.positions + ref_pos = coords[ref_id] + triclinic_box = triclinic_vectors(universe.dimensions) + + # Run pkdtree search + pkdt = PeriodicKDTree(universe.atoms.dimensions, bucket_size=10) + pkdt.set_coords(coords) + pkdt.search(ref_pos, cutoff) + + results_pkdtree = pkdt.get_indices() + results_pkdtree.remove(ref_id) + results_pkdtree = np.array(results_pkdtree) + results_pkdtree.sort() + + # Run grid search + searcher = grid.FastNS(triclinic_box) + searcher.set_cutoff(cutoff) + searcher.set_coords(coords) + searcher.prepare() + + results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0][0] + results_grid.sort() + + return results_pkdtree, results_grid + +def test_gridsearch(universe): + """Check that pkdtree and grid search return the same results (No PBC needed)""" + + ref_id = 0 + results_pkdtree, results_grid = run_search(universe, ref_id) + assert_equal(results_pkdtree, results_grid) + +def test_gridsearch_PBC(universe): + """Check that pkdtree and grid search return the same results (PBC needed)""" + + ref_id = 13937 + results_pkdtree, results_grid = run_search(universe, ref_id) + assert_equal(results_pkdtree, results_grid) From 8913e2109feb9b26854980a24b7488688e3c2143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Wed, 4 Jul 2018 18:13:41 +0200 Subject: [PATCH 17/30] Removed abort() from c_gridsearch.pyx and grid allocation moved to FastNS method. --- package/MDAnalysis/lib/c_gridsearch.pyx | 267 +++++++----------- .../MDAnalysisTests/lib/test_gridsearch.py | 2 +- 2 files changed, 104 insertions(+), 165 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 7a6de80389d..98d97eded65 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -31,18 +31,15 @@ DEF DIM = 3 DEF XX = 0 DEF YY = 1 DEF ZZ = 2 -DEF RET_OK = 1 -DEF RET_ERROR = 0 + DEF EPSILON = 1e-5 + DEF NEIGHBORHOOD_ALLOCATION_INCREMENT = 50 -DEF GRID_ALLOCATION_INCREMENT = 50 DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 -from libc.stdlib cimport malloc, realloc, free, abort -from libc.stdio cimport fprintf, stderr - +from libc.stdlib cimport malloc, realloc, free import numpy as np cimport numpy as np @@ -64,8 +61,7 @@ cdef struct ns_neighborhood: ns_int allocated_size ns_int size ns_int *beadids - real *beaddist - ### + cdef struct ns_neighborhood_holder: ns_int size ns_neighborhood **neighborhoods @@ -80,6 +76,7 @@ cdef void rvec_clear(rvec a) nogil: a[YY]=0.0 a[ZZ]=0.0 + cdef struct cPBCBox_t: matrix box rvec fbox_diag @@ -90,6 +87,8 @@ cdef struct cPBCBox_t: ns_int[DIM] tric_shift[MAX_NTRICVEC] real[DIM] tric_vec[MAX_NTRICVEC] + +# Class to handle PBC calculations cdef class PBCBox(object): cdef cPBCBox_t c_pbcbox cdef rvec center @@ -232,6 +231,7 @@ cdef class PBCBox(object): raise ValueError("Box does not correspond to PBC=xyz") self.fast_update(box) + cdef void fast_pbc_dx(self, rvec ref, rvec other, rvec dx) nogil: cdef ns_int i, j cdef rvec dx_start, trial @@ -284,102 +284,12 @@ cdef struct ns_grid: ns_int *nbeads ns_int **beadids -cdef ns_grid initialize_nsgrid(matrix box, - float cutoff) nogil: - cdef ns_grid grid - cdef ns_int i - - for i in range(DIM): - grid.ncells[i] = (box[i][i] / cutoff) - if grid.ncells[i] == 0: - grid.ncells[i] = 1 - grid.cellsize[i] = box[i][i] / grid.ncells[i] - - grid.size = grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ] - return grid - -cdef ns_int populate_grid(ns_grid *grid, - real[:,::1] coords) nogil: - cdef ns_int ncoords = coords.shape[0] - cdef bint ret_val - - ret_val = populate_grid_array(grid, - &coords[0, 0], - ncoords) - - return ret_val - -cdef ns_int populate_grid_array(ns_grid *grid, - rvec *coords, - ns_int ncoords) nogil: - cdef ns_int i, cellindex = -1 - cdef ns_int grid_size = grid.size - cdef ns_int *allocated_size = NULL - - if grid_size != grid.ncells[XX] * grid.ncells[YY] * grid.ncells[ZZ]: # Grid not initialized - return RET_ERROR - - - # Allocate memory - grid.nbeads = malloc(sizeof(ns_int) * grid_size) - if grid.nbeads == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS grid.nbeads (requested: %i bytes)\n", - sizeof(ns_int) * grid_size) - abort() - - allocated_size = malloc(sizeof(ns_int) * grid_size) - if allocated_size == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS allocated_size (requested: %i bytes)\n", - sizeof(ns_int) * grid_size) - abort() - - for i in range(grid_size): - grid.nbeads[i] = 0 - allocated_size[i] = GRID_ALLOCATION_INCREMENT - - grid.beadids = malloc(sizeof(ns_int *) * grid_size) - if grid.beadids == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids (requested: %i bytes)\n", - sizeof(ns_int *) * grid_size) - abort() - - for i in range(grid_size): - grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) - if grid.beadids[i] == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS grid.beadids[i] (requested: %i bytes)\n", - sizeof(ns_int) * allocated_size[i]) - abort() - - # Get cell indices for coords - for i in range(ncoords): - cellindex = (coords[i][ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ - (coords[i][YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ - (coords[i][XX] / grid.cellsize[XX]) - - grid.beadids[cellindex][grid.nbeads[cellindex]] = i - grid.nbeads[cellindex] += 1 - - if grid.nbeads[cellindex] >= allocated_size[cellindex]: - allocated_size[cellindex] += GRID_ALLOCATION_INCREMENT - grid.beadids[cellindex] = realloc( grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) - free(allocated_size) - return RET_OK - -cdef void destroy_nsgrid(ns_grid *grid) nogil: - cdef ns_int i - if grid.nbeads != NULL: - free(grid.nbeads) - - for i in range(grid.size): - if grid.beadids[i] != NULL: - free(grid.beadids[i]) - free(grid.beadids) - - cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: cdef ns_neighborhood_holder *holder holder = malloc(sizeof(ns_neighborhood_holder)) + holder.size = 0 + holder.neighborhoods = NULL return holder @@ -393,7 +303,9 @@ cdef void free_neighborhood_holder(ns_neighborhood_holder *holder) nogil: if holder.neighborhoods[i].beadids != NULL: free(holder.neighborhoods[i].beadids) free(holder.neighborhoods[i]) - free(holder.neighborhoods) + + if holder.neighborhoods != NULL: + free(holder.neighborhoods) free(holder) cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]neighborcoords, ns_grid *grid, PBCBox box, real cutoff2) nogil: @@ -409,16 +321,15 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei cdef ns_neighborhood *neighborhood = malloc(sizeof(ns_neighborhood)) if neighborhood == NULL: - abort() + return NULL neighborhood.size = 0 neighborhood.allocated_size = NEIGHBORHOOD_ALLOCATION_INCREMENT neighborhood.beadids = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(ns_int)) - ###Modified here - neighborhood.beaddist = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(real)) - ### + if neighborhood.beadids == NULL: - abort() + free(neighborhood) + return NULL for zi in range(3): for yi in range(3): @@ -472,28 +383,23 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei # Update neighbor lists neighborhood.beadids[neighborhood.size] = bid - ### Modified here - neighborhood.beaddist[neighborhood.size] = d2 - ### neighborhood.size += 1 if neighborhood.size >= neighborhood.allocated_size: neighborhood.allocated_size += NEIGHBORHOOD_ALLOCATION_INCREMENT neighborhood.beadids = realloc( neighborhood.beadids, neighborhood.allocated_size * sizeof(ns_int)) - ###Modified here - neighborhood.beaddist = realloc( neighborhood.beaddist, neighborhood.allocated_size * sizeof(real)) - ### + if neighborhood.beadids == NULL: - abort() - ###Modified - if neighborhood.beaddist == NULL: - abort() - ### + free(neighborhood) + return NULL + # Register the cell as checked already_checked[nchecked] = cell_index nchecked += 1 return neighborhood + + cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, real[:, ::1] neighborcoords, ns_grid *grid, @@ -510,16 +416,12 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, holder = create_neighborhood_holder() if holder == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder\n", - sizeof(ns_int) * ncoords) - abort() + return NULL - holder.size = ncoords holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) if holder.neighborhoods == NULL: - fprintf(stderr,"FATAL: Could not allocate memory for NS holder.neighborhoods (requested: %i bytes)\n", - sizeof(ns_neighborhood) * ncoords) - abort() + free_neighborhood_holder(holder) + return NULL # Here starts the real core and the iteration over coordinates for coordid in range(ncoords): @@ -528,14 +430,18 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, grid, box, cutoff2) + if holder.neighborhoods[coordid] == NULL: + free_neighborhood_holder(holder) + return NULL + holder.neighborhoods[coordid].cutoff = cutoff + holder.size += 1 return holder # Python interface cdef class FastNS(object): cdef PBCBox box - cdef readonly int nthreads cdef readonly real[:, ::1] coords cdef real[:, ::1] coords_bbox cdef readonly real cutoff @@ -548,8 +454,6 @@ cdef class FastNS(object): self.box = PBCBox(box) - self.nthreads = 1 - self.coords = None self.coords_bbox = None @@ -561,7 +465,15 @@ cdef class FastNS(object): def __dealloc__(self): - destroy_nsgrid(&self.grid) + cdef ns_int i + if self.grid.nbeads != NULL: + free(self.grid.nbeads) + + for i in range(self.grid.size): + if self.grid.beadids[i] != NULL: + free(self.grid.beadids[i]) + free(self.grid.beadids) + self.grid.size = 0 def set_coords(self, real[:, ::1] coords): @@ -580,8 +492,11 @@ cdef class FastNS(object): def prepare(self, force=False): - cdef ns_int i - cdef bint initialization_ok + cdef ns_int i, cellindex = -1 + cdef ns_int *allocated_size = NULL + cdef ns_int ncoords = self.coords.shape[0] + cdef ns_int allocation_guess + cdef rvec *coords = &self.coords_bbox[0, 0] if self.prepared and not force: print("NS already prepared, nothing to do!") @@ -593,7 +508,6 @@ cdef class FastNS(object): raise ValueError("Cutoff must be set before NS preparation!") with nogil: - initialization_ok = False # Initializing grid for i in range(DIM): @@ -601,18 +515,61 @@ cdef class FastNS(object): if self.grid.ncells[i] == 0: self.grid.ncells[i] = 1 self.grid.cellsize[i] = self.box.c_pbcbox.box[i][i] / self.grid.ncells[i] - self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] - # Populating grid - if populate_grid(&self.grid, self.coords_bbox) == RET_OK: - initialization_ok = True - - - if initialization_ok: - self.prepared = True - else: - raise RuntimeError("Could not initialize NS grid") + # This is just a guess on how much memory we might for each grid cell: + # we just assume an average bead density and we take four times this density just to be safe + allocation_guess = (4 * (ncoords / self.grid.size + 1)) + + # Allocate memory for the grid + self.grid.nbeads = malloc(sizeof(ns_int) * self.grid.size) + if self.grid.nbeads == NULL: + with gil: + raise MemoryError("Could not allocate memory for NS grid") + + # Allocate memory from temporary allocation counter + allocated_size = malloc(sizeof(ns_int) * self.grid.size) + if allocated_size == NULL: + # No need to free grid.nbeads as it will be freed by destroy_nsgrid called by __dealloc___ + with gil: + raise MemoryError("Could not allocate memory for allocation buffer") + + # Pre-allocate some memory for grid cells + for i in range(self.grid.size): + self.grid.nbeads[i] = 0 + allocated_size[i] = allocation_guess + + self.grid.beadids = malloc(sizeof(ns_int *) * self.grid.size) + if self.grid.beadids == NULL: + with gil: + raise MemoryError("Could not allocate memory for grid cells") + + for i in range(self.grid.size): + self.grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) + if self.grid.beadids[i] == NULL: + with gil: + raise MemoryError("Could not allocate memory for grid cell") + + # Populate grid cells using the coordinates (ie do the heavy work) + for i in range(ncoords): + cellindex = (coords[i][ZZ] / self.grid.cellsize[ZZ]) * (self.grid.ncells[XX] * self.grid.ncells[YY]) +\ + (coords[i][YY] / self.grid.cellsize[YY]) * self.grid.ncells[XX] + \ + (coords[i][XX] / self.grid.cellsize[XX]) + + self.grid.beadids[cellindex][self.grid.nbeads[cellindex]] = i + self.grid.nbeads[cellindex] += 1 + + # We need to allocate more memory (simply double the amount of memory as + # 1. it should barely be needed + # 2. the size should stay fairly reasonable + if self.grid.nbeads[cellindex] >= allocated_size[cellindex]: + allocated_size[cellindex] *= 2 + self.grid.beadids[cellindex] = realloc( self.grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) + + # Now we can free the allocation buffer + free(allocated_size) + + self.prepared = True def search(self, real[:, ::1]search_coords, return_ids=False): @@ -631,45 +588,27 @@ cdef class FastNS(object): with nogil: holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) + if holder == NULL: + raise MemoryError("Could not allocate memory to run NS core") + neighbors = [] - ###Modify for distance - sqdist = [] - indx = [] - ### for nid in range(holder.size): neighborhood = holder.neighborhoods[nid] if return_ids: neighborhood_py = np.empty(neighborhood.size, dtype=np.int64) - ###Modify for distance - neighborhood_dis = np.empty(neighborhood.size, dtype=np.float32) - neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) - ### + for i in range(neighborhood.size): neighborhood_py[i] = neighborhood.beadids[i] - ###Modify for distance - neighborhood_dis[i] = neighborhood.beaddist[i] - ### else: neighborhood_py = np.empty((neighborhood.size, DIM), dtype=np.float32) - ###Modify for distance - neighborhood_dis = np.empty((neighborhood.size), dtype=np.float32) - neighborhood_indx = np.empty(neighborhood.size, dtype=np.int64) - ### for i in range(neighborhood.size): - ###Modify for distance - neighborhood_dis[i] = neighborhood.beaddist[i] - neighborhood_indx[i] = neighborhood.beadids[i] - ### - for j in range(DIM): neighborhood_py[i,j] = self.coords[neighborhood.beadids[i], j] neighbors.append(neighborhood_py) - sqdist.append(neighborhood_dis) - indx.append(neighborhood_indx) # Free Memory free_neighborhood_holder(holder) - return neighbors, sqdist, indx + return neighbors diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py index cab23f3f372..c8d526e9074 100644 --- a/testsuite/MDAnalysisTests/lib/test_gridsearch.py +++ b/testsuite/MDAnalysisTests/lib/test_gridsearch.py @@ -61,7 +61,7 @@ def run_search(universe, ref_id): searcher.set_coords(coords) searcher.prepare() - results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0][0] + results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0] results_grid.sort() return results_pkdtree, results_grid From 1bbf1760d0925e91cab2f26ef26f0090a33ad6d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 8 Jul 2018 10:35:17 +0200 Subject: [PATCH 18/30] NSResults object added to store results from NS. Started documentation --- package/MDAnalysis/lib/c_gridsearch.pyx | 235 ++++++++++++++++-- .../source/documentation_pages/lib/grid.rst | 2 + .../documentation_pages/lib_modules.rst | 2 + .../MDAnalysisTests/lib/test_gridsearch.py | 79 +++++- 4 files changed, 282 insertions(+), 36 deletions(-) create mode 100644 package/doc/sphinx/source/documentation_pages/lib/grid.rst diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 98d97eded65..9c50a5b1777 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -25,6 +25,15 @@ #cython: cdivision=True #cython: boundscheck=False +#cython: initializedcheck=False + +""" +Neighbor search library --- :mod:`MDAnalysis.lib.grid` +====================================================== + +This Neighbor search library is a serialized Cython port of the NS grid search implemented in GROMACS. +""" + # Preprocessor DEFs DEF DIM = 3 @@ -40,6 +49,7 @@ DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 from libc.stdlib cimport malloc, realloc, free +from libc.math cimport sqrt import numpy as np cimport numpy as np @@ -248,16 +258,21 @@ cdef class PBCBox(object): for j in range (i, -1, -1): dx[j] += self.c_pbcbox.box[i][j] + cdef real fast_distance2(self, rvec a, rvec b) nogil: + cdef rvec dx + self.fast_pbc_dx(a, b, dx) + return rvec_norm2(dx) + + cdef real fast_distance(self, rvec a, rvec b) nogil: + return sqrt(self.fast_distance2(a,b)) + cdef real[:, ::1]fast_put_atoms_in_bbox(self, real[:,::1] coords) nogil: cdef ns_int i, m, d, natoms, wd = 0 cdef real[:,::1] bbox_coords natoms = coords.shape[0] with gil: - if natoms == 0: - bbox_coords = np.empty((0, DIM)) - else: - bbox_coords = coords.copy() + bbox_coords = coords.copy() for i in range(natoms): for m in range(DIM - 1, -1, -1): @@ -270,6 +285,8 @@ cdef class PBCBox(object): return bbox_coords def put_atoms_in_bbox(self, real[:,::1] coords): + if coords.shape[0] == 0: + return np.zeros((0, DIM), dtype=np.float32) return np.asarray(self.fast_put_atoms_in_bbox(coords)) ######################################################################################################################## @@ -439,6 +456,167 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, return holder +cdef class NSResults(object): + """ + Class used to store results returned by `MDAnalysis.lib.grid.FastNS.search` + """ + cdef PBCBox box + cdef readonly real cutoff + cdef real[:, ::1] grid_coords + cdef real[:, ::1] ref_coords + cdef ns_int **nids + cdef ns_int *nsizes + cdef ns_int size + cdef list indices + cdef list coordinates + cdef list distances + + def __init__(self, PBCBox box, real cutoff): + self.box = box + self.cutoff = cutoff + + self.size = 0 + self.nids = NULL + self.nsizes = NULL + + self.grid_coords = None + self.ref_coords = None + + self.indices = None + self.coordinates = None + self.distances = None + + + cdef populate(self, ns_neighborhood_holder *holder, grid_coords, ref_coords): + cdef ns_int nid, i + cdef ns_neighborhood *neighborhood + + self.grid_coords = grid_coords.copy() + self.ref_coords = ref_coords.copy() + + # Allocate memory + self.nsizes = malloc(sizeof(ns_int) * holder.size) + if self.nsizes == NULL: + raise MemoryError("Could not allocate memory for NSResults") + + self.nids = malloc(sizeof(ns_int *) * holder.size) + if self.nids == NULL: + raise MemoryError("Could not allocate memory for NSResults") + + for nid in range(holder.size): + neighborhood = holder.neighborhoods[nid] + + self.nsizes[nid] = neighborhood.size + + self.nids[nid] = malloc(sizeof(ns_int *) * neighborhood.size) + if self.nids[nid] == NULL: + raise MemoryError("Could not allocate memory for NSResults") + + with nogil: + for nid in range(holder.size): + neighborhood = holder.neighborhoods[nid] + + for i in range(neighborhood.size): + self.nids[nid][i] = neighborhood.beadids[i] + + self.size = holder.size + + def __dealloc__(self): + if self.nids != NULL: + for i in range(self.size): + if self.nids[i] != NULL: + free(self.nids[i]) + free(self.nids) + + if self.nsizes != NULL: + free(self.nsizes) + + + def get_indices(self): + """ + Return Neighbors indices. + + :return: list of indices + """ + cdef ns_int i, nid, size + + if self.indices is None: + indices = [] + + for nid in range(self.size): + size = self.nsizes[nid] + + tmp_incides = np.empty((size), dtype=np.int) + + for i in range(size): + tmp_incides[i] = self.nids[nid][i] + + indices.append(tmp_incides) + + self.indices = indices + + return self.indices + + + def get_coordinates(self): + """ + Return coordinates of neighbors. + + :return: list of coordinates + """ + cdef ns_int i, nid, size, beadid + + if self.coordinates is None: + coordinates = [] + + for nid in range(self.size): + size = self.nsizes[nid] + + tmp_values = np.empty((size, DIM), dtype=np.float32) + + for i in range(size): + beadid = self.nids[nid][i] + tmp_values[i] = self.grid_coords[beadid] + + coordinates.append(tmp_values) + + self.coordinates = coordinates + + return self.coordinates + + + def get_distances(self): + """ + Return coordinates of neighbors. + + :return: list of distances + """ + cdef ns_int i, nid, size, j, beadid + cdef rvec ref, other, dx + cdef real dist + + if self.distances is None: + distances = [] + + for nid in range(self.size): + size = self.nsizes[nid] + + tmp_values = np.empty((size), dtype=np.float32) + ref = &self.ref_coords[nid, 0] + + for i in range(size): + beadid = self.nids[nid][i] + other = &self.grid_coords[beadid, 0] + + tmp_values[i] = self.box.fast_distance(ref, other) + + distances.append(tmp_values) + + self.distances = distances + + return self.distances + + # Python interface cdef class FastNS(object): cdef PBCBox box @@ -448,7 +626,14 @@ cdef class FastNS(object): cdef bint prepared cdef ns_grid grid - def __init__(self, box): + def __init__(self, u): + import MDAnalysis as mda + from MDAnalysis.lib.mdamath import triclinic_vectors + + if not isinstance(u, mda.Universe): + raise TypeError("FastNS class must be initialized with a valid MDAnalysis.Universe instance") + box = triclinic_vectors(u.dimensions) + if box.shape != (3, 3): raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") @@ -466,6 +651,7 @@ cdef class FastNS(object): def __dealloc__(self): cdef ns_int i + # Deallocate NS grid if self.grid.nbeads != NULL: free(self.grid.nbeads) @@ -475,7 +661,8 @@ cdef class FastNS(object): free(self.grid.beadids) self.grid.size = 0 - + + def set_coords(self, real[:, ::1] coords): self.coords = coords @@ -568,12 +755,12 @@ cdef class FastNS(object): # Now we can free the allocation buffer free(allocated_size) - self.prepared = True - def search(self, real[:, ::1]search_coords, return_ids=False): + def search(self, search_coords): cdef real[:, ::1] search_coords_bbox + cdef real[:, ::1] search_coords_view cdef ns_int nid, i, j cdef ns_neighborhood_holder *holder cdef ns_neighborhood *neighborhood @@ -581,9 +768,18 @@ cdef class FastNS(object): if not self.prepared: self.prepare() + # Check the shape of search_coords as a array of 3D coords if needed + shape = search_coords.shape + if len(shape) == 1: + if not shape[0] == 3: + raise ValueError("Coordinates must be 3D") + else: + search_coords_view = np.array([search_coords,], dtype=np.float32) + else: + search_coords_view = search_coords # Make sure atoms are inside the brick-shaped box - search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords) + search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords_view) with nogil: holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) @@ -591,24 +787,11 @@ cdef class FastNS(object): if holder == NULL: raise MemoryError("Could not allocate memory to run NS core") - neighbors = [] - for nid in range(holder.size): - neighborhood = holder.neighborhoods[nid] - if return_ids: - neighborhood_py = np.empty(neighborhood.size, dtype=np.int64) - - for i in range(neighborhood.size): - neighborhood_py[i] = neighborhood.beadids[i] - else: - neighborhood_py = np.empty((neighborhood.size, DIM), dtype=np.float32) - for i in range(neighborhood.size): - for j in range(DIM): - neighborhood_py[i,j] = self.coords[neighborhood.beadids[i], j] - - neighbors.append(neighborhood_py) + results = NSResults(self.box, self.cutoff) + results.populate(holder, self.coords, search_coords_view) - # Free Memory + # Free memory allocated to holder free_neighborhood_holder(holder) - return neighbors + return results diff --git a/package/doc/sphinx/source/documentation_pages/lib/grid.rst b/package/doc/sphinx/source/documentation_pages/lib/grid.rst new file mode 100644 index 00000000000..a6143e9ca38 --- /dev/null +++ b/package/doc/sphinx/source/documentation_pages/lib/grid.rst @@ -0,0 +1,2 @@ +.. automodule:: MDAnalysis.lib.grid + :members: \ No newline at end of file diff --git a/package/doc/sphinx/source/documentation_pages/lib_modules.rst b/package/doc/sphinx/source/documentation_pages/lib_modules.rst index 5bbe3d041ed..83d7f81cfd9 100644 --- a/package/doc/sphinx/source/documentation_pages/lib_modules.rst +++ b/package/doc/sphinx/source/documentation_pages/lib_modules.rst @@ -36,6 +36,7 @@ functions whereas mathematical functions are to be found in :mod:`MDAnalysis.lib.NeighborSearch` contains classes to do neighbor searches with MDAnalysis objects. +:mod:`MDAnalysis.lib.grid` contains a fast implementation of grid neighbor search. List of modules --------------- @@ -50,6 +51,7 @@ List of modules ./lib/transformations ./lib/qcprot ./lib/util + ./lib/grid Low level file formats ---------------------- diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py index c8d526e9074..359d32544b7 100644 --- a/testsuite/MDAnalysisTests/lib/test_gridsearch.py +++ b/testsuite/MDAnalysisTests/lib/test_gridsearch.py @@ -23,27 +23,46 @@ from __future__ import print_function, absolute_import import pytest -from numpy.testing import assert_equal +from numpy.testing import assert_equal, assert_allclose import numpy as np import MDAnalysis as mda from MDAnalysis.lib import grid from MDAnalysis.lib.pkdtree import PeriodicKDTree -from MDAnalysis.lib.mdamath import triclinic_vectors from MDAnalysisTests.datafiles import GRO + @pytest.fixture def universe(): u = mda.Universe(GRO) return u + +@pytest.fixture +def grid_results(): + u = mda.Universe(GRO) + cutoff = 2 + ref_pos = u.atoms.positions[13937] + return run_grid_search(u, ref_pos, cutoff) + + +def run_grid_search(u, ref_pos, cutoff): + coords = u.atoms.positions + + # Run grid search + searcher = grid.FastNS(u) + searcher.set_cutoff(cutoff) + searcher.set_coords(coords) + searcher.prepare() + + return searcher.search(ref_pos) + + def run_search(universe, ref_id): cutoff = 3 - coords = universe.atoms.positions ref_pos = coords[ref_id] - triclinic_box = triclinic_vectors(universe.dimensions) # Run pkdtree search pkdt = PeriodicKDTree(universe.atoms.dimensions, bucket_size=10) @@ -56,16 +75,13 @@ def run_search(universe, ref_id): results_pkdtree.sort() # Run grid search - searcher = grid.FastNS(triclinic_box) - searcher.set_cutoff(cutoff) - searcher.set_coords(coords) - searcher.prepare() - - results_grid = searcher.search(np.array([ref_pos, ]), return_ids=True)[0] + results_grid = run_grid_search(universe, ref_pos, cutoff) + results_grid = results_grid.get_indices()[0] results_grid.sort() return results_pkdtree, results_grid + def test_gridsearch(universe): """Check that pkdtree and grid search return the same results (No PBC needed)""" @@ -73,9 +89,52 @@ def test_gridsearch(universe): results_pkdtree, results_grid = run_search(universe, ref_id) assert_equal(results_pkdtree, results_grid) + def test_gridsearch_PBC(universe): """Check that pkdtree and grid search return the same results (PBC needed)""" ref_id = 13937 results_pkdtree, results_grid = run_search(universe, ref_id) assert_equal(results_pkdtree, results_grid) + + +def test_gridsearch_arraycoord(universe): + """Check the NS routine accepts a single bead coordinate as well as array of coordinates""" + cutoff = 2 + ref_pos = universe.atoms.positions[:5] + + results = [ + np.array([2, 1, 4, 3]), + np.array([2, 0, 3]), + np.array([0, 1, 3]), + np.array([ 2, 0, 1, 38341]), + np.array([ 6, 0, 5, 17]) + ] + + results_grid = run_grid_search(universe, ref_pos, cutoff).get_indices() + + assert_equal(results_grid, results) + + +def test_gridsearch_search_coordinates(grid_results): + """Check the NS routine can return coordinates instead of ids""" + + results = np.array( + [ + [40.32, 34.25, 55.9], + [0.61, 76.33, -0.56], + [0.48999998, 75.9, 0.19999999], + [-0.11, 76.19, 0.77] + ]) + + assert_allclose(grid_results.get_coordinates()[0], results) + + +def test_gridsearch_search_distances(grid_results): + """Check the NS routine can return PBC distances from neighbors""" + results = np.array([0.096, 0.096, 0.015, 0.179]) * 10 # These distances were obtained using gmx distance + results.sort() + + rounded_results = np.round(grid_results.get_distances()[0], 2) + + assert_allclose(sorted(rounded_results), results) \ No newline at end of file From fda157e96d28a6d6179c20ee8bb6fc72c9dfd953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Sun, 8 Jul 2018 20:10:33 +0200 Subject: [PATCH 19/30] Memory allocation changed to PyMem to enhance speed in c_gridsearch.pyx Pre-converting np.resize to PyMem stuff --- package/MDAnalysis/lib/c_gridsearch.pyx | 856 +++++++++++++++++++++--- 1 file changed, 771 insertions(+), 85 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index 9c50a5b1777..ffdb91fed7d 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -49,6 +49,8 @@ DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 from libc.stdlib cimport malloc, realloc, free +from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free + from libc.math cimport sqrt import numpy as np @@ -57,14 +59,18 @@ cimport numpy as np ctypedef np.int_t ns_int ctypedef np.float32_t real ctypedef real rvec[DIM] +ctypedef ns_int ivec[DIM] +ctypedef ns_int ipair[2] ctypedef real matrix[DIM][DIM] cdef struct ns_grid: + ns_int beadpercell ns_int size ns_int[DIM] ncells real[DIM] cellsize - ns_int *nbeads - ns_int **beadids + ns_int *nbeads # size + ns_int *beadids # size * beadpercell + ns_int *cellids # size cdef struct ns_neighborhood: real cutoff @@ -103,6 +109,7 @@ cdef class PBCBox(object): cdef cPBCBox_t c_pbcbox cdef rvec center cdef rvec bbox_center + cdef bint is_triclinic def __init__(self, real[:,::1] box): self.update(box) @@ -118,10 +125,15 @@ cdef class PBCBox(object): rvec_clear(self.center) # Update matrix + self.is_triclinic = False for i in range(DIM): for j in range(DIM): self.c_pbcbox.box[i][j] = box[i, j] self.center[j] += 0.5 * box[i, j] + + if i != j: + if box[i, j] > EPSILON: + self.is_triclinic = True self.bbox_center[i] = 0.5 * box[i, i] # Update diagonals @@ -274,14 +286,22 @@ cdef class PBCBox(object): with gil: bbox_coords = coords.copy() - for i in range(natoms): - for m in range(DIM - 1, -1, -1): - while bbox_coords[i, m] < 0: - for d in range(m+1): - bbox_coords[i, d] += self.c_pbcbox.box[m][d] - while bbox_coords[i, m] >= self.c_pbcbox.box[m][m]: - for d in range(m+1): - bbox_coords[i, d] -= self.c_pbcbox.box[m][d] + if self.is_triclinic: + for i in range(natoms): + for m in range(DIM - 1, -1, -1): + while bbox_coords[i, m] < 0: + for d in range(m+1): + bbox_coords[i, d] += self.c_pbcbox.box[m][d] + while bbox_coords[i, m] >= self.c_pbcbox.box[m][m]: + for d in range(m+1): + bbox_coords[i, d] -= self.c_pbcbox.box[m][d] + else: + for i in range(natoms): + for m in range(DIM): + while bbox_coords[i, m] < 0: + bbox_coords[i, m] += self.c_pbcbox.box[m][m] + while bbox_coords[i, m] >= self.c_pbcbox.box[m][m]: + bbox_coords[i, m] -= self.c_pbcbox.box[m][m] return bbox_coords def put_atoms_in_bbox(self, real[:,::1] coords): @@ -294,12 +314,6 @@ cdef class PBCBox(object): # Neighbor Search Stuff # ######################################################################################################################## -cdef struct ns_grid: - ns_int size - ns_int[DIM] ncells - real[DIM] cellsize - ns_int *nbeads - ns_int **beadids cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: cdef ns_neighborhood_holder *holder @@ -327,7 +341,7 @@ cdef void free_neighborhood_holder(ns_neighborhood_holder *holder) nogil: cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]neighborcoords, ns_grid *grid, PBCBox box, real cutoff2) nogil: cdef ns_int d, m - cdef ns_int xi, yi, zi, bid + cdef ns_int xi, yi, zi, bid, i_bead cdef real d2 cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords @@ -335,6 +349,7 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei cdef bint skip cdef ns_int nchecked = 0, icheck cdef ns_int cell_index + cdef ns_int cell_offset cdef ns_neighborhood *neighborhood = malloc(sizeof(ns_neighborhood)) if neighborhood == NULL: @@ -374,21 +389,23 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei (shifted_coord[XX] / grid.cellsize[XX]) # Just a safeguard - if cell_index >= grid.size: - continue + #if cell_index >= grid.size: + # continue # Check the cell index was not already selected - skip = False - for icheck in range(nchecked): - if already_checked[icheck] == cell_index: - skip = True - break - if skip: - continue + #skip = False + #for icheck in range(nchecked): + # if already_checked[icheck] == cell_index: + # skip = True + # break + #if skip: + # continue # Search for neighbors inside this cell for i_bead in range(grid.nbeads[cell_index]): - bid = grid.beadids[cell_index][i_bead] + cell_offset = cell_index * grid.beadpercell + i_bead + + bid = grid.beadids[cell_offset] box.fast_pbc_dx(current_coords, &neighborcoords[bid, XX], dx) @@ -411,8 +428,8 @@ cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]nei return NULL # Register the cell as checked - already_checked[nchecked] = cell_index - nchecked += 1 + #already_checked[nchecked] = cell_index + #nchecked += 1 return neighborhood @@ -462,8 +479,8 @@ cdef class NSResults(object): """ cdef PBCBox box cdef readonly real cutoff - cdef real[:, ::1] grid_coords - cdef real[:, ::1] ref_coords + cdef np.ndarray grid_coords + cdef np.ndarray ref_coords cdef ns_int **nids cdef ns_int *nsizes cdef ns_int size @@ -471,6 +488,16 @@ cdef class NSResults(object): cdef list coordinates cdef list distances + def __dealloc__(self): + if self.nids != NULL: + for i in range(self.size): + if self.nids[i] != NULL: + free(self.nids[i]) + free(self.nids) + + if self.nsizes != NULL: + free(self.nsizes) + def __init__(self, PBCBox box, real cutoff): self.box = box self.cutoff = cutoff @@ -491,8 +518,8 @@ cdef class NSResults(object): cdef ns_int nid, i cdef ns_neighborhood *neighborhood - self.grid_coords = grid_coords.copy() - self.ref_coords = ref_coords.copy() + self.grid_coords = np.asarray(grid_coords) + self.ref_coords = np.asarray(ref_coords) # Allocate memory self.nsizes = malloc(sizeof(ns_int) * holder.size) @@ -521,17 +548,6 @@ cdef class NSResults(object): self.size = holder.size - def __dealloc__(self): - if self.nids != NULL: - for i in range(self.size): - if self.nids[i] != NULL: - free(self.nids[i]) - free(self.nids) - - if self.nsizes != NULL: - free(self.nsizes) - - def get_indices(self): """ Return Neighbors indices. @@ -594,6 +610,8 @@ cdef class NSResults(object): cdef ns_int i, nid, size, j, beadid cdef rvec ref, other, dx cdef real dist + cdef real[:, ::1] ref_coords = self.ref_coords + cdef real[:, ::1] grid_coords = self.grid_coords if self.distances is None: distances = [] @@ -602,11 +620,11 @@ cdef class NSResults(object): size = self.nsizes[nid] tmp_values = np.empty((size), dtype=np.float32) - ref = &self.ref_coords[nid, 0] + ref = &ref_coords[nid, 0] for i in range(size): beadid = self.nids[nid][i] - other = &self.grid_coords[beadid, 0] + other = &grid_coords[beadid, 0] tmp_values[i] = self.box.fast_distance(ref, other) @@ -616,6 +634,577 @@ cdef class NSResults(object): return self.distances +cdef class NSResults2(object): + cdef readonly real cutoff + cdef ns_int npairs + cdef bint debug + + cdef real[:, ::1] grid_coords # shape: size, DIM + cdef ns_int[:] search_ids + + cdef ns_int allocation_size + cdef ns_int[:, ::1] pairs# shape: pair_allocation, 2 + cdef real[:] pair_distances2 # shape: pair_allocation, 2 + + cdef list indices_buffer + cdef list coordinates_buffer + cdef list distances_buffer + cdef np.ndarray pairs_buffer + cdef np.ndarray pair_distances_buffer + + def __init__(self, real cutoff, real[:, ::1]grid_coords,ns_int[:] search_ids, debug=False): + + self.debug = debug + self.cutoff = cutoff + self.grid_coords = grid_coords + self.search_ids = search_ids + + # Preallocate memory + self.allocation_size = grid_coords.shape[0] + self.pairs = np.empty((self.allocation_size, 2), dtype=np.int) + self.pair_distances2 = np.empty(self.allocation_size, dtype=np.float32) + self.npairs = 0 + + # Buffer + self.indices_buffer = None + self.coordinates_buffer = None + self.distances_buffer = None + self.pair_distances_buffer = None + + cdef int add_neighbors(self, ns_int beadid_i, ns_int beadid_j, real distance2) nogil except 0: + # Reallocate memory if needed + if self.npairs >= self.allocation_size: + # We need to reallocate memory + with gil: + self.resize(self.allocation_size + (self.allocation_size * 0.5 + 1)) + + # Actually store pair and distance squared + if beadid_i < beadid_j: + self.pairs[self.npairs, 0] = beadid_i + self.pairs[self.npairs, 1] = beadid_j + else: + self.pairs[self.npairs, 1] = beadid_i + self.pairs[self.npairs, 0] = beadid_j + self.pair_distances2[self.npairs] = distance2 + self.npairs += 1 + + return self.npairs + + cdef resize(self, ns_int new_size): + cdef ns_int[:, ::1] pair_buffer + cdef real[:] dists_buffer + cdef ns_int i, j + + if new_size < self.npairs: + # Silently ignored the request + return + + if self.allocation_size >= new_size: + if self.debug: + print("NSresults reallocation requested but not needed ({} requested but {} already allocated)".format(new_size, self.allocation_size)) + return + + self.allocation_size = new_size + + if self.debug: + print("NSresults reallocated to {} pairs".format(self.allocation_size)) + + # Note: np.empty + update is faster than resize + # Allocating memory + pair_buffer = self.pairs + self.pairs = np.empty((self.allocation_size, 2), dtype=np.int) + + + dists_buffer = self.pair_distances2 + self.pair_distances2 = np.empty(self.allocation_size, dtype=np.float32) + + + # Update values + with nogil: + for i in range(self.npairs): + for j in range(2): + self.pairs[i, j] = pair_buffer[i, j] + + self.pair_distances2[i] = dists_buffer[i] + + def get_pairs(self): + if self.pairs_buffer is None: + self.pairs_buffer = np.array(self.pairs[:self.npairs]) + return self.pairs_buffer + + def get_pair_distances(self): + if self.pair_distances_buffer is None: + self.pair_distances_buffer = np.sqrt(self.pair_distances2[:self.npairs]) + return self.pair_distances_buffer + + cdef create_buffers(self): + cdef ns_int i, beadid_i, beadid_j + cdef real dist2 + cdef real[:] coord_i, coord_j + from collections import defaultdict + + indices_buffer = defaultdict(list) + coords_buffer = defaultdict(list) + dists_buffer = defaultdict(list) + + for i in range(self.npairs): + beadid_i = self.pairs[i, 0] + beadid_j = self.pairs[i, 1] + + dist2 = self.pair_distances2[i] + coord_i = self.grid_coords[beadid_i] + coord_j = self.grid_coords[beadid_j] + + indices_buffer[beadid_i].append(beadid_j) + indices_buffer[beadid_j].append(beadid_i) + + coords_buffer[beadid_i].append(coord_j) + coords_buffer[beadid_j].append((coord_i)) + + dists_buffer[beadid_i].append(dist2) + dists_buffer[beadid_j].append(dist2) + + self.indices_buffer = [] + self.coordinates_buffer = [] + self.distances_buffer = [] + + for elm in self.search_ids: + sorted_indices = np.argsort(indices_buffer[elm]) + self.indices_buffer.append(np.array(indices_buffer[elm])[sorted_indices]) + self.coordinates_buffer.append(np.array(coords_buffer[elm])[sorted_indices]) + self.distances_buffer.append(np.sqrt(dists_buffer[elm])[sorted_indices]) + + def get_indices(self): + if self.indices_buffer is None: + self.create_buffers() + return self.indices_buffer + + def get_distances(self): + if self.distances_buffer is None: + self.create_buffers() + return self.distances_buffer + + def get_coordinates(self): + if self.coordinates_buffer is None: + self.create_buffers() + return self.coordinates_buffer + + + +cdef class NSGrid(object): + cdef bint debug + cdef readonly real cutoff + cdef ns_int size + cdef ns_int ncoords + cdef ns_int[DIM] ncells + cdef ns_int[DIM] cell_offsets + cdef real[DIM] cellsize + cdef ns_int[:] nbeads # size + cdef ns_int[:, ::1] beadids # size * beadpercell + cdef ns_int[:] cellids # ncoords + cdef ns_int[:, ::1] cellxyz + + def __init__(self, ncoords, cutoff, PBCBox box, max_size, debug=False): + cdef ns_int i, x, y, z + cdef ns_int ncellx, ncelly, ncellz, size, nbeadspercell + cdef real bbox_vol + self.debug = debug + + self.ncoords = ncoords + + # Calculate best cutoff + self.cutoff = cutoff + bbox_vol = box.c_pbcbox.box[XX][XX] * box.c_pbcbox.box[YY][YY] * box.c_pbcbox.box[YY][YY] + size = bbox_vol/cutoff**3 + nbeadspercell = ncoords/size + while bbox_vol/self.cutoff**3 > max_size: + self.cutoff *= 1.2 + + + for i in range(DIM): + self.ncells[i] = (box.c_pbcbox.box[i][i] / self.cutoff) + if self.ncells[i] == 0: + self.ncells[i] = 1 + self.cellsize[i] = box.c_pbcbox.box[i][i] / self.ncells[i] + self.size = self.ncells[XX] * self.ncells[YY] * self.ncells[ZZ] + + if self.debug: + print("NSGrid: Requested cutoff: {:.3f} (Ncells={}, Avg # of beads per cell={}), Optimized cutoff= {:.3f} (Ncells={}, Avg # of beads per cell={})".format( + cutoff, size, nbeadspercell, + self.cutoff, self.size, (ncoords / self.size) + )) + print("NSGrid: Size={}x{}x{}={}".format(self.ncells[XX], self.ncells[YY], self.ncells[ZZ], self.size)) + + self.cell_offsets[XX] = 0 + self.cell_offsets[YY] = self.ncells[XX] + self.cell_offsets[ZZ] = self.ncells[XX] * self.ncells[YY] + + # Allocate memory + self.nbeads = np.zeros(self.size, dtype=np.int) + self.cellids = np.empty(self.ncoords, dtype=np.int) + #self.cellxyz = np.empty((self.size, DIM), dtype=np.int) + + #i = 0 + #for z in range(self.ncells[ZZ]): + # for y in range(self.ncells[YY]): + # for x in range(self.ncells[XX]): + # self.cellxyz[i, XX] = x + # self.cellxyz[i, YY] = y + # self.cellxyz[i, ZZ] = z + # i += 1 + + # Number of maximum bead per cell is not known, so wa can not allocate memory + self.beadids = None + + cdef ns_int coord2cellid(self, rvec coord) nogil: + return (coord[ZZ] / self.cellsize[ZZ]) * (self.ncells[XX] * self.ncells[YY]) +\ + (coord[YY] / self.cellsize[YY]) * self.ncells[XX] + \ + (coord[XX] / self.cellsize[XX]) + + cdef bint cellid2cellxyz(self, ns_int cellid, ivec cellxyz) nogil: + if cellid < 0: + return False + if cellid >= self.size: + return False + + cellxyz[ZZ] = (cellid / self.cell_offsets[ZZ]) + cellid -= cellxyz[ZZ] * self.cell_offsets[ZZ] + + cellxyz[YY] = (cellid / self.cell_offsets[YY]) + cellxyz[XX] = cellid - cellxyz[YY] * self.cell_offsets[YY] + + return True + + cdef fill_grid(self, real[:, ::1] coords): + cdef ns_int i, cellindex = -1, nbeads_max = 0 + cdef ns_int ncoords = coords.shape[0] + + cdef ns_int *beadcounts = NULL + + # Allocate memory + beadcounts = PyMem_Malloc(sizeof(ns_int) * self.size) + if not beadcounts: + raise MemoryError("Could not allocate memory for bead count buffer") + + + with nogil: + # Initialize buffers + for i in range(self.size): + beadcounts[i] = 0 + + # First loop: find cellindex for each bead + for i in range(ncoords): + cellindex = self.coord2cellid(&coords[i, 0]) + + self.nbeads[cellindex] += 1 + self.cellids[i] = cellindex + + if self.nbeads[cellindex] > nbeads_max: + nbeads_max = self.nbeads[cellindex] + + # Allocate memory + with gil: + self.beadids = np.empty((self.size, nbeads_max), dtype=np.int) + + # Second loop: fill grid + for i in range(ncoords): + + # Add bead to grid cell + cellindex = self.cellids[i] + self.beadids[cellindex, beadcounts[cellindex]] = i + beadcounts[cellindex] += 1 + + + # Now we can free the allocation buffer + PyMem_Free(beadcounts) + + + +# Python interface +cdef class FastNS2(object): + cdef bint debug + cdef PBCBox box + cdef readonly real[:, ::1] coords + cdef real[:, ::1] coords_bbox + cdef readonly real cutoff + cdef bint prepared + cdef NSGrid grid + + def __init__(self, u, cutoff, coords=None, prepare=True, debug=False, max_gridsize=5000): + import MDAnalysis as mda + from MDAnalysis.lib.mdamath import triclinic_vectors + + self.debug = debug + + if not isinstance(u, mda.Universe): + raise TypeError("FastNS class must be initialized with a valid MDAnalysis.Universe instance") + box = triclinic_vectors(u.dimensions) + + if box.shape != (3, 3): + raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") + + self.box = PBCBox(box) + + + if coords is None: + coords = u.atoms.positions + + self.coords = coords.copy() + + self.coords_bbox = self.box.fast_put_atoms_in_bbox(coords) + + if not prepare: + return + + if self.cutoff < 0: + raise ValueError("Cutoff must be positive!") + if self.cutoff * self.cutoff > self.box.c_pbcbox.max_cutoff2: + raise ValueError("Cutoff greater than maximum cutoff ({:.3f}) given the PBC") + self.cutoff = cutoff + + self.grid = NSGrid(self.coords_bbox.shape[0], cutoff, self.box, max_gridsize, debug=debug) + self.prepared = False + + if prepare: + self.prepare() + + + def prepare(self, force=False): + if self.prepared and not force: + return + + self.grid.fill_grid(self.coords_bbox) + + self.prepared = True + + def search(self, search_ids, min_size_increment=10): + cdef ns_int[:] search_ids_view, neighbors_view + cdef ns_int nid, i, j, size_search + cdef NSResults2 results + + cdef ns_int current_beadid + cdef rvec current_coords + + cdef ns_int cellx, celly, cellz, cellindex_adjacent + cdef ivec cellxyz, debug_cellxyz + + cdef real[:, ::1] search_coords + + #Temp stuff + cdef ns_int size = self.coords_bbox.shape[0] + cdef ns_int d, m + cdef ns_int xi, yi, zi, bid, i_bead + cdef real d2 + cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords + + cdef bint already_checked[27] + cdef bint skip + cdef ns_int nchecked = 0, icheck + cdef ns_int cellindex + cdef ns_int cell_offset + + cdef ns_int[:] neighbor_ids + cdef ns_int[:] nneighbors + cdef ns_int nid_offset + + cdef real cutoff2 = self.cutoff * self.cutoff + + cdef ns_int[:] checked + + cdef ns_int npairs = 0 + cdef ns_int guessed_size = 0 + cdef bint first_ns = True + + import sys + flush = sys.stdout.flush + + if not self.prepared: + self.prepare() + + + search_ids_view = search_ids + size_search = search_ids.shape[0] + + checked = np.zeros(size, dtype=np.int) + + results = NSResults2(self.cutoff, self.coords_bbox, search_ids, self.debug) + + with nogil: + for i in range(size_search): + current_beadid = search_ids_view[i] + + cellindex = self.grid.cellids[current_beadid] + self.grid.cellid2cellxyz(cellindex, cellxyz) + + if self.debug: + + with gil: + print("Checking neighbors for bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]:" .format( + current_beadid, + self.coords[current_beadid, XX], self.coords[current_beadid, YY], self.coords[current_beadid, ZZ], + self.coords_bbox[current_beadid, XX], self.coords_bbox[current_beadid, YY], self.coords_bbox[current_beadid, ZZ], + cellxyz[XX], cellxyz[YY], cellxyz[ZZ])) + + + for xi in range(DIM): + if xi == 0: + if self.coords_bbox[current_beadid, XX] - self.cutoff > self.grid.cellsize[XX] * cellxyz[XX]: + if self.debug: + with gil: + print("\n#-> Bead X={:.3f}, Cell X={:.3f}, cutoff={:.3f} -> -X shift is ignored".format(self.coords_bbox[current_beadid, XX], self.grid.cellsize[XX] * cellxyz[XX], self.cutoff)) + continue + + if xi == 2: + if self.coords_bbox[current_beadid, XX] + self.cutoff < self.grid.cellsize[XX] * (cellxyz[XX] + 1): + if self.debug: + with gil: + print("\n#-> Bead X={:.3f}, Next cell X={:.3f}, cutoff={:.3f} -> +X shift is ignored".format(self.coords_bbox[current_beadid, XX], self.grid.cellsize[XX] * (cellxyz[XX] + 1), self.cutoff)) + continue + + for yi in range(DIM): + if xi == 0: + if self.coords_bbox[current_beadid, YY] - self.cutoff > self.grid.cellsize[YY] * cellxyz[YY]: + if self.debug: + with gil: + print("\n#-> Bead Y={:.3f}, Cell Y={:.3f}, cutoff={:.3f} -> -Y shift is ignored".format(self.coords_bbox[current_beadid, YY], self.grid.cellsize[YY] * cellxyz[YY], self.cutoff)) + continue + + if xi == 2: + if self.coords_bbox[current_beadid, YY] + self.cutoff < self.grid.cellsize[YY] * (cellxyz[YY] + 1): + if self.debug: + with gil: + print("\n#-> Bead Y={:.3f}, Next cell Y={:.3f}, cutoff={:.3f} -> +Y shift is ignored".format(self.coords_bbox[current_beadid, YY], self.grid.cellsize[YY] * (cellxyz[YY] + 1), self.cutoff)) + continue + + + for zi in range(DIM): + if xi == 0: + if self.coords_bbox[current_beadid, ZZ] - self.cutoff > self.grid.cellsize[ZZ] * cellxyz[ZZ]: + if self.debug: + with gil: + print("\n#-> Bead Z={:.3f}, Cell Z={:.3f}, cutoff={:.3f} -> -Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[ZZ] * cellxyz[ZZ], self.cutoff)) + continue + + if xi == 2: + if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.grid.cellsize[ZZ] * (cellxyz[ZZ] + 1): + if self.debug: + with gil: + print("\n#-> Bead Z={:.3f}, Next cell Z={:.3f}, cutoff={:.3f} -> +Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[XX] * (cellxyz[ZZ] + 1), self.cutoff)) + continue + + # Calculate and/or reinitialize shifted coordinates + shifted_coord[XX] = self.coords[current_beadid, XX] + (xi - 1) * self.grid.cellsize[XX] + shifted_coord[YY] = self.coords[current_beadid, YY] + (yi - 1) * self.grid.cellsize[YY] + shifted_coord[ZZ] = self.coords[current_beadid, ZZ] + (zi - 1) * self.grid.cellsize[ZZ] + + # Make sure the shifted coordinates is inside the brick-shaped box + for m in range(DIM - 1, -1, -1): + + while shifted_coord[m] < 0: + for d in range(m+1): + shifted_coord[d] += self.box.c_pbcbox.box[m][d] + + + while shifted_coord[m] >= self.box.c_pbcbox.box[m][m]: + for d in range(m+1): + shifted_coord[d] -= self.box.c_pbcbox.box[m][d] + + # Get the cell index corresponding to the coord + cellindex = self.grid.coord2cellid(shifted_coord) + + if self.debug: + self.grid.cellid2cellxyz(cellindex, debug_cellxyz) + with gil: + dist_shift = self.box.fast_distance(&self.coords[current_beadid, XX], shifted_coord) + grid_shift = np.array([(xi - 1) * self.grid.cellsize[XX], + (yi - 1) * self.grid.cellsize[YY], + (zi - 1) * self.grid.cellsize[ZZ]]) + print("-> Checking cell#{} ({},{},{}) for neighbors (dshift={:.3f}, grid_shift=({:.3f},{:.3f},{:.3f}->{:.3f})".format( + cellindex, + debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], + dist_shift, + grid_shift[XX], grid_shift[YY], grid_shift[ZZ], + np.sqrt(np.sum(grid_shift**2)) + )) + if dist_shift > 2* self.cutoff: + print(" \_ This cell should be ignored!") + else: + print(" \_ This cell is needed") + + for j in range(self.grid.nbeads[cellindex]): + bid = self.grid.beadids[cellindex, j] + + if checked[bid] != 0: + continue + + d2 = self.box.fast_distance2(&self.coords_bbox[current_beadid, XX], &self.coords_bbox[bid, XX]) + + if self.debug and False: + self.grid.cellid2cellxyz(cellindex, debug_cellxyz) + with gil: + print( + "Beads #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) and #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) are tested (d2={:.3f})".format( + current_beadid, + cellxyz[XX], cellxyz[YY], cellxyz[ZZ], + self.coords_bbox[current_beadid, XX], + self.coords_bbox[current_beadid, YY], + self.coords_bbox[current_beadid, ZZ], + bid, + debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], + self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], + self.coords_bbox[bid, ZZ], + d2)) + + if d2 < cutoff2: + + if d2 < EPSILON: + continue + + if self.debug: + self.grid.cellid2cellxyz(cellindex, debug_cellxyz) + with gil: + self.box.fast_pbc_dx(&self.coords[current_beadid, XX], &self.coords[bid, XX], dx) + dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) + + print(" \_ Neighbor found: bead#{} (cell[{},{},{}]) -> dx={} -> d={:.3f}".format(bid, debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], dx_py, np.sqrt(d2))) + results.add_neighbors(current_beadid, bid, d2) + npairs += 1 + + if first_ns: + guessed_size = ((npairs + 1) * 0.75 * size_search) + with gil: + if self.debug: + print("Neighbors of first beads= {} -> Approximated total of pairs={}".format(npairs, guessed_size)) + results.resize(guessed_size) + + first_ns = False + checked[current_beadid] = 1 + + if self.debug: + print("Total number of pairs={}".format(npairs)) + + ref_bead = 13937 + beads = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, 47451]) - 1 + for bid in beads: + self.box.fast_pbc_dx(&self.coords[ref_bead, XX], &self.coords[bid, XX], dx) + dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) + self.box.fast_pbc_dx(&self.coords_bbox[ref_bead, XX], &self.coords_bbox[bid, XX], dx) + rect_dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) + self.grid.cellid2cellxyz(self.grid.coord2cellid(&self.coords_bbox[bid, XX]), cellxyz) + print("Bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]: dx=[{:.3f},{:.3f},{:.3f}] -> dist: {:.3f} ({}) - rect_dist: {:.3f} ({})".format( + bid, + self.coords[bid, XX], self.coords[bid, YY], self.coords[bid,ZZ], + self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], self.coords_bbox[bid,ZZ], + cellxyz[XX], cellxyz[YY], cellxyz[ZZ], + dx[XX], dx[YY], dx[ZZ], + np.sqrt(np.sum(dx_py**2)), + self.box.fast_distance(&self.coords[ref_bead, XX], &self.coords[bid, XX]) <= self.cutoff, + np.sqrt(np.sum(rect_dx_py**2)), + np.sqrt(np.sum(rect_dx_py**2)) <= self.cutoff + )) + + return results + # Python interface cdef class FastNS(object): @@ -651,14 +1240,16 @@ cdef class FastNS(object): def __dealloc__(self): cdef ns_int i + # Deallocate NS grid if self.grid.nbeads != NULL: free(self.grid.nbeads) - for i in range(self.grid.size): - if self.grid.beadids[i] != NULL: - free(self.grid.beadids[i]) - free(self.grid.beadids) + if self.grid.beadids != NULL: + free(self.grid.beadids) + + if self.grid.cellids != NULL: + free(self.grid.cellids) self.grid.size = 0 @@ -680,11 +1271,11 @@ cdef class FastNS(object): def prepare(self, force=False): cdef ns_int i, cellindex = -1 - cdef ns_int *allocated_size = NULL cdef ns_int ncoords = self.coords.shape[0] - cdef ns_int allocation_guess cdef rvec *coords = &self.coords_bbox[0, 0] + cdef ns_int *beadcounts = NULL + if self.prepared and not force: print("NS already prepared, nothing to do!") @@ -704,57 +1295,55 @@ cdef class FastNS(object): self.grid.cellsize[i] = self.box.c_pbcbox.box[i][i] / self.grid.ncells[i] self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] - # This is just a guess on how much memory we might for each grid cell: - # we just assume an average bead density and we take four times this density just to be safe - allocation_guess = (4 * (ncoords / self.grid.size + 1)) + # Allocate memory for temporary buffers + self.grid.cellids = malloc(sizeof(ns_int) * ncoords) + if self.grid.cellids == NULL: + with gil: + raise MemoryError("Could not allocate memory for cell ids") + + beadcounts = malloc(sizeof(ns_int) * self.grid.size) + if beadcounts == NULL: + with gil: + raise MemoryError("Could not allocate memory for bead count buffer") - # Allocate memory for the grid self.grid.nbeads = malloc(sizeof(ns_int) * self.grid.size) if self.grid.nbeads == NULL: with gil: raise MemoryError("Could not allocate memory for NS grid") - # Allocate memory from temporary allocation counter - allocated_size = malloc(sizeof(ns_int) * self.grid.size) - if allocated_size == NULL: - # No need to free grid.nbeads as it will be freed by destroy_nsgrid called by __dealloc___ - with gil: - raise MemoryError("Could not allocate memory for allocation buffer") - - # Pre-allocate some memory for grid cells + # Initialize buffers for i in range(self.grid.size): self.grid.nbeads[i] = 0 - allocated_size[i] = allocation_guess + beadcounts[i] = 0 - self.grid.beadids = malloc(sizeof(ns_int *) * self.grid.size) - if self.grid.beadids == NULL: - with gil: - raise MemoryError("Could not allocate memory for grid cells") - for i in range(self.grid.size): - self.grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) - if self.grid.beadids[i] == NULL: - with gil: - raise MemoryError("Could not allocate memory for grid cell") - - # Populate grid cells using the coordinates (ie do the heavy work) + # First loop: find cellindex for each bead for i in range(ncoords): cellindex = (coords[i][ZZ] / self.grid.cellsize[ZZ]) * (self.grid.ncells[XX] * self.grid.ncells[YY]) +\ (coords[i][YY] / self.grid.cellsize[YY]) * self.grid.ncells[XX] + \ (coords[i][XX] / self.grid.cellsize[XX]) - self.grid.beadids[cellindex][self.grid.nbeads[cellindex]] = i self.grid.nbeads[cellindex] += 1 + self.grid.cellids[i] = cellindex + + if self.grid.nbeads[cellindex] > self.grid.beadpercell: + self.grid.beadpercell = self.grid.nbeads[cellindex] + + # Allocate memory and set offsets + self.grid.beadids = malloc(sizeof(ns_int) * self.grid.size * self.grid.beadpercell) + if self.grid.beadids == NULL: + with gil: + raise MemoryError("Could not allocate memory for NS grid bead ids") - # We need to allocate more memory (simply double the amount of memory as - # 1. it should barely be needed - # 2. the size should stay fairly reasonable - if self.grid.nbeads[cellindex] >= allocated_size[cellindex]: - allocated_size[cellindex] *= 2 - self.grid.beadids[cellindex] = realloc( self.grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) + # Second loop: fill grid + for i in range(ncoords): + cellindex = self.grid.cellids[i] + self.grid.beadids[cellindex * self.grid.beadpercell + beadcounts[cellindex]] = i + beadcounts[cellindex] += 1 # Now we can free the allocation buffer - free(allocated_size) + free(beadcounts) + self.prepared = True @@ -787,7 +1376,6 @@ cdef class FastNS(object): if holder == NULL: raise MemoryError("Could not allocate memory to run NS core") - results = NSResults(self.box, self.cutoff) results.populate(holder, self.coords, search_coords_view) @@ -795,3 +1383,101 @@ cdef class FastNS(object): free_neighborhood_holder(holder) return results + + def search2(self, search_ids): + cdef ns_int[:] search_ids_view, neighbors_view + cdef ns_int nid, i, j, size_search + cdef ns_neighborhood_holder *holder + cdef ns_neighborhood *neighborhood + + cdef ns_int current_beadid + cdef rvec current_coords + + #Temp stuff + cdef ns_int size = self.coords_bbox.shape[0] + cdef ns_int d, m + cdef ns_int xi, yi, zi, bid, i_bead + cdef real d2 + cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords + + cdef bint already_checked[27] + cdef bint skip + cdef ns_int nchecked = 0, icheck + cdef ns_int cell_index + cdef ns_int cell_offset + + cdef ns_int[:] neighbor_ids + cdef ns_int[:] nneighbors + cdef ns_int nid_offset + + cdef real cutoff2 = self.cutoff * self.cutoff + + + if not self.prepared: + self.prepare() + + nid_offset = self.grid.beadpercell * 27 + neighbor_ids = np.empty(size * nid_offset, dtype=np.int) + nneighbors = np.zeros(size, dtype=np.int) + + + search_ids_view = search_ids + size_search = search_ids_view.shape[0] + + with nogil: + + for current_beadid in range(size): + + for zi in range(3): + for yi in range(3): + for xi in range(3): + # Calculate and/or reinitialize shifted coordinates + shifted_coord[XX] = self.coords_bbox[current_beadid, XX] + (xi - 1) * self.grid.cellsize[XX] + shifted_coord[YY] = self.coords_bbox[current_beadid, YY] + (yi - 1) * self.grid.cellsize[YY] + shifted_coord[ZZ] = self.coords_bbox[current_beadid, ZZ] + (zi - 1) * self.grid.cellsize[ZZ] + + # Make sure the shifted coordinates is inside the brick-shaped box + for m in range(DIM - 1, -1, -1): + + while shifted_coord[m] < 0: + for d in range(m+1): + shifted_coord[d] += self.box.c_pbcbox.box[m][d] + + + while shifted_coord[m] >= self.box.c_pbcbox.box[m][m]: + for d in range(m+1): + shifted_coord[d] -= self.box.c_pbcbox.box[m][d] + + # Get the cell index corresponding to the coord + cell_index = (shifted_coord[ZZ] / self.grid.cellsize[ZZ]) * (self.grid.ncells[XX] * self.grid.ncells[YY]) +\ + (shifted_coord[YY] / self.grid.cellsize[YY]) * self.grid.ncells[XX] + \ + (shifted_coord[XX] / self.grid.cellsize[XX]) + + # Search for neighbors inside this cell + for i_bead in range(self.grid.nbeads[cell_index]): + cell_offset = cell_index * self.grid.beadpercell + i_bead + + bid = self.grid.beadids[cell_offset] + + if bid <= current_beadid: + continue + + self.box.fast_pbc_dx(&self.coords_bbox[current_beadid, XX], &self.coords_bbox[bid, XX], dx) + + d2 = rvec_norm2(dx) + + if d2 < cutoff2: + neighbor_ids[current_beadid * nid_offset + nneighbors[current_beadid]] = bid + nneighbors[current_beadid] += 1 + + neighbor_ids[bid * nid_offset + nneighbors[bid]] = current_beadid + nneighbors[bid] += 1 + + + + + results = [] + #for bid in search_ids: + # results.append(np.asarray(neighbor_ids[bid * nid_offset : bid * nid_offset + nneighbors[bid]])) + + return results From 089eb09e815d3f240c43f1709dc3eeac9c936a7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Mon, 16 Jul 2018 11:45:11 +0200 Subject: [PATCH 20/30] Unneeded code removed from c_gridsearch.pyx --- package/MDAnalysis/lib/c_gridsearch.pyx | 526 ++++++++---------- .../MDAnalysisTests/lib/test_gridsearch.py | 145 +++-- 2 files changed, 307 insertions(+), 364 deletions(-) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx index ffdb91fed7d..a1f519eac37 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/c_gridsearch.pyx @@ -55,6 +55,7 @@ from libc.math cimport sqrt import numpy as np cimport numpy as np +cimport cython ctypedef np.int_t ns_int ctypedef np.float32_t real @@ -473,7 +474,7 @@ cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, return holder -cdef class NSResults(object): +cdef class NSResultsOld(object): """ Class used to store results returned by `MDAnalysis.lib.grid.FastNS.search` """ @@ -634,7 +635,7 @@ cdef class NSResults(object): return self.distances -cdef class NSResults2(object): +cdef class NSResults(object): cdef readonly real cutoff cdef ns_int npairs cdef bint debug @@ -643,8 +644,8 @@ cdef class NSResults2(object): cdef ns_int[:] search_ids cdef ns_int allocation_size - cdef ns_int[:, ::1] pairs# shape: pair_allocation, 2 - cdef real[:] pair_distances2 # shape: pair_allocation, 2 + cdef ipair *pairs # shape: pair_allocation + cdef real *pair_distances2 # shape: pair_allocation cdef list indices_buffer cdef list coordinates_buffer @@ -660,81 +661,96 @@ cdef class NSResults2(object): self.search_ids = search_ids # Preallocate memory - self.allocation_size = grid_coords.shape[0] - self.pairs = np.empty((self.allocation_size, 2), dtype=np.int) - self.pair_distances2 = np.empty(self.allocation_size, dtype=np.float32) + self.allocation_size = search_ids.shape[0] + 1 + self.pairs = PyMem_Malloc(sizeof(ipair) * self.allocation_size) + if not self.pairs: + MemoryError("Could not allocate memory for NSResults.pairs ({} bits requested)".format(sizeof(ipair) * self.allocation_size)) + self.pair_distances2 = PyMem_Malloc(sizeof(real) * self.allocation_size) + if not self.pair_distances2: + raise MemoryError("Could not allocate memory for NSResults.pair_distances2 ({} bits requested)".format(sizeof(real) * self.allocation_size)) self.npairs = 0 # Buffer self.indices_buffer = None self.coordinates_buffer = None self.distances_buffer = None + self.pairs_buffer = None self.pair_distances_buffer = None - cdef int add_neighbors(self, ns_int beadid_i, ns_int beadid_j, real distance2) nogil except 0: + def __dealloc__(self): + PyMem_Free(self.pairs) + PyMem_Free(self.pair_distances2) + + cdef int add_neighbors(self, ns_int beadid_i, ns_int beadid_j, real distance2) nogil: + # Important: If this function returns 0, it means that memory allocation failed + # Reallocate memory if needed if self.npairs >= self.allocation_size: # We need to reallocate memory - with gil: - self.resize(self.allocation_size + (self.allocation_size * 0.5 + 1)) + if self.resize(self.allocation_size + (self.allocation_size * 0.5 + 1)) == 0: + return 0 # Actually store pair and distance squared if beadid_i < beadid_j: - self.pairs[self.npairs, 0] = beadid_i - self.pairs[self.npairs, 1] = beadid_j + self.pairs[self.npairs][0] = beadid_i + self.pairs[self.npairs][1] = beadid_j else: - self.pairs[self.npairs, 1] = beadid_i - self.pairs[self.npairs, 0] = beadid_j + self.pairs[self.npairs][1] = beadid_i + self.pairs[self.npairs][0] = beadid_j self.pair_distances2[self.npairs] = distance2 self.npairs += 1 return self.npairs - cdef resize(self, ns_int new_size): - cdef ns_int[:, ::1] pair_buffer - cdef real[:] dists_buffer - cdef ns_int i, j + cdef int resize(self, ns_int new_size) nogil: + # Important: If this function returns 0, it means that memory allocation failed if new_size < self.npairs: # Silently ignored the request - return + return 1 if self.allocation_size >= new_size: if self.debug: - print("NSresults reallocation requested but not needed ({} requested but {} already allocated)".format(new_size, self.allocation_size)) - return + with gil: + print("NSresults: Reallocation requested but not needed ({} requested but {} already allocated)".format(new_size, self.allocation_size)) + return 1 self.allocation_size = new_size if self.debug: - print("NSresults reallocated to {} pairs".format(self.allocation_size)) + with gil: + print("NSresults: Reallocated to {} pairs".format(self.allocation_size)) - # Note: np.empty + update is faster than resize # Allocating memory - pair_buffer = self.pairs - self.pairs = np.empty((self.allocation_size, 2), dtype=np.int) - - - dists_buffer = self.pair_distances2 - self.pair_distances2 = np.empty(self.allocation_size, dtype=np.float32) + with gil: + self.pairs = PyMem_Realloc(self.pairs, sizeof(ipair) * self.allocation_size) + self.pair_distances2 = PyMem_Realloc(self.pair_distances2, sizeof(real) * self.allocation_size) + if not self.pairs: + return 0 - # Update values - with nogil: - for i in range(self.npairs): - for j in range(2): - self.pairs[i, j] = pair_buffer[i, j] + if not self.pair_distances2: + return 0 - self.pair_distances2[i] = dists_buffer[i] + return 1 def get_pairs(self): + cdef ns_int i + if self.pairs_buffer is None: - self.pairs_buffer = np.array(self.pairs[:self.npairs]) + self.pairs_buffer = np.empty((self.npairs, 2), dtype=np.int) + for i in range(self.npairs): + self.pairs_buffer[i, 0] = self.pairs[i][0] + self.pairs_buffer[i, 1] = self.pairs[i][1] return self.pairs_buffer def get_pair_distances(self): + cdef ns_int i if self.pair_distances_buffer is None: - self.pair_distances_buffer = np.sqrt(self.pair_distances2[:self.npairs]) + self.pair_distances_buffer = np.empty(self.npairs, dtype=np.float32) + for i in range(self.npairs): + self.pair_distances_buffer[i] = self.pair_distances2[i] + self.pair_distances_buffer = np.sqrt(self.pair_distances_buffer) return self.pair_distances_buffer cdef create_buffers(self): @@ -748,8 +764,8 @@ cdef class NSResults2(object): dists_buffer = defaultdict(list) for i in range(self.npairs): - beadid_i = self.pairs[i, 0] - beadid_j = self.pairs[i, 1] + beadid_i = self.pairs[i][0] + beadid_j = self.pairs[i][1] dist2 = self.pair_distances2[i] coord_i = self.grid_coords[beadid_i] @@ -799,10 +815,10 @@ cdef class NSGrid(object): cdef ns_int[DIM] ncells cdef ns_int[DIM] cell_offsets cdef real[DIM] cellsize - cdef ns_int[:] nbeads # size - cdef ns_int[:, ::1] beadids # size * beadpercell - cdef ns_int[:] cellids # ncoords - cdef ns_int[:, ::1] cellxyz + cdef ns_int nbeads_per_cell + cdef ns_int *nbeads # size + cdef ns_int *beadids # size * nbeads_per_cell + cdef ns_int *cellids # ncoords def __init__(self, ncoords, cutoff, PBCBox box, max_size, debug=False): cdef ns_int i, x, y, z @@ -820,7 +836,6 @@ cdef class NSGrid(object): while bbox_vol/self.cutoff**3 > max_size: self.cutoff *= 1.2 - for i in range(DIM): self.ncells[i] = (box.c_pbcbox.box[i][i] / self.cutoff) if self.ncells[i] == 0: @@ -840,21 +855,22 @@ cdef class NSGrid(object): self.cell_offsets[ZZ] = self.ncells[XX] * self.ncells[YY] # Allocate memory - self.nbeads = np.zeros(self.size, dtype=np.int) - self.cellids = np.empty(self.ncoords, dtype=np.int) - #self.cellxyz = np.empty((self.size, DIM), dtype=np.int) - - #i = 0 - #for z in range(self.ncells[ZZ]): - # for y in range(self.ncells[YY]): - # for x in range(self.ncells[XX]): - # self.cellxyz[i, XX] = x - # self.cellxyz[i, YY] = y - # self.cellxyz[i, ZZ] = z - # i += 1 - - # Number of maximum bead per cell is not known, so wa can not allocate memory - self.beadids = None + self.nbeads = PyMem_Malloc(sizeof(ns_int) * self.size) + if not self.nbeads: + raise MemoryError("Could not allocate memory from NSGrid.nbeads ({} bits requested)".format(sizeof(ns_int) * self.size)) + self.beadids = NULL + self.cellids = PyMem_Malloc(sizeof(ns_int) * self.ncoords) + if not self.cellids: + raise MemoryError("Could not allocate memory from NSGrid.cellids ({} bits requested)".format(sizeof(ns_int) * self.ncoords)) + self.nbeads_per_cell = 0 + + for i in range(self.size): + self.nbeads[i] = 0 + + def __dealloc__(self): + PyMem_Free(self.nbeads) + PyMem_Free(self.beadids) + PyMem_Free(self.cellids) cdef ns_int coord2cellid(self, rvec coord) nogil: return (coord[ZZ] / self.cellsize[ZZ]) * (self.ncells[XX] * self.ncells[YY]) +\ @@ -876,16 +892,14 @@ cdef class NSGrid(object): return True cdef fill_grid(self, real[:, ::1] coords): - cdef ns_int i, cellindex = -1, nbeads_max = 0 + cdef ns_int i, cellindex = -1 cdef ns_int ncoords = coords.shape[0] - cdef ns_int *beadcounts = NULL # Allocate memory beadcounts = PyMem_Malloc(sizeof(ns_int) * self.size) if not beadcounts: - raise MemoryError("Could not allocate memory for bead count buffer") - + raise MemoryError("Could not allocate memory for bead count buffer ({} bits requested)".format(sizeof(ns_int) * self.size)) with nogil: # Initialize buffers @@ -899,29 +913,30 @@ cdef class NSGrid(object): self.nbeads[cellindex] += 1 self.cellids[i] = cellindex - if self.nbeads[cellindex] > nbeads_max: - nbeads_max = self.nbeads[cellindex] + if self.nbeads[cellindex] > self.nbeads_per_cell: + self.nbeads_per_cell = self.nbeads[cellindex] # Allocate memory with gil: - self.beadids = np.empty((self.size, nbeads_max), dtype=np.int) + self.beadids = PyMem_Malloc(sizeof(ns_int) * self.size * self.nbeads_per_cell) #np.empty((self.size, nbeads_max), dtype=np.int) + if not self.beadids: + raise MemoryError("Could not allocate memory for NSGrid.beadids ({} bits requested)".format(sizeof(ns_int) * self.size * self.nbeads_per_cell)) # Second loop: fill grid for i in range(ncoords): # Add bead to grid cell cellindex = self.cellids[i] - self.beadids[cellindex, beadcounts[cellindex]] = i + self.beadids[cellindex * self.nbeads_per_cell + beadcounts[cellindex]] = i beadcounts[cellindex] += 1 - # Now we can free the allocation buffer PyMem_Free(beadcounts) # Python interface -cdef class FastNS2(object): +cdef class FastNS(object): cdef bint debug cdef PBCBox box cdef readonly real[:, ::1] coords @@ -977,69 +992,68 @@ cdef class FastNS2(object): self.prepared = True - def search(self, search_ids, min_size_increment=10): - cdef ns_int[:] search_ids_view, neighbors_view - cdef ns_int nid, i, j, size_search - cdef NSResults2 results + def search(self, search_ids=None, bint debug=False): + cdef ns_int i, j, size_search + cdef ns_int d, m + cdef NSResults results + cdef ns_int size = self.coords_bbox.shape[0] - cdef ns_int current_beadid + cdef ns_int current_beadid, bid cdef rvec current_coords - cdef ns_int cellx, celly, cellz, cellindex_adjacent + cdef ns_int cellindex, cellindex_adjacent, cellindex_probe cdef ivec cellxyz, debug_cellxyz cdef real[:, ::1] search_coords + cdef ns_int[:] search_ids_view - #Temp stuff - cdef ns_int size = self.coords_bbox.shape[0] - cdef ns_int d, m - cdef ns_int xi, yi, zi, bid, i_bead + cdef ns_int xi, yi, zi cdef real d2 - cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords + cdef rvec shifted_coord, probe, dx - cdef bint already_checked[27] - cdef bint skip - cdef ns_int nchecked = 0, icheck - cdef ns_int cellindex - cdef ns_int cell_offset + cdef ns_int nchecked = 0 - cdef ns_int[:] neighbor_ids - cdef ns_int[:] nneighbors - cdef ns_int nid_offset - cdef real cutoff2 = self.cutoff * self.cutoff + cdef real cutoff2 = self.cutoff * self.cutoff cdef ns_int[:] checked - cdef ns_int npairs = 0 - cdef ns_int guessed_size = 0 - cdef bint first_ns = True - import sys - flush = sys.stdout.flush if not self.prepared: self.prepare() + if search_ids is None: + search_ids=np.arange(size) + elif type(search_ids) == np.int: + search_ids = np.array([search_ids,], dtype=np.int) + elif type(search_ids) != np.ndarray: + search_ids = np.array(search_ids, dtype=np.int) search_ids_view = search_ids size_search = search_ids.shape[0] checked = np.zeros(size, dtype=np.int) - results = NSResults2(self.cutoff, self.coords_bbox, search_ids, self.debug) + results = NSResults(self.cutoff, self.coords_bbox, search_ids, self.debug) + + cdef bint memory_error = False + + if self.debug and debug: + print("FastNS: Debug flag is set to True for FastNS.search()") with nogil: for i in range(size_search): + if memory_error: + break current_beadid = search_ids_view[i] cellindex = self.grid.cellids[current_beadid] self.grid.cellid2cellxyz(cellindex, cellxyz) - if self.debug: - + if self.debug and debug: with gil: - print("Checking neighbors for bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]:" .format( + print("FastNS: Checking neighbors for bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]:" .format( current_beadid, self.coords[current_beadid, XX], self.coords[current_beadid, YY], self.coords[current_beadid, ZZ], self.coords_bbox[current_beadid, XX], self.coords_bbox[current_beadid, YY], self.coords_bbox[current_beadid, ZZ], @@ -1047,56 +1061,89 @@ cdef class FastNS2(object): for xi in range(DIM): - if xi == 0: - if self.coords_bbox[current_beadid, XX] - self.cutoff > self.grid.cellsize[XX] * cellxyz[XX]: - if self.debug: - with gil: - print("\n#-> Bead X={:.3f}, Cell X={:.3f}, cutoff={:.3f} -> -X shift is ignored".format(self.coords_bbox[current_beadid, XX], self.grid.cellsize[XX] * cellxyz[XX], self.cutoff)) - continue + if memory_error: + break - if xi == 2: - if self.coords_bbox[current_beadid, XX] + self.cutoff < self.grid.cellsize[XX] * (cellxyz[XX] + 1): - if self.debug: - with gil: - print("\n#-> Bead X={:.3f}, Next cell X={:.3f}, cutoff={:.3f} -> +X shift is ignored".format(self.coords_bbox[current_beadid, XX], self.grid.cellsize[XX] * (cellxyz[XX] + 1), self.cutoff)) - continue - - for yi in range(DIM): + if not self.box.is_triclinic: + # If box is not triclinic (ie rect), when can already check if the shift can be skipped (ie cutoff inside the cell) if xi == 0: - if self.coords_bbox[current_beadid, YY] - self.cutoff > self.grid.cellsize[YY] * cellxyz[YY]: - if self.debug: + if self.coords_bbox[current_beadid, XX] - self.cutoff > self.grid.cellsize[XX] * cellxyz[XX]: + if self.debug and debug: with gil: - print("\n#-> Bead Y={:.3f}, Cell Y={:.3f}, cutoff={:.3f} -> -Y shift is ignored".format(self.coords_bbox[current_beadid, YY], self.grid.cellsize[YY] * cellxyz[YY], self.cutoff)) + print("FastNS: -> Bead X={:.3f}, Cell X={:.3f}, cutoff={:.3f} -> -X shift ignored".format( + self.coords_bbox[current_beadid, XX], + self.grid.cellsize[XX] * cellxyz[XX], + self.cutoff + )) continue if xi == 2: - if self.coords_bbox[current_beadid, YY] + self.cutoff < self.grid.cellsize[YY] * (cellxyz[YY] + 1): - if self.debug: + if self.coords_bbox[current_beadid, XX] + self.cutoff < self.grid.cellsize[XX] * (cellxyz[XX] + 1): + if self.debug and debug: with gil: - print("\n#-> Bead Y={:.3f}, Next cell Y={:.3f}, cutoff={:.3f} -> +Y shift is ignored".format(self.coords_bbox[current_beadid, YY], self.grid.cellsize[YY] * (cellxyz[YY] + 1), self.cutoff)) + print( + "FastNS: -> Bead X={:.3f}, Next cell X={:.3f}, cutoff={:.3f} -> +X shift ignored".format( + self.coords_bbox[current_beadid, XX], + self.grid.cellsize[XX] * (cellxyz[XX] + 1), + self.cutoff + )) continue + for yi in range(DIM): + if memory_error: + break - for zi in range(DIM): - if xi == 0: - if self.coords_bbox[current_beadid, ZZ] - self.cutoff > self.grid.cellsize[ZZ] * cellxyz[ZZ]: - if self.debug: + if not self.box.is_triclinic: + if yi == 0: + if self.coords_bbox[current_beadid, YY] - self.cutoff > self.grid.cellsize[YY] * cellxyz[YY]: + if self.debug and debug: with gil: - print("\n#-> Bead Z={:.3f}, Cell Z={:.3f}, cutoff={:.3f} -> -Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[ZZ] * cellxyz[ZZ], self.cutoff)) + print("FastNS: -> Bead Y={:.3f}, Cell Y={:.3f}, cutoff={:.3f} -> -Y shift is ignored".format( + self.coords_bbox[current_beadid, YY], + self.grid.cellsize[YY] * cellxyz[YY], + self.cutoff, + )) continue - if xi == 2: - if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.grid.cellsize[ZZ] * (cellxyz[ZZ] + 1): - if self.debug: + if yi == 2: + if self.coords_bbox[current_beadid, YY] + self.cutoff < self.grid.cellsize[YY] * (cellxyz[YY] + 1): + if self.debug and debug: with gil: - print("\n#-> Bead Z={:.3f}, Next cell Z={:.3f}, cutoff={:.3f} -> +Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[XX] * (cellxyz[ZZ] + 1), self.cutoff)) + print("FastNS: -> Bead Y={:.3f}, Next cell Y={:.3f}, cutoff={:.3f} -> +Y shift is ignored".format( + self.coords_bbox[current_beadid, YY], + self.grid.cellsize[YY] * (cellxyz[YY] +1), + self.cutoff, + )) continue + + for zi in range(DIM): + if not self.box.is_triclinic: + if zi == 0: + if self.coords_bbox[current_beadid, ZZ] - self.cutoff > self.grid.cellsize[ZZ] * cellxyz[ZZ]: + if self.coords_bbox[current_beadid, ZZ] - self.cutoff > 0: + if self.debug and debug: + with gil: + print("FastNS: -> Bead Z={:.3f}, Cell Z={:.3f}, cutoff={:.3f} -> -Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[ZZ] * cellxyz[ZZ], self.cutoff)) + continue + + if zi == 2: + if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.grid.cellsize[ZZ] * (cellxyz[ZZ] + 1): + if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.box.c_pbcbox.box[ZZ][ZZ]: + if self.debug and debug: + with gil: + print("FastNS: -> Bead Z={:.3f}, Next cell Z={:.3f}, cutoff={:.3f} -> +Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[XX] * (cellxyz[ZZ] + 1), self.cutoff)) + continue + # Calculate and/or reinitialize shifted coordinates shifted_coord[XX] = self.coords[current_beadid, XX] + (xi - 1) * self.grid.cellsize[XX] shifted_coord[YY] = self.coords[current_beadid, YY] + (yi - 1) * self.grid.cellsize[YY] shifted_coord[ZZ] = self.coords[current_beadid, ZZ] + (zi - 1) * self.grid.cellsize[ZZ] + probe[XX] = self.coords[current_beadid, XX] + (xi - 1) * self.cutoff + probe[YY] = self.coords[current_beadid, YY] + (yi - 1) * self.cutoff + probe[ZZ] = self.coords[current_beadid, ZZ] + (zi - 1) * self.cutoff + # Make sure the shifted coordinates is inside the brick-shaped box for m in range(DIM - 1, -1, -1): @@ -1109,105 +1156,116 @@ cdef class FastNS2(object): for d in range(m+1): shifted_coord[d] -= self.box.c_pbcbox.box[m][d] + while probe[m] < 0: + for d in range(m+1): + probe[d] += self.box.c_pbcbox.box[m][d] + + + while probe[m] >= self.box.c_pbcbox.box[m][m]: + for d in range(m+1): + probe[d] -= self.box.c_pbcbox.box[m][d] + # Get the cell index corresponding to the coord - cellindex = self.grid.coord2cellid(shifted_coord) + cellindex_adjacent = self.grid.coord2cellid(shifted_coord) + cellindex_probe = self.grid.coord2cellid(probe) + + if cellindex == cellindex_probe and xi != 1 and yi != 1 and zi != 1: + if self.debug and debug: + with gil: + print("FastNS: Grid shift [{}][{}][{}]: Cutoff is inside current cell -> This shift is ignored".format( + xi - 1, + yi -1, + zi -1 + )) + continue - if self.debug: - self.grid.cellid2cellxyz(cellindex, debug_cellxyz) + if self.debug and debug: + self.grid.cellid2cellxyz(cellindex_adjacent, debug_cellxyz) with gil: dist_shift = self.box.fast_distance(&self.coords[current_beadid, XX], shifted_coord) grid_shift = np.array([(xi - 1) * self.grid.cellsize[XX], (yi - 1) * self.grid.cellsize[YY], (zi - 1) * self.grid.cellsize[ZZ]]) - print("-> Checking cell#{} ({},{},{}) for neighbors (dshift={:.3f}, grid_shift=({:.3f},{:.3f},{:.3f}->{:.3f})".format( + print("FastNS: -> Checking cell#{} ({},{},{}) for neighbors (dshift={:.3f}, grid_shift=({:.3f},{:.3f},{:.3f}->{:.3f})".format( cellindex, debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], dist_shift, grid_shift[XX], grid_shift[YY], grid_shift[ZZ], np.sqrt(np.sum(grid_shift**2)) )) - if dist_shift > 2* self.cutoff: - print(" \_ This cell should be ignored!") - else: - print(" \_ This cell is needed") - for j in range(self.grid.nbeads[cellindex]): - bid = self.grid.beadids[cellindex, j] + + for j in range(self.grid.nbeads[cellindex_adjacent]): + bid = self.grid.beadids[cellindex_adjacent * self.grid.nbeads_per_cell + j] if checked[bid] != 0: continue d2 = self.box.fast_distance2(&self.coords_bbox[current_beadid, XX], &self.coords_bbox[bid, XX]) - if self.debug and False: - self.grid.cellid2cellxyz(cellindex, debug_cellxyz) - with gil: - print( - "Beads #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) and #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) are tested (d2={:.3f})".format( - current_beadid, - cellxyz[XX], cellxyz[YY], cellxyz[ZZ], - self.coords_bbox[current_beadid, XX], - self.coords_bbox[current_beadid, YY], - self.coords_bbox[current_beadid, ZZ], - bid, - debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], - self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], - self.coords_bbox[bid, ZZ], - d2)) + # if self.debug: + # self.grid.cellid2cellxyz(cellindex, debug_cellxyz) + # with gil: + # print( + # "Beads #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) and #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) are tested (d2={:.3f})".format( + # current_beadid, + # cellxyz[XX], cellxyz[YY], cellxyz[ZZ], + # self.coords_bbox[current_beadid, XX], + # self.coords_bbox[current_beadid, YY], + # self.coords_bbox[current_beadid, ZZ], + # bid, + # debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], + # self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], + # self.coords_bbox[bid, ZZ], + # d2)) if d2 < cutoff2: if d2 < EPSILON: continue - if self.debug: + if self.debug and debug: self.grid.cellid2cellxyz(cellindex, debug_cellxyz) with gil: self.box.fast_pbc_dx(&self.coords[current_beadid, XX], &self.coords[bid, XX], dx) dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) - - print(" \_ Neighbor found: bead#{} (cell[{},{},{}]) -> dx={} -> d={:.3f}".format(bid, debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], dx_py, np.sqrt(d2))) - results.add_neighbors(current_beadid, bid, d2) + print("FastNS: \_ Neighbor found: bead#{} (cell[{},{},{}]) -> dx={} -> d={:.3f}".format(bid, debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], dx_py, np.sqrt(d2))) + if results.add_neighbors(current_beadid, bid, d2) == 0: + memory_error = True + break npairs += 1 + checked[current_beadid] = 1 - if first_ns: - guessed_size = ((npairs + 1) * 0.75 * size_search) - with gil: - if self.debug: - print("Neighbors of first beads= {} -> Approximated total of pairs={}".format(npairs, guessed_size)) - results.resize(guessed_size) + if memory_error: + raise MemoryError("Could not allocate memory to store NS results") - first_ns = False - checked[current_beadid] = 1 if self.debug: print("Total number of pairs={}".format(npairs)) - ref_bead = 13937 - beads = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, 47451]) - 1 - for bid in beads: - self.box.fast_pbc_dx(&self.coords[ref_bead, XX], &self.coords[bid, XX], dx) - dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) - self.box.fast_pbc_dx(&self.coords_bbox[ref_bead, XX], &self.coords_bbox[bid, XX], dx) - rect_dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) - self.grid.cellid2cellxyz(self.grid.coord2cellid(&self.coords_bbox[bid, XX]), cellxyz) - print("Bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]: dx=[{:.3f},{:.3f},{:.3f}] -> dist: {:.3f} ({}) - rect_dist: {:.3f} ({})".format( - bid, - self.coords[bid, XX], self.coords[bid, YY], self.coords[bid,ZZ], - self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], self.coords_bbox[bid,ZZ], - cellxyz[XX], cellxyz[YY], cellxyz[ZZ], - dx[XX], dx[YY], dx[ZZ], - np.sqrt(np.sum(dx_py**2)), - self.box.fast_distance(&self.coords[ref_bead, XX], &self.coords[bid, XX]) <= self.cutoff, - np.sqrt(np.sum(rect_dx_py**2)), - np.sqrt(np.sum(rect_dx_py**2)) <= self.cutoff - )) + # ref_bead = 13937 + # beads = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, 47451]) - 1 + # for bid in beads: + # self.box.fast_pbc_dx(&self.coords[ref_bead, XX], &self.coords[bid, XX], dx) + # dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) + # self.box.fast_pbc_dx(&self.coords_bbox[ref_bead, XX], &self.coords_bbox[bid, XX], dx) + # rect_dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) + # self.grid.cellid2cellxyz(self.grid.coord2cellid(&self.coords_bbox[bid, XX]), cellxyz) + # print("Bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]: dx=[{:.3f},{:.3f},{:.3f}] -> dist: {:.3f} ({})".format( + # bid, + # self.coords[bid, XX], self.coords[bid, YY], self.coords[bid,ZZ], + # self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], self.coords_bbox[bid,ZZ], + # cellxyz[XX], cellxyz[YY], cellxyz[ZZ], + # dx[XX], dx[YY], dx[ZZ], + # np.sqrt(np.sum(dx_py**2)), + # self.box.fast_distance(&self.coords[ref_bead, XX], &self.coords[bid, XX]) <= self.cutoff, + # )) return results # Python interface -cdef class FastNS(object): +cdef class FastNSOld(object): cdef PBCBox box cdef readonly real[:, ::1] coords cdef real[:, ::1] coords_bbox @@ -1376,108 +1434,10 @@ cdef class FastNS(object): if holder == NULL: raise MemoryError("Could not allocate memory to run NS core") - results = NSResults(self.box, self.cutoff) + results = NSResultsOld(self.box, self.cutoff) results.populate(holder, self.coords, search_coords_view) # Free memory allocated to holder free_neighborhood_holder(holder) return results - - def search2(self, search_ids): - cdef ns_int[:] search_ids_view, neighbors_view - cdef ns_int nid, i, j, size_search - cdef ns_neighborhood_holder *holder - cdef ns_neighborhood *neighborhood - - cdef ns_int current_beadid - cdef rvec current_coords - - #Temp stuff - cdef ns_int size = self.coords_bbox.shape[0] - cdef ns_int d, m - cdef ns_int xi, yi, zi, bid, i_bead - cdef real d2 - cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords - - cdef bint already_checked[27] - cdef bint skip - cdef ns_int nchecked = 0, icheck - cdef ns_int cell_index - cdef ns_int cell_offset - - cdef ns_int[:] neighbor_ids - cdef ns_int[:] nneighbors - cdef ns_int nid_offset - - cdef real cutoff2 = self.cutoff * self.cutoff - - - if not self.prepared: - self.prepare() - - nid_offset = self.grid.beadpercell * 27 - neighbor_ids = np.empty(size * nid_offset, dtype=np.int) - nneighbors = np.zeros(size, dtype=np.int) - - - search_ids_view = search_ids - size_search = search_ids_view.shape[0] - - with nogil: - - for current_beadid in range(size): - - for zi in range(3): - for yi in range(3): - for xi in range(3): - # Calculate and/or reinitialize shifted coordinates - shifted_coord[XX] = self.coords_bbox[current_beadid, XX] + (xi - 1) * self.grid.cellsize[XX] - shifted_coord[YY] = self.coords_bbox[current_beadid, YY] + (yi - 1) * self.grid.cellsize[YY] - shifted_coord[ZZ] = self.coords_bbox[current_beadid, ZZ] + (zi - 1) * self.grid.cellsize[ZZ] - - # Make sure the shifted coordinates is inside the brick-shaped box - for m in range(DIM - 1, -1, -1): - - while shifted_coord[m] < 0: - for d in range(m+1): - shifted_coord[d] += self.box.c_pbcbox.box[m][d] - - - while shifted_coord[m] >= self.box.c_pbcbox.box[m][m]: - for d in range(m+1): - shifted_coord[d] -= self.box.c_pbcbox.box[m][d] - - # Get the cell index corresponding to the coord - cell_index = (shifted_coord[ZZ] / self.grid.cellsize[ZZ]) * (self.grid.ncells[XX] * self.grid.ncells[YY]) +\ - (shifted_coord[YY] / self.grid.cellsize[YY]) * self.grid.ncells[XX] + \ - (shifted_coord[XX] / self.grid.cellsize[XX]) - - # Search for neighbors inside this cell - for i_bead in range(self.grid.nbeads[cell_index]): - cell_offset = cell_index * self.grid.beadpercell + i_bead - - bid = self.grid.beadids[cell_offset] - - if bid <= current_beadid: - continue - - self.box.fast_pbc_dx(&self.coords_bbox[current_beadid, XX], &self.coords_bbox[bid, XX], dx) - - d2 = rvec_norm2(dx) - - if d2 < cutoff2: - neighbor_ids[current_beadid * nid_offset + nneighbors[current_beadid]] = bid - nneighbors[current_beadid] += 1 - - neighbor_ids[bid * nid_offset + nneighbors[bid]] = current_beadid - nneighbors[bid] += 1 - - - - - results = [] - #for bid in search_ids: - # results.append(np.asarray(neighbor_ids[bid * nid_offset : bid * nid_offset + nneighbors[bid]])) - - return results diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py index 359d32544b7..5ce7d9f4e04 100644 --- a/testsuite/MDAnalysisTests/lib/test_gridsearch.py +++ b/testsuite/MDAnalysisTests/lib/test_gridsearch.py @@ -27,7 +27,7 @@ import numpy as np import MDAnalysis as mda -from MDAnalysis.lib import grid +from MDAnalysis.lib import nsgrid from MDAnalysis.lib.pkdtree import PeriodicKDTree from MDAnalysisTests.datafiles import GRO @@ -39,102 +39,85 @@ def universe(): return u -@pytest.fixture -def grid_results(): - u = mda.Universe(GRO) - cutoff = 2 - ref_pos = u.atoms.positions[13937] - return run_grid_search(u, ref_pos, cutoff) - - -def run_grid_search(u, ref_pos, cutoff): +def run_grid_search(u, ids, cutoff=3): coords = u.atoms.positions # Run grid search - searcher = grid.FastNS(u) - searcher.set_cutoff(cutoff) - searcher.set_coords(coords) - searcher.prepare() - - return searcher.search(ref_pos) - - -def run_search(universe, ref_id): - cutoff = 3 - coords = universe.atoms.positions - ref_pos = coords[ref_id] - - # Run pkdtree search - pkdt = PeriodicKDTree(universe.atoms.dimensions, bucket_size=10) - pkdt.set_coords(coords) - pkdt.search(ref_pos, cutoff) - - results_pkdtree = pkdt.get_indices() - results_pkdtree.remove(ref_id) - results_pkdtree = np.array(results_pkdtree) - results_pkdtree.sort() - - # Run grid search - results_grid = run_grid_search(universe, ref_pos, cutoff) - results_grid = results_grid.get_indices()[0] - results_grid.sort() + searcher = grid.FastNS(u, cutoff, coords, debug=True) - return results_pkdtree, results_grid + return searcher.search(ids, debug=False) def test_gridsearch(universe): """Check that pkdtree and grid search return the same results (No PBC needed)""" ref_id = 0 - results_pkdtree, results_grid = run_search(universe, ref_id) - assert_equal(results_pkdtree, results_grid) - - -def test_gridsearch_PBC(universe): - """Check that pkdtree and grid search return the same results (PBC needed)""" - - ref_id = 13937 - results_pkdtree, results_grid = run_search(universe, ref_id) - assert_equal(results_pkdtree, results_grid) - - -def test_gridsearch_arraycoord(universe): - """Check the NS routine accepts a single bead coordinate as well as array of coordinates""" - cutoff = 2 - ref_pos = universe.atoms.positions[:5] - - results = [ - np.array([2, 1, 4, 3]), - np.array([2, 0, 3]), - np.array([0, 1, 3]), - np.array([ 2, 0, 1, 38341]), - np.array([ 6, 0, 5, 17]) - ] - - results_grid = run_grid_search(universe, ref_pos, cutoff).get_indices() + cutoff = 3 + results = np.array([2, 3, 4, 5, 6, 7, 8, 9, 18, 19, 1211, 10862, 10865, 17582, 17585, 38342, + 38345]) - 1 # Atomid are from gmx select so there start from 1 and not 0. hence -1! - assert_equal(results_grid, results) + results_grid = run_grid_search(universe, ref_id, cutoff).get_indices()[0] + assert_equal(results, results_grid) -def test_gridsearch_search_coordinates(grid_results): - """Check the NS routine can return coordinates instead of ids""" - results = np.array( - [ - [40.32, 34.25, 55.9], - [0.61, 76.33, -0.56], - [0.48999998, 75.9, 0.19999999], - [-0.11, 76.19, 0.77] - ]) +def test_gridsearch_PBC(universe): + """Check that pkdtree and grid search return the same results (No PBC needed)""" - assert_allclose(grid_results.get_coordinates()[0], results) + ref_id = 13937 + results = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, + 47451]) - 1 # Atomid are from gmx select so there start from 1 and not 0. hence -1! + results_grid = run_grid_search(universe, ref_id).get_indices()[0] -def test_gridsearch_search_distances(grid_results): - """Check the NS routine can return PBC distances from neighbors""" - results = np.array([0.096, 0.096, 0.015, 0.179]) * 10 # These distances were obtained using gmx distance - results.sort() + assert_equal(results, results_grid) - rounded_results = np.round(grid_results.get_distances()[0], 2) - assert_allclose(sorted(rounded_results), results) \ No newline at end of file +# def test_gridsearch_PBC(universe): +# """Check that pkdtree and grid search return the same results (PBC needed)""" +# +# ref_id = 13937 +# results_pkdtree, results_grid = run_search(universe, ref_id) +# assert_equal(results_pkdtree, results_grid) +# +# +# def test_gridsearch_arraycoord(universe): +# """Check the NS routine accepts a single bead coordinate as well as array of coordinates""" +# cutoff = 2 +# ref_pos = universe.atoms.positions[:5] +# +# results = [ +# np.array([2, 1, 4, 3]), +# np.array([2, 0, 3]), +# np.array([0, 1, 3]), +# np.array([ 2, 0, 1, 38341]), +# np.array([ 6, 0, 5, 17]) +# ] +# +# results_grid = run_grid_search(universe, ref_pos, cutoff).get_indices() +# +# assert_equal(results_grid, results) +# +# +# def test_gridsearch_search_coordinates(grid_results): +# """Check the NS routine can return coordinates instead of ids""" +# +# results = np.array( +# [ +# [40.32, 34.25, 55.9], +# [0.61, 76.33, -0.56], +# [0.48999998, 75.9, 0.19999999], +# [-0.11, 76.19, 0.77] +# ]) +# +# assert_allclose(grid_results.get_coordinates()[0], results) +# +# +# def test_gridsearch_search_distances(grid_results): +# """Check the NS routine can return PBC distances from neighbors""" +# results = np.array([0.096, 0.096, 0.015, 0.179]) * 10 # These distances were obtained using gmx distance +# results.sort() +# +# rounded_results = np.round(grid_results.get_distances()[0], 2) +# +# assert_allclose(sorted(rounded_results), results) \ No newline at end of file From da8db15c7e21689151fc13d4deb5176bbd14210c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Wed, 18 Jul 2018 10:10:29 +0200 Subject: [PATCH 21/30] Module MDAnalysis/lib/c_gridsearch.pyx renamed to nsgrid.pyx --- package/MDAnalysis/lib/{c_gridsearch.pyx => nsgrid.pyx} | 7 +++---- .../lib/{test_gridsearch.py => test_nsgrid.py} | 8 +++----- 2 files changed, 6 insertions(+), 9 deletions(-) rename package/MDAnalysis/lib/{c_gridsearch.pyx => nsgrid.pyx} (97%) rename testsuite/MDAnalysisTests/lib/{test_gridsearch.py => test_nsgrid.py} (96%) diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/nsgrid.pyx similarity index 97% rename from package/MDAnalysis/lib/c_gridsearch.pyx rename to package/MDAnalysis/lib/nsgrid.pyx index a1f519eac37..134356bd572 100644 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -23,9 +23,9 @@ # # -#cython: cdivision=True -#cython: boundscheck=False -#cython: initializedcheck=False +# cython: cdivision=True +# cython: boundscheck=False +# cython: initializedcheck=False """ Neighbor search library --- :mod:`MDAnalysis.lib.grid` @@ -93,7 +93,6 @@ cdef void rvec_clear(rvec a) nogil: a[YY]=0.0 a[ZZ]=0.0 - cdef struct cPBCBox_t: matrix box rvec fbox_diag diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_nsgrid.py similarity index 96% rename from testsuite/MDAnalysisTests/lib/test_gridsearch.py rename to testsuite/MDAnalysisTests/lib/test_nsgrid.py index 5ce7d9f4e04..b34bfe22336 100644 --- a/testsuite/MDAnalysisTests/lib/test_gridsearch.py +++ b/testsuite/MDAnalysisTests/lib/test_nsgrid.py @@ -23,14 +23,12 @@ from __future__ import print_function, absolute_import import pytest -from numpy.testing import assert_equal, assert_allclose +from numpy.testing import assert_equal import numpy as np import MDAnalysis as mda -from MDAnalysis.lib import nsgrid -from MDAnalysis.lib.pkdtree import PeriodicKDTree - from MDAnalysisTests.datafiles import GRO +from MDAnalysis.lib import nsgrid @pytest.fixture @@ -43,7 +41,7 @@ def run_grid_search(u, ids, cutoff=3): coords = u.atoms.positions # Run grid search - searcher = grid.FastNS(u, cutoff, coords, debug=True) + searcher = nsgrid.FastNS(u, cutoff, coords, debug=True) return searcher.search(ids, debug=False) From 66e2151949712521173619484380ed56b3b9bacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Wed, 18 Jul 2018 16:44:11 +0200 Subject: [PATCH 22/30] Tests written for FastNS --- package/MDAnalysis/lib/__init__.py | 2 +- package/MDAnalysis/lib/nsgrid.pyx | 769 ++++--------------- testsuite/MDAnalysisTests/lib/test_nsgrid.py | 245 ++++-- 3 files changed, 326 insertions(+), 690 deletions(-) diff --git a/package/MDAnalysis/lib/__init__.py b/package/MDAnalysis/lib/__init__.py index a740bd8d5bf..d5a583f9aea 100644 --- a/package/MDAnalysis/lib/__init__.py +++ b/package/MDAnalysis/lib/__init__.py @@ -39,4 +39,4 @@ from . import NeighborSearch from . import formats from . import pkdtree -from . import grid \ No newline at end of file +from . import nsgrid \ No newline at end of file diff --git a/package/MDAnalysis/lib/nsgrid.pyx b/package/MDAnalysis/lib/nsgrid.pyx index 134356bd572..e4de849ffff 100644 --- a/package/MDAnalysis/lib/nsgrid.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -43,19 +43,14 @@ DEF ZZ = 2 DEF EPSILON = 1e-5 -DEF NEIGHBORHOOD_ALLOCATION_INCREMENT = 50 - DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 -from libc.stdlib cimport malloc, realloc, free +# Used to handle memory allocation from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free - from libc.math cimport sqrt - import numpy as np cimport numpy as np -cimport cython ctypedef np.int_t ns_int ctypedef np.float32_t real @@ -64,27 +59,8 @@ ctypedef ns_int ivec[DIM] ctypedef ns_int ipair[2] ctypedef real matrix[DIM][DIM] -cdef struct ns_grid: - ns_int beadpercell - ns_int size - ns_int[DIM] ncells - real[DIM] cellsize - ns_int *nbeads # size - ns_int *beadids # size * beadpercell - ns_int *cellids # size - -cdef struct ns_neighborhood: - real cutoff - ns_int allocated_size - ns_int size - ns_int *beadids - -cdef struct ns_neighborhood_holder: - ns_int size - ns_neighborhood **neighborhoods - -# Useful stuff +# Useful Functions cdef real rvec_norm2(const rvec a) nogil: return a[XX]*a[XX]+a[YY]*a[YY]+a[ZZ]*a[ZZ] @@ -93,6 +69,11 @@ cdef void rvec_clear(rvec a) nogil: a[YY]=0.0 a[ZZ]=0.0 +######################################################################################################################## +# +# Utility class to handle PBC +# +######################################################################################################################## cdef struct cPBCBox_t: matrix box rvec fbox_diag @@ -186,15 +167,10 @@ cdef class PBCBox(object): # Choose the vector within the brick around 0,0,0 that # will become the shortest due to shift try. - - if d == DIM: - trial[d] = 0 - pos[d] = 0 + if trial[d] < 0: + pos[d] = min(self.c_pbcbox.hbox_diag[d], -trial[d]) else: - if trial[d] < 0: - pos[d] = min(self.c_pbcbox.hbox_diag[d], -trial[d]) - else: - pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) + pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) d2old += pos[d]**2 d2new += (pos[d] + trial[d])**2 @@ -270,14 +246,34 @@ cdef class PBCBox(object): for j in range (i, -1, -1): dx[j] += self.c_pbcbox.box[i][j] + def dx(self, real[:] a, real[:] b): + cdef rvec dx + if a.shape[0] != DIM or b.shape[0] != DIM: + raise ValueError("Not 3 D coordinates") + + self.fast_pbc_dx(&a[XX], &b[XX], dx) + + return np.array([dx[XX], dx[YY], dx[ZZ]], dtype=np.float32) + + cdef real fast_distance2(self, rvec a, rvec b) nogil: cdef rvec dx self.fast_pbc_dx(a, b, dx) return rvec_norm2(dx) + def distance2(self, real[:] a, real[:] b): + if a.shape[0] != DIM or b.shape[0] != DIM: + raise ValueError("Not 3 D coordinates") + return self.fast_distance2(&a[XX], &b[XX]) + cdef real fast_distance(self, rvec a, rvec b) nogil: return sqrt(self.fast_distance2(a,b)) + def distance(self, real[:] a, real[:] b): + if a.shape[0] != DIM or b.shape[0] != DIM: + raise ValueError("Not 3 D coordinates") + return self.fast_distance(&a[XX], &b[XX]) + cdef real[:, ::1]fast_put_atoms_in_bbox(self, real[:,::1] coords) nogil: cdef ns_int i, m, d, natoms, wd = 0 cdef real[:,::1] bbox_coords @@ -305,341 +301,20 @@ cdef class PBCBox(object): return bbox_coords def put_atoms_in_bbox(self, real[:,::1] coords): - if coords.shape[0] == 0: - return np.zeros((0, DIM), dtype=np.float32) return np.asarray(self.fast_put_atoms_in_bbox(coords)) + ######################################################################################################################## # # Neighbor Search Stuff # ######################################################################################################################## - -cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: - cdef ns_neighborhood_holder *holder - - holder = malloc(sizeof(ns_neighborhood_holder)) - holder.size = 0 - holder.neighborhoods = NULL - - return holder - -cdef void free_neighborhood_holder(ns_neighborhood_holder *holder) nogil: - cdef ns_int i - - if holder == NULL: - return - - for i in range(holder.size): - if holder.neighborhoods[i].beadids != NULL: - free(holder.neighborhoods[i].beadids) - free(holder.neighborhoods[i]) - - if holder.neighborhoods != NULL: - free(holder.neighborhoods) - free(holder) - -cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]neighborcoords, ns_grid *grid, PBCBox box, real cutoff2) nogil: - cdef ns_int d, m - cdef ns_int xi, yi, zi, bid, i_bead - cdef real d2 - cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords - - cdef bint already_checked[27] - cdef bint skip - cdef ns_int nchecked = 0, icheck - cdef ns_int cell_index - cdef ns_int cell_offset - - cdef ns_neighborhood *neighborhood = malloc(sizeof(ns_neighborhood)) - if neighborhood == NULL: - return NULL - - neighborhood.size = 0 - neighborhood.allocated_size = NEIGHBORHOOD_ALLOCATION_INCREMENT - neighborhood.beadids = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(ns_int)) - - if neighborhood.beadids == NULL: - free(neighborhood) - return NULL - - for zi in range(3): - for yi in range(3): - for xi in range(3): - # Calculate and/or reinitialize shifted coordinates - shifted_coord[XX] = current_coords[XX] + (xi - 1) * grid.cellsize[XX] - shifted_coord[YY] = current_coords[YY] + (yi - 1) * grid.cellsize[YY] - shifted_coord[ZZ] = current_coords[ZZ] + (zi - 1) * grid.cellsize[ZZ] - - # Make sure the shifted coordinates is inside the brick-shaped box - for m in range(DIM - 1, -1, -1): - - while shifted_coord[m] < 0: - for d in range(m+1): - shifted_coord[d] += box.c_pbcbox.box[m][d] - - - while shifted_coord[m] >= box.c_pbcbox.box[m][m]: - for d in range(m+1): - shifted_coord[d] -= box.c_pbcbox.box[m][d] - - # Get the cell index corresponding to the coord - cell_index = (shifted_coord[ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ - (shifted_coord[YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ - (shifted_coord[XX] / grid.cellsize[XX]) - - # Just a safeguard - #if cell_index >= grid.size: - # continue - - # Check the cell index was not already selected - #skip = False - #for icheck in range(nchecked): - # if already_checked[icheck] == cell_index: - # skip = True - # break - #if skip: - # continue - - # Search for neighbors inside this cell - for i_bead in range(grid.nbeads[cell_index]): - cell_offset = cell_index * grid.beadpercell + i_bead - - bid = grid.beadids[cell_offset] - - box.fast_pbc_dx(current_coords, &neighborcoords[bid, XX], dx) - - d2 = rvec_norm2(dx) - - if d2 < cutoff2: - if d2 < EPSILON: # Don't add the current bead as its own neighbor! - continue - - # Update neighbor lists - neighborhood.beadids[neighborhood.size] = bid - neighborhood.size += 1 - - if neighborhood.size >= neighborhood.allocated_size: - neighborhood.allocated_size += NEIGHBORHOOD_ALLOCATION_INCREMENT - neighborhood.beadids = realloc( neighborhood.beadids, neighborhood.allocated_size * sizeof(ns_int)) - - if neighborhood.beadids == NULL: - free(neighborhood) - return NULL - - # Register the cell as checked - #already_checked[nchecked] = cell_index - #nchecked += 1 - - return neighborhood - - -cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, - real[:, ::1] neighborcoords, - ns_grid *grid, - PBCBox box, - real cutoff) nogil: - cdef ns_int coordid, i, j - cdef ns_int ncoords = refcoords.shape[0] - cdef ns_int ncoords_neighbors = neighborcoords.shape[0] - cdef real cutoff2 = cutoff * cutoff - cdef ns_neighborhood_holder *holder - - cdef ns_int *neighbor_buf - cdef ns_int buf_size, ibuf - - holder = create_neighborhood_holder() - if holder == NULL: - return NULL - - holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) - if holder.neighborhoods == NULL: - free_neighborhood_holder(holder) - return NULL - - # Here starts the real core and the iteration over coordinates - for coordid in range(ncoords): - holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], - neighborcoords, - grid, - box, - cutoff2) - if holder.neighborhoods[coordid] == NULL: - free_neighborhood_holder(holder) - return NULL - - holder.neighborhoods[coordid].cutoff = cutoff - holder.size += 1 - - return holder - -cdef class NSResultsOld(object): - """ - Class used to store results returned by `MDAnalysis.lib.grid.FastNS.search` - """ - cdef PBCBox box - cdef readonly real cutoff - cdef np.ndarray grid_coords - cdef np.ndarray ref_coords - cdef ns_int **nids - cdef ns_int *nsizes - cdef ns_int size - cdef list indices - cdef list coordinates - cdef list distances - - def __dealloc__(self): - if self.nids != NULL: - for i in range(self.size): - if self.nids[i] != NULL: - free(self.nids[i]) - free(self.nids) - - if self.nsizes != NULL: - free(self.nsizes) - - def __init__(self, PBCBox box, real cutoff): - self.box = box - self.cutoff = cutoff - - self.size = 0 - self.nids = NULL - self.nsizes = NULL - - self.grid_coords = None - self.ref_coords = None - - self.indices = None - self.coordinates = None - self.distances = None - - - cdef populate(self, ns_neighborhood_holder *holder, grid_coords, ref_coords): - cdef ns_int nid, i - cdef ns_neighborhood *neighborhood - - self.grid_coords = np.asarray(grid_coords) - self.ref_coords = np.asarray(ref_coords) - - # Allocate memory - self.nsizes = malloc(sizeof(ns_int) * holder.size) - if self.nsizes == NULL: - raise MemoryError("Could not allocate memory for NSResults") - - self.nids = malloc(sizeof(ns_int *) * holder.size) - if self.nids == NULL: - raise MemoryError("Could not allocate memory for NSResults") - - for nid in range(holder.size): - neighborhood = holder.neighborhoods[nid] - - self.nsizes[nid] = neighborhood.size - - self.nids[nid] = malloc(sizeof(ns_int *) * neighborhood.size) - if self.nids[nid] == NULL: - raise MemoryError("Could not allocate memory for NSResults") - - with nogil: - for nid in range(holder.size): - neighborhood = holder.neighborhoods[nid] - - for i in range(neighborhood.size): - self.nids[nid][i] = neighborhood.beadids[i] - - self.size = holder.size - - def get_indices(self): - """ - Return Neighbors indices. - - :return: list of indices - """ - cdef ns_int i, nid, size - - if self.indices is None: - indices = [] - - for nid in range(self.size): - size = self.nsizes[nid] - - tmp_incides = np.empty((size), dtype=np.int) - - for i in range(size): - tmp_incides[i] = self.nids[nid][i] - - indices.append(tmp_incides) - - self.indices = indices - - return self.indices - - - def get_coordinates(self): - """ - Return coordinates of neighbors. - - :return: list of coordinates - """ - cdef ns_int i, nid, size, beadid - - if self.coordinates is None: - coordinates = [] - - for nid in range(self.size): - size = self.nsizes[nid] - - tmp_values = np.empty((size, DIM), dtype=np.float32) - - for i in range(size): - beadid = self.nids[nid][i] - tmp_values[i] = self.grid_coords[beadid] - - coordinates.append(tmp_values) - - self.coordinates = coordinates - - return self.coordinates - - - def get_distances(self): - """ - Return coordinates of neighbors. - - :return: list of distances - """ - cdef ns_int i, nid, size, j, beadid - cdef rvec ref, other, dx - cdef real dist - cdef real[:, ::1] ref_coords = self.ref_coords - cdef real[:, ::1] grid_coords = self.grid_coords - - if self.distances is None: - distances = [] - - for nid in range(self.size): - size = self.nsizes[nid] - - tmp_values = np.empty((size), dtype=np.float32) - ref = &ref_coords[nid, 0] - - for i in range(size): - beadid = self.nids[nid][i] - other = &grid_coords[beadid, 0] - - tmp_values[i] = self.box.fast_distance(ref, other) - - distances.append(tmp_values) - - self.distances = distances - - return self.distances - cdef class NSResults(object): cdef readonly real cutoff cdef ns_int npairs cdef bint debug - cdef real[:, ::1] grid_coords # shape: size, DIM + cdef real[:, ::1] coords # shape: size, DIM cdef ns_int[:] search_ids cdef ns_int allocation_size @@ -651,12 +326,13 @@ cdef class NSResults(object): cdef list distances_buffer cdef np.ndarray pairs_buffer cdef np.ndarray pair_distances_buffer + cdef np.ndarray pair_coordinates_buffer - def __init__(self, real cutoff, real[:, ::1]grid_coords,ns_int[:] search_ids, debug=False): + def __init__(self, real cutoff, real[:, ::1]coords, ns_int[:] search_ids, debug=False): self.debug = debug self.cutoff = cutoff - self.grid_coords = grid_coords + self.coords = coords self.search_ids = search_ids # Preallocate memory @@ -674,7 +350,7 @@ cdef class NSResults(object): self.coordinates_buffer = None self.distances_buffer = None self.pairs_buffer = None - self.pair_distances_buffer = None + self.pair_coordinates_buffer = None def __dealloc__(self): PyMem_Free(self.pairs) @@ -745,12 +421,25 @@ cdef class NSResults(object): def get_pair_distances(self): cdef ns_int i - if self.pair_distances_buffer is None: - self.pair_distances_buffer = np.empty(self.npairs, dtype=np.float32) + if self.pair_coordinates_buffer is None: + self.pair_coordinates_buffer = np.empty(self.npairs, dtype=np.float32) + for i in range(self.npairs): + self.pair_coordinates_buffer[i] = self.pair_distances2[i] + self.pair_coordinates_buffer = np.sqrt(self.pair_coordinates_buffer) + return self.pair_coordinates_buffer + + def get_pair_coordinates(self): + cdef ns_int i, j, bead_i, bead_j + if self.pair_coordinates_buffer is None: + self.pair_coordinates_buffer = np.empty((self.npairs, 2, DIM), dtype=np.float32) for i in range(self.npairs): - self.pair_distances_buffer[i] = self.pair_distances2[i] - self.pair_distances_buffer = np.sqrt(self.pair_distances_buffer) - return self.pair_distances_buffer + bead_i = self.pairs[i][0] + bead_j = self.pairs[i][1] + + for j in range(DIM): + self.pair_coordinates_buffer[i, 0, j] = self.coords[bead_i, j] + self.pair_coordinates_buffer[i, 1, j] = self.coords[bead_j, j] + return self.pair_coordinates_buffer cdef create_buffers(self): cdef ns_int i, beadid_i, beadid_j @@ -767,8 +456,8 @@ cdef class NSResults(object): beadid_j = self.pairs[i][1] dist2 = self.pair_distances2[i] - coord_i = self.grid_coords[beadid_i] - coord_j = self.grid_coords[beadid_j] + coord_i = self.coords[beadid_i] + coord_j = self.coords[beadid_j] indices_buffer[beadid_i].append(beadid_j) indices_buffer[beadid_j].append(beadid_i) @@ -933,12 +622,10 @@ cdef class NSGrid(object): PyMem_Free(beadcounts) - -# Python interface cdef class FastNS(object): cdef bint debug cdef PBCBox box - cdef readonly real[:, ::1] coords + cdef real[:, ::1] coords cdef real[:, ::1] coords_bbox cdef readonly real cutoff cdef bint prepared @@ -954,9 +641,6 @@ cdef class FastNS(object): raise TypeError("FastNS class must be initialized with a valid MDAnalysis.Universe instance") box = triclinic_vectors(u.dimensions) - if box.shape != (3, 3): - raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") - self.box = PBCBox(box) @@ -967,18 +651,14 @@ cdef class FastNS(object): self.coords_bbox = self.box.fast_put_atoms_in_bbox(coords) - if not prepare: - return - - if self.cutoff < 0: + if cutoff < 0: raise ValueError("Cutoff must be positive!") - if self.cutoff * self.cutoff > self.box.c_pbcbox.max_cutoff2: + if cutoff * cutoff > self.box.c_pbcbox.max_cutoff2: raise ValueError("Cutoff greater than maximum cutoff ({:.3f}) given the PBC") self.cutoff = cutoff self.grid = NSGrid(self.coords_bbox.shape[0], cutoff, self.box, max_gridsize, debug=debug) self.prepared = False - if prepare: self.prepare() @@ -991,7 +671,7 @@ cdef class FastNS(object): self.prepared = True - def search(self, search_ids=None, bint debug=False): + def search(self, search_ids=None): cdef ns_int i, j, size_search cdef ns_int d, m cdef NSResults results @@ -1012,12 +692,12 @@ cdef class FastNS(object): cdef ns_int nchecked = 0 - - cdef real cutoff2 = self.cutoff * self.cutoff cdef ns_int[:] checked cdef ns_int npairs = 0 + #cdef bint debug=False + if not self.prepared: self.prepare() @@ -1034,12 +714,12 @@ cdef class FastNS(object): checked = np.zeros(size, dtype=np.int) - results = NSResults(self.cutoff, self.coords_bbox, search_ids, self.debug) + results = NSResults(self.cutoff, self.coords, search_ids, self.debug) cdef bint memory_error = False - if self.debug and debug: - print("FastNS: Debug flag is set to True for FastNS.search()") + # if self.debug and debug: + # print("FastNS: Debug flag is set to True for FastNS.search()") with nogil: for i in range(size_search): @@ -1050,13 +730,13 @@ cdef class FastNS(object): cellindex = self.grid.cellids[current_beadid] self.grid.cellid2cellxyz(cellindex, cellxyz) - if self.debug and debug: - with gil: - print("FastNS: Checking neighbors for bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]:" .format( - current_beadid, - self.coords[current_beadid, XX], self.coords[current_beadid, YY], self.coords[current_beadid, ZZ], - self.coords_bbox[current_beadid, XX], self.coords_bbox[current_beadid, YY], self.coords_bbox[current_beadid, ZZ], - cellxyz[XX], cellxyz[YY], cellxyz[ZZ])) + # if self.debug and debug: + # with gil: + # print("FastNS: Checking neighbors for bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]:" .format( + # current_beadid, + # self.coords[current_beadid, XX], self.coords[current_beadid, YY], self.coords[current_beadid, ZZ], + # self.coords_bbox[current_beadid, XX], self.coords_bbox[current_beadid, YY], self.coords_bbox[current_beadid, ZZ], + # cellxyz[XX], cellxyz[YY], cellxyz[ZZ])) for xi in range(DIM): @@ -1067,25 +747,25 @@ cdef class FastNS(object): # If box is not triclinic (ie rect), when can already check if the shift can be skipped (ie cutoff inside the cell) if xi == 0: if self.coords_bbox[current_beadid, XX] - self.cutoff > self.grid.cellsize[XX] * cellxyz[XX]: - if self.debug and debug: - with gil: - print("FastNS: -> Bead X={:.3f}, Cell X={:.3f}, cutoff={:.3f} -> -X shift ignored".format( - self.coords_bbox[current_beadid, XX], - self.grid.cellsize[XX] * cellxyz[XX], - self.cutoff - )) + # if self.debug and debug: + # with gil: + # print("FastNS: -> Bead X={:.3f}, Cell X={:.3f}, cutoff={:.3f} -> -X shift ignored".format( + # self.coords_bbox[current_beadid, XX], + # self.grid.cellsize[XX] * cellxyz[XX], + # self.cutoff + # )) continue if xi == 2: if self.coords_bbox[current_beadid, XX] + self.cutoff < self.grid.cellsize[XX] * (cellxyz[XX] + 1): - if self.debug and debug: - with gil: - print( - "FastNS: -> Bead X={:.3f}, Next cell X={:.3f}, cutoff={:.3f} -> +X shift ignored".format( - self.coords_bbox[current_beadid, XX], - self.grid.cellsize[XX] * (cellxyz[XX] + 1), - self.cutoff - )) + # if self.debug and debug: + # with gil: + # print( + # "FastNS: -> Bead X={:.3f}, Next cell X={:.3f}, cutoff={:.3f} -> +X shift ignored".format( + # self.coords_bbox[current_beadid, XX], + # self.grid.cellsize[XX] * (cellxyz[XX] + 1), + # self.cutoff + # )) continue for yi in range(DIM): @@ -1095,24 +775,24 @@ cdef class FastNS(object): if not self.box.is_triclinic: if yi == 0: if self.coords_bbox[current_beadid, YY] - self.cutoff > self.grid.cellsize[YY] * cellxyz[YY]: - if self.debug and debug: - with gil: - print("FastNS: -> Bead Y={:.3f}, Cell Y={:.3f}, cutoff={:.3f} -> -Y shift is ignored".format( - self.coords_bbox[current_beadid, YY], - self.grid.cellsize[YY] * cellxyz[YY], - self.cutoff, - )) + # if self.debug and debug: + # with gil: + # print("FastNS: -> Bead Y={:.3f}, Cell Y={:.3f}, cutoff={:.3f} -> -Y shift is ignored".format( + # self.coords_bbox[current_beadid, YY], + # self.grid.cellsize[YY] * cellxyz[YY], + # self.cutoff, + # )) continue if yi == 2: if self.coords_bbox[current_beadid, YY] + self.cutoff < self.grid.cellsize[YY] * (cellxyz[YY] + 1): - if self.debug and debug: - with gil: - print("FastNS: -> Bead Y={:.3f}, Next cell Y={:.3f}, cutoff={:.3f} -> +Y shift is ignored".format( - self.coords_bbox[current_beadid, YY], - self.grid.cellsize[YY] * (cellxyz[YY] +1), - self.cutoff, - )) + # if self.debug and debug: + # with gil: + # print("FastNS: -> Bead Y={:.3f}, Next cell Y={:.3f}, cutoff={:.3f} -> +Y shift is ignored".format( + # self.coords_bbox[current_beadid, YY], + # self.grid.cellsize[YY] * (cellxyz[YY] +1), + # self.cutoff, + # )) continue @@ -1121,17 +801,17 @@ cdef class FastNS(object): if zi == 0: if self.coords_bbox[current_beadid, ZZ] - self.cutoff > self.grid.cellsize[ZZ] * cellxyz[ZZ]: if self.coords_bbox[current_beadid, ZZ] - self.cutoff > 0: - if self.debug and debug: - with gil: - print("FastNS: -> Bead Z={:.3f}, Cell Z={:.3f}, cutoff={:.3f} -> -Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[ZZ] * cellxyz[ZZ], self.cutoff)) + # if self.debug and debug: + # with gil: + # print("FastNS: -> Bead Z={:.3f}, Cell Z={:.3f}, cutoff={:.3f} -> -Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[ZZ] * cellxyz[ZZ], self.cutoff)) continue if zi == 2: if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.grid.cellsize[ZZ] * (cellxyz[ZZ] + 1): if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.box.c_pbcbox.box[ZZ][ZZ]: - if self.debug and debug: - with gil: - print("FastNS: -> Bead Z={:.3f}, Next cell Z={:.3f}, cutoff={:.3f} -> +Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[XX] * (cellxyz[ZZ] + 1), self.cutoff)) + # if self.debug and debug: + # with gil: + # print("FastNS: -> Bead Z={:.3f}, Next cell Z={:.3f}, cutoff={:.3f} -> +Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[XX] * (cellxyz[ZZ] + 1), self.cutoff)) continue # Calculate and/or reinitialize shifted coordinates @@ -1169,29 +849,29 @@ cdef class FastNS(object): cellindex_probe = self.grid.coord2cellid(probe) if cellindex == cellindex_probe and xi != 1 and yi != 1 and zi != 1: - if self.debug and debug: - with gil: - print("FastNS: Grid shift [{}][{}][{}]: Cutoff is inside current cell -> This shift is ignored".format( - xi - 1, - yi -1, - zi -1 - )) + # if self.debug and debug: + # with gil: + # print("FastNS: Grid shift [{}][{}][{}]: Cutoff is inside current cell -> This shift is ignored".format( + # xi - 1, + # yi -1, + # zi -1 + # )) continue - if self.debug and debug: - self.grid.cellid2cellxyz(cellindex_adjacent, debug_cellxyz) - with gil: - dist_shift = self.box.fast_distance(&self.coords[current_beadid, XX], shifted_coord) - grid_shift = np.array([(xi - 1) * self.grid.cellsize[XX], - (yi - 1) * self.grid.cellsize[YY], - (zi - 1) * self.grid.cellsize[ZZ]]) - print("FastNS: -> Checking cell#{} ({},{},{}) for neighbors (dshift={:.3f}, grid_shift=({:.3f},{:.3f},{:.3f}->{:.3f})".format( - cellindex, - debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], - dist_shift, - grid_shift[XX], grid_shift[YY], grid_shift[ZZ], - np.sqrt(np.sum(grid_shift**2)) - )) + # if self.debug and debug: + # self.grid.cellid2cellxyz(cellindex_adjacent, debug_cellxyz) + # with gil: + # dist_shift = self.box.fast_distance(&self.coords[current_beadid, XX], shifted_coord) + # grid_shift = np.array([(xi - 1) * self.grid.cellsize[XX], + # (yi - 1) * self.grid.cellsize[YY], + # (zi - 1) * self.grid.cellsize[ZZ]]) + # print("FastNS: -> Checking cell#{} ({},{},{}) for neighbors (dshift={:.3f}, grid_shift=({:.3f},{:.3f},{:.3f}->{:.3f})".format( + # cellindex, + # debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], + # dist_shift, + # grid_shift[XX], grid_shift[YY], grid_shift[ZZ], + # np.sqrt(np.sum(grid_shift**2)) + # )) for j in range(self.grid.nbeads[cellindex_adjacent]): @@ -1223,12 +903,12 @@ cdef class FastNS(object): if d2 < EPSILON: continue - if self.debug and debug: - self.grid.cellid2cellxyz(cellindex, debug_cellxyz) - with gil: - self.box.fast_pbc_dx(&self.coords[current_beadid, XX], &self.coords[bid, XX], dx) - dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) - print("FastNS: \_ Neighbor found: bead#{} (cell[{},{},{}]) -> dx={} -> d={:.3f}".format(bid, debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], dx_py, np.sqrt(d2))) + # if self.debug and debug: + # self.grid.cellid2cellxyz(cellindex, debug_cellxyz) + # with gil: + # self.box.fast_pbc_dx(&self.coords[current_beadid, XX], &self.coords[bid, XX], dx) + # dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) + # print("FastNS: \_ Neighbor found: bead#{} (cell[{},{},{}]) -> dx={} -> d={:.3f}".format(bid, debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], dx_py, np.sqrt(d2))) if results.add_neighbors(current_beadid, bid, d2) == 0: memory_error = True break @@ -1260,183 +940,4 @@ cdef class FastNS(object): # self.box.fast_distance(&self.coords[ref_bead, XX], &self.coords[bid, XX]) <= self.cutoff, # )) - return results - - -# Python interface -cdef class FastNSOld(object): - cdef PBCBox box - cdef readonly real[:, ::1] coords - cdef real[:, ::1] coords_bbox - cdef readonly real cutoff - cdef bint prepared - cdef ns_grid grid - - def __init__(self, u): - import MDAnalysis as mda - from MDAnalysis.lib.mdamath import triclinic_vectors - - if not isinstance(u, mda.Universe): - raise TypeError("FastNS class must be initialized with a valid MDAnalysis.Universe instance") - box = triclinic_vectors(u.dimensions) - - if box.shape != (3, 3): - raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") - - self.box = PBCBox(box) - - self.coords = None - self.coords_bbox = None - - self.cutoff = -1 - - self.prepared = False - - self.grid.size = 0 - - - def __dealloc__(self): - cdef ns_int i - - # Deallocate NS grid - if self.grid.nbeads != NULL: - free(self.grid.nbeads) - - if self.grid.beadids != NULL: - free(self.grid.beadids) - - if self.grid.cellids != NULL: - free(self.grid.cellids) - - self.grid.size = 0 - - - def set_coords(self, real[:, ::1] coords): - self.coords = coords - - # Make sure atoms are inside the brick-shaped box - self.coords_bbox = self.box.fast_put_atoms_in_bbox(coords) - - self.prepared = False - - - def set_cutoff(self, real cutoff): - self.cutoff = cutoff - - self.prepared = False - - - def prepare(self, force=False): - cdef ns_int i, cellindex = -1 - cdef ns_int ncoords = self.coords.shape[0] - cdef rvec *coords = &self.coords_bbox[0, 0] - - cdef ns_int *beadcounts = NULL - - if self.prepared and not force: - print("NS already prepared, nothing to do!") - - if self.coords is None: - raise ValueError("Coordinates must be set before NS preparation!") - - if self.cutoff < 0: - raise ValueError("Cutoff must be set before NS preparation!") - - with nogil: - - # Initializing grid - for i in range(DIM): - self.grid.ncells[i] = (self.box.c_pbcbox.box[i][i] / self.cutoff) - if self.grid.ncells[i] == 0: - self.grid.ncells[i] = 1 - self.grid.cellsize[i] = self.box.c_pbcbox.box[i][i] / self.grid.ncells[i] - self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] - - # Allocate memory for temporary buffers - self.grid.cellids = malloc(sizeof(ns_int) * ncoords) - if self.grid.cellids == NULL: - with gil: - raise MemoryError("Could not allocate memory for cell ids") - - beadcounts = malloc(sizeof(ns_int) * self.grid.size) - if beadcounts == NULL: - with gil: - raise MemoryError("Could not allocate memory for bead count buffer") - - self.grid.nbeads = malloc(sizeof(ns_int) * self.grid.size) - if self.grid.nbeads == NULL: - with gil: - raise MemoryError("Could not allocate memory for NS grid") - - # Initialize buffers - for i in range(self.grid.size): - self.grid.nbeads[i] = 0 - beadcounts[i] = 0 - - - # First loop: find cellindex for each bead - for i in range(ncoords): - cellindex = (coords[i][ZZ] / self.grid.cellsize[ZZ]) * (self.grid.ncells[XX] * self.grid.ncells[YY]) +\ - (coords[i][YY] / self.grid.cellsize[YY]) * self.grid.ncells[XX] + \ - (coords[i][XX] / self.grid.cellsize[XX]) - - self.grid.nbeads[cellindex] += 1 - self.grid.cellids[i] = cellindex - - if self.grid.nbeads[cellindex] > self.grid.beadpercell: - self.grid.beadpercell = self.grid.nbeads[cellindex] - - # Allocate memory and set offsets - self.grid.beadids = malloc(sizeof(ns_int) * self.grid.size * self.grid.beadpercell) - if self.grid.beadids == NULL: - with gil: - raise MemoryError("Could not allocate memory for NS grid bead ids") - - # Second loop: fill grid - for i in range(ncoords): - cellindex = self.grid.cellids[i] - self.grid.beadids[cellindex * self.grid.beadpercell + beadcounts[cellindex]] = i - beadcounts[cellindex] += 1 - - # Now we can free the allocation buffer - free(beadcounts) - - self.prepared = True - - - def search(self, search_coords): - cdef real[:, ::1] search_coords_bbox - cdef real[:, ::1] search_coords_view - cdef ns_int nid, i, j - cdef ns_neighborhood_holder *holder - cdef ns_neighborhood *neighborhood - - if not self.prepared: - self.prepare() - - # Check the shape of search_coords as a array of 3D coords if needed - shape = search_coords.shape - if len(shape) == 1: - if not shape[0] == 3: - raise ValueError("Coordinates must be 3D") - else: - search_coords_view = np.array([search_coords,], dtype=np.float32) - else: - search_coords_view = search_coords - - # Make sure atoms are inside the brick-shaped box - search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords_view) - - with nogil: - holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) - - if holder == NULL: - raise MemoryError("Could not allocate memory to run NS core") - - results = NSResultsOld(self.box, self.cutoff) - results.populate(holder, self.coords, search_coords_view) - - # Free memory allocated to holder - free_neighborhood_holder(holder) - - return results + return results \ No newline at end of file diff --git a/testsuite/MDAnalysisTests/lib/test_nsgrid.py b/testsuite/MDAnalysisTests/lib/test_nsgrid.py index b34bfe22336..33f7f1c67fe 100644 --- a/testsuite/MDAnalysisTests/lib/test_nsgrid.py +++ b/testsuite/MDAnalysisTests/lib/test_nsgrid.py @@ -23,11 +23,11 @@ from __future__ import print_function, absolute_import import pytest -from numpy.testing import assert_equal +from numpy.testing import assert_equal, assert_allclose import numpy as np import MDAnalysis as mda -from MDAnalysisTests.datafiles import GRO +from MDAnalysisTests.datafiles import GRO, Martini_membrane_gro from MDAnalysis.lib import nsgrid @@ -43,11 +43,95 @@ def run_grid_search(u, ids, cutoff=3): # Run grid search searcher = nsgrid.FastNS(u, cutoff, coords, debug=True) - return searcher.search(ids, debug=False) + return searcher.search(ids) -def test_gridsearch(universe): - """Check that pkdtree and grid search return the same results (No PBC needed)""" +def test_pbc_badbox(): + """Check that PBC box accepts only well-formated boxes""" + with pytest.raises(TypeError): + nsgrid.PBCBox([]) + + with pytest.raises(ValueError): + nsgrid.PBCBox(np.zeros((3))) # Bad shape + nsgrid.PBCBox(np.zeros((3, 3))) # Collapsed box + nsgrid.PBCBOX(np.array([[0, 0, 0], [0, 1, 0], [0, 0, 1]])) # 2D box + nsgrid.PBCBOX(np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]])) # Box provided as array of integers + nsgrid.PBCBOX(np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=float)) # Box provided as array of double + + +def test_pbc_distances(): + """Check that PBC box computes distances""" + box = np.identity(3, dtype=np.float32) + bad = np.array([0.1, 0.2], dtype=np.float32) + a = np.array([0.1, 0.1, 0.1], dtype=np.float32) + b = np.array([1.1, -0.1, 0.2], dtype=np.float32) + dx = np.array([0, -0.2, 0.1], dtype=np.float32) + pbcbox = nsgrid.PBCBox(box) + + with pytest.raises(ValueError): + pbcbox.distance(a, bad) + pbcbox.distance(bad, a) + + pbcbox.distance2(a, bad) + pbcbox.distance2(bad, a) + + pbcbox.dx(bad, a) + pbcbox.dx(a, bad) + + assert_equal(pbcbox.dx(a, b), dx) + assert pbcbox.distance(a, b) == np.sqrt(np.sum(dx*dx)) + assert pbcbox.distance2(a, b) == np.sum(dx*dx) + + +def test_pbc_put_in_bbox(): + "Check that PBC put beads in brick-shaped box" + box = np.identity(3, dtype=np.float32) + coords = np.array( + [ + [0.1, 0.1, 0.1], + [-0.1, 1.1, 0.9] + ], + dtype=np.float32 + ) + results = np.array( + [ + [0.1, 0.1, 0.1], + [0.9, 0.1, 0.9] + ], + dtype=np.float32 + ) + + pbcbox = nsgrid.PBCBox(box) + + assert_allclose(pbcbox.put_atoms_in_bbox(coords), results, atol=1e-5) + + +def test_nsgrid_badinit(): + with pytest.raises(TypeError): + nsgrid.FastNS(None, 1) + +def test_nsgrid_badcutoff(universe): + with pytest.raises(ValueError): + run_grid_search(universe, 0, -4) + run_grid_search(universe, 0, 100000) + +def test_ns_grid_noneighbor(universe): + """Check that grid search returns empty lists/arrays when there is no neighbors""" + ref_id = 0 + cutoff = 0.5 + + results_grid = run_grid_search(universe, ref_id, cutoff) + + assert len(results_grid.get_coordinates()[0]) == 0 + assert len(results_grid.get_distances()[0]) == 0 + assert len(results_grid.get_indices()[0]) == 0 + assert len(results_grid.get_pairs()) == 0 + assert len(results_grid.get_pair_distances()) == 0 + assert len(results_grid.get_pair_coordinates()) == 0 + + +def test_nsgrid_noPBC(universe): + """Check that grid search works when no PBC is needed""" ref_id = 0 cutoff = 3 @@ -59,8 +143,31 @@ def test_gridsearch(universe): assert_equal(results, results_grid) -def test_gridsearch_PBC(universe): - """Check that pkdtree and grid search return the same results (No PBC needed)""" +def test_nsgrid_PBC_rect(): + """Check that nsgrid works with rect boxes and PBC""" + ref_id = 191 + results = np.array([191, 672, 682, 683, 684, 995, 996, 2060, 2808, 3300, 3791, + 3792]) - 1 # Atomid are from gmx select so there start from 1 and not 0. hence -1! + + universe = mda.Universe(Martini_membrane_gro) + cutoff = 7 + + # FastNS is called differently to max coverage + searcher = nsgrid.FastNS(universe, cutoff, prepare=False) + + results_grid = searcher.search([ref_id,]).get_indices()[0] # pass the id as a list for test+coverage purpose + + searcher.prepare() # Does nothing, called here for coverage + results_grid2 = searcher.search().get_indices() # call without specifying any ids, should do NS for all beads + + assert_equal(results, results_grid) + assert_equal(len(universe.atoms), len(results_grid2)) + assert searcher.cutoff == 7 + assert_equal(results_grid, results_grid2[ref_id]) + + +def test_nsgrid_PBC(universe): + """Check that grid search works when PBC is needed""" ref_id = 13937 results = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, @@ -71,51 +178,79 @@ def test_gridsearch_PBC(universe): assert_equal(results, results_grid) -# def test_gridsearch_PBC(universe): -# """Check that pkdtree and grid search return the same results (PBC needed)""" -# -# ref_id = 13937 -# results_pkdtree, results_grid = run_search(universe, ref_id) -# assert_equal(results_pkdtree, results_grid) -# -# -# def test_gridsearch_arraycoord(universe): -# """Check the NS routine accepts a single bead coordinate as well as array of coordinates""" -# cutoff = 2 -# ref_pos = universe.atoms.positions[:5] -# -# results = [ -# np.array([2, 1, 4, 3]), -# np.array([2, 0, 3]), -# np.array([0, 1, 3]), -# np.array([ 2, 0, 1, 38341]), -# np.array([ 6, 0, 5, 17]) -# ] -# -# results_grid = run_grid_search(universe, ref_pos, cutoff).get_indices() -# -# assert_equal(results_grid, results) -# -# -# def test_gridsearch_search_coordinates(grid_results): -# """Check the NS routine can return coordinates instead of ids""" -# -# results = np.array( -# [ -# [40.32, 34.25, 55.9], -# [0.61, 76.33, -0.56], -# [0.48999998, 75.9, 0.19999999], -# [-0.11, 76.19, 0.77] -# ]) -# -# assert_allclose(grid_results.get_coordinates()[0], results) -# -# -# def test_gridsearch_search_distances(grid_results): -# """Check the NS routine can return PBC distances from neighbors""" -# results = np.array([0.096, 0.096, 0.015, 0.179]) * 10 # These distances were obtained using gmx distance -# results.sort() -# -# rounded_results = np.round(grid_results.get_distances()[0], 2) -# -# assert_allclose(sorted(rounded_results), results) \ No newline at end of file +def test_nsgrid_pairs(universe): + """Check that grid search returns the proper pairs""" + + ref_id = 13937 + neighbors = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, + 47451]) - 1 # Atomid are from gmx select so there start from 1 and not 0. hence -1! + results = [] + for nid in neighbors: + if nid < ref_id: + results.append([nid, ref_id]) + else: + results.append([ref_id, nid]) + results = np.array(results) + + results_grid = run_grid_search(universe, ref_id).get_pairs() + + assert_equal(np.sort(results, axis=0), np.sort(results_grid, axis=0)) + + +def test_nsgrid_pair_distances(universe): + """Check that grid search returns the proper pair distances""" + + ref_id = 13937 + results = np.array([0.270, 0.285, 0.096, 0.096, 0.015, 0.278, 0.268, 0.179, 0.259, 0.290, + 0.270]) * 10 # These distances where obtained by gmx distance so they are in nm + + results_grid = run_grid_search(universe, ref_id).get_pair_distances() + + assert_allclose(np.sort(results), np.sort(results_grid), atol=1e-2) + + +def test_nsgrid_pair_coordinates(universe): + """Check that grid search return the proper pair coordinates""" + + ref_id = 13937 + neighbors = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, + 47451]) - 1 # Atomid are from gmx select so there start from 1 and not 0. hence -1! + coords = universe.atoms.positions + + results = [] + for nid in neighbors: + if nid < ref_id: + results.append([coords[nid], coords[ref_id]]) + else: + results.append([coords[ref_id], coords[nid]]) + results = np.array(results) + + results_grid = run_grid_search(universe, ref_id).get_pair_coordinates() + + assert_allclose(np.sort(results, axis=0), np.sort(results_grid, axis=0), atol=1e-5) + + +def test_nsgrid_distances(universe): + """Check that grid search returns the proper distances""" + + ref_id = 13937 + results = np.array([0.270, 0.285, 0.096, 0.096, 0.015, 0.278, 0.268, 0.179, 0.259, 0.290, + 0.270]) * 10 # These distances where obtained by gmx distance so they are in nm + + results_grid = run_grid_search(universe, ref_id).get_distances()[0] + + assert_allclose(np.sort(results), np.sort(results_grid), atol=1e-2) + + +def test_nsgrid_coordinates(universe): + """Check that grid search return the proper coordinates""" + + ref_id = 13937 + neighbors = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, + 47451]) - 1 # Atomid are from gmx select so there start from 1 and not 0. hence -1! + + results = universe.atoms.positions[neighbors] + + results_grid = run_grid_search(universe, ref_id).get_coordinates()[0] + + assert_allclose(np.sort(results, axis=0), np.sort(results_grid, axis=0), atol=1e-5) \ No newline at end of file From 4503462ac10c3f8fcacc8ac64358d8005ee247c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Wed, 18 Jul 2018 17:20:37 +0200 Subject: [PATCH 23/30] Removed MDAnalysis/lib/c_gridsearch.pyx --- package/MDAnalysis/lib/c_gridsearch.pyx | 797 ------------------------ 1 file changed, 797 deletions(-) delete mode 100644 package/MDAnalysis/lib/c_gridsearch.pyx diff --git a/package/MDAnalysis/lib/c_gridsearch.pyx b/package/MDAnalysis/lib/c_gridsearch.pyx deleted file mode 100644 index 1ece4c5a1d6..00000000000 --- a/package/MDAnalysis/lib/c_gridsearch.pyx +++ /dev/null @@ -1,797 +0,0 @@ -# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; -*- -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 -# -# MDAnalysis --- https://www.mdanalysis.org -# -# Copyright (C) 2013-2018 Sébastien Buchoux -# Copyright (c) 2018 The MDAnalysis Development Team and contributors -# (see the file AUTHORS for the full list of names) -# -# Released under the GNU Public Licence, v3 or any higher version -# -# Please cite your use of MDAnalysis in published work: -# -# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler, -# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. -# MDAnalysis: A Python package for the rapid analysis of molecular dynamics -# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th -# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. -# -# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. -# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. -# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 -# -# - -#cython: cdivision=True -#cython: boundscheck=False -#cython: initializedcheck=False - -""" -Neighbor search library --- :mod:`MDAnalysis.lib.grid` -====================================================== - -This Neighbor search library is a serialized Cython port of the NS grid search implemented in GROMACS. -""" - - -# Preprocessor DEFs -DEF DIM = 3 -DEF XX = 0 -DEF YY = 1 -DEF ZZ = 2 - -DEF EPSILON = 1e-5 - -DEF NEIGHBORHOOD_ALLOCATION_INCREMENT = 50 - -DEF BOX_MARGIN=1.0010 -DEF MAX_NTRICVEC=12 - -from libc.stdlib cimport malloc, realloc, free -from libc.math cimport sqrt - -import numpy as np -cimport numpy as np - -ctypedef np.int_t ns_int -ctypedef np.float32_t real -ctypedef real rvec[DIM] -ctypedef real matrix[DIM][DIM] - -cdef struct ns_grid: - ns_int size - ns_int[DIM] ncells - real[DIM] cellsize - ns_int *nbeads - ns_int **beadids - -cdef struct ns_neighborhood: - real cutoff - ns_int allocated_size - ns_int size - ns_int *beadids - -cdef struct ns_neighborhood_holder: - ns_int size - ns_neighborhood **neighborhoods - -# Useful stuff - -cdef real rvec_norm2(const rvec a) nogil: - return a[XX]*a[XX]+a[YY]*a[YY]+a[ZZ]*a[ZZ] - -cdef void rvec_clear(rvec a) nogil: - a[XX]=0.0 - a[YY]=0.0 - a[ZZ]=0.0 - - -cdef struct cPBCBox_t: - matrix box - rvec fbox_diag - rvec hbox_diag - rvec mhbox_diag - real max_cutoff2 - ns_int ntric_vec - ns_int[DIM] tric_shift[MAX_NTRICVEC] - real[DIM] tric_vec[MAX_NTRICVEC] - - -# Class to handle PBC calculations -cdef class PBCBox(object): - cdef cPBCBox_t c_pbcbox - cdef rvec center - cdef rvec bbox_center - - def __init__(self, real[:,::1] box): - self.update(box) - - cdef void fast_update(self, real[:,::1] box) nogil: - cdef ns_int i, j, k, d, jc, kc, shift - cdef real d2old, d2new, d2new_c - cdef rvec trial, pos - cdef ns_int ii, jj ,kk - cdef ns_int *order = [0, -1, 1, -2, 2] - cdef bint use - cdef real min_hv2, min_ss, tmp - - rvec_clear(self.center) - # Update matrix - for i in range(DIM): - for j in range(DIM): - self.c_pbcbox.box[i][j] = box[i, j] - self.center[j] += 0.5 * box[i, j] - self.bbox_center[i] = 0.5 * box[i, i] - - # Update diagonals - for i in range(DIM): - self.c_pbcbox.fbox_diag[i] = box[i, i] - self.c_pbcbox.hbox_diag[i] = self.c_pbcbox.fbox_diag[i] * 0.5 - self.c_pbcbox.mhbox_diag[i] = - self.c_pbcbox.hbox_diag[i] - - # Update maximum cutoff - - # Physical limitation of the cut-off - # by half the length of the shortest box vector. - min_hv2 = min(0.25 * rvec_norm2(&box[XX, XX]), 0.25 * rvec_norm2(&box[YY, XX])) - min_hv2 = min(min_hv2, 0.25 * rvec_norm2(&box[ZZ, XX])) - - # Limitation to the smallest diagonal element due to optimizations: - # checking only linear combinations of single box-vectors (2 in x) - # in the grid search and pbc_dx is a lot faster - # than checking all possible combinations. - tmp = box[YY, YY] - if box[ZZ, YY] < 0: - tmp -= box[ZZ, YY] - else: - tmp += box[ZZ, YY] - - min_ss = min(box[XX, XX], min(tmp, box[ZZ, ZZ])) - - self.c_pbcbox.max_cutoff2 = min(min_hv2, min_ss * min_ss) - - # Update shift vectors - self.c_pbcbox.ntric_vec = 0 - - # We will only use single shifts - for kk in range(3): - k = order[kk] - - for jj in range(3): - j = order[jj] - - for ii in range(3): - i = order[ii] - - # A shift is only useful when it is trilinic - if j != 0 or k != 0: - d2old = 0 - d2new = 0 - - for d in range(DIM): - trial[d] = i*box[XX, d] + j*box[YY, d] + k*box[ZZ, d] - - # Choose the vector within the brick around 0,0,0 that - # will become the shortest due to shift try. - - if d == DIM: - trial[d] = 0 - pos[d] = 0 - else: - if trial[d] < 0: - pos[d] = min(self.c_pbcbox.hbox_diag[d], -trial[d]) - else: - pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) - - d2old += pos[d]**2 - d2new += (pos[d] + trial[d])**2 - - if BOX_MARGIN*d2new < d2old: - if not (j < -1 or j > 1 or k < -1 or k > 1): - use = True - - for dd in range(DIM): - if dd == 0: - shift = i - elif dd == 1: - shift = j - else: - shift = k - - if shift: - d2new_c = 0 - - for d in range(DIM): - d2new_c += (pos[d] + trial[d] - shift*box[dd, d])**2 - - if d2new_c <= BOX_MARGIN*d2new: - use = False - - if use: # Accept this shift vector. - if self.c_pbcbox.ntric_vec >= MAX_NTRICVEC: - with gil: - print("\nWARNING: Found more than %d triclinic " - "correction vectors, ignoring some." - % MAX_NTRICVEC) - print(" There is probably something wrong with " - "your box.") - print(np.array(box)) - - for i in range(self.c_pbcbox.ntric_vec): - print(" -> shift #{}: [{}, {}, {}]".format(i+1, - self.c_pbcbox.tric_shift[i][XX], - self.c_pbcbox.tric_shift[i][YY], - self.c_pbcbox.tric_shift[i][ZZ])) - else: - for d in range(DIM): - self.c_pbcbox.tric_vec[self.c_pbcbox.ntric_vec][d] = \ - trial[d] - self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][XX] = i - self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][YY] = j - self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][ZZ] = k - self.c_pbcbox.ntric_vec += 1 - - - def update(self, real[:,::1] box): - if box.shape[0] != DIM or box.shape[1] != DIM: - raise ValueError("Box must be a %i x %i matrix. (shape: %i x %i)" % - (DIM, DIM, box.shape[0], box.shape[1])) - if (box[XX, XX] == 0) or (box[YY, YY] == 0) or (box[ZZ, ZZ] == 0): - raise ValueError("Box does not correspond to PBC=xyz") - self.fast_update(box) - - - cdef void fast_pbc_dx(self, rvec ref, rvec other, rvec dx) nogil: - cdef ns_int i, j - cdef rvec dx_start, trial - - for i in range(DIM): - dx[i] = other[i] - ref[i] - - for i in range (DIM-1, -1, -1): - while dx[i] > self.c_pbcbox.hbox_diag[i]: - for j in range (i, -1, -1): - dx[j] -= self.c_pbcbox.box[i][j] - - while dx[i] <= self.c_pbcbox.mhbox_diag[i]: - for j in range (i, -1, -1): - dx[j] += self.c_pbcbox.box[i][j] - - cdef real fast_distance2(self, rvec a, rvec b) nogil: - cdef rvec dx - self.fast_pbc_dx(a, b, dx) - return rvec_norm2(dx) - - cdef real fast_distance(self, rvec a, rvec b) nogil: - return sqrt(self.fast_distance2(a,b)) - - cdef real[:, ::1]fast_put_atoms_in_bbox(self, real[:,::1] coords) nogil: - cdef ns_int i, m, d, natoms, wd = 0 - cdef real[:,::1] bbox_coords - - natoms = coords.shape[0] - with gil: - bbox_coords = coords.copy() - - for i in range(natoms): - for m in range(DIM - 1, -1, -1): - while bbox_coords[i, m] < 0: - for d in range(m+1): - bbox_coords[i, d] += self.c_pbcbox.box[m][d] - while bbox_coords[i, m] >= self.c_pbcbox.box[m][m]: - for d in range(m+1): - bbox_coords[i, d] -= self.c_pbcbox.box[m][d] - return bbox_coords - - def put_atoms_in_bbox(self, real[:,::1] coords): - if coords.shape[0] == 0: - return np.zeros((0, DIM), dtype=np.float32) - return np.asarray(self.fast_put_atoms_in_bbox(coords)) - -######################################################################################################################## -# -# Neighbor Search Stuff -# -######################################################################################################################## -cdef struct ns_grid: - ns_int size - ns_int[DIM] ncells - real[DIM] cellsize - ns_int *nbeads - ns_int **beadids - -cdef ns_neighborhood_holder *create_neighborhood_holder() nogil: - cdef ns_neighborhood_holder *holder - - holder = malloc(sizeof(ns_neighborhood_holder)) - holder.size = 0 - holder.neighborhoods = NULL - - return holder - -cdef void free_neighborhood_holder(ns_neighborhood_holder *holder) nogil: - cdef ns_int i - - if holder == NULL: - return - - for i in range(holder.size): - if holder.neighborhoods[i].beadids != NULL: - free(holder.neighborhoods[i].beadids) - free(holder.neighborhoods[i]) - - if holder.neighborhoods != NULL: - free(holder.neighborhoods) - free(holder) - -cdef ns_neighborhood *retrieve_neighborhood(rvec current_coords, real[:, ::1]neighborcoords, ns_grid *grid, PBCBox box, real cutoff2) nogil: - cdef ns_int d, m - cdef ns_int xi, yi, zi, bid - cdef real d2 - cdef rvec shifted_coord, dx, neighbor_coord, corrected_coords - - cdef bint already_checked[27] - cdef bint skip - cdef ns_int nchecked = 0, icheck - cdef ns_int cell_index - - cdef ns_neighborhood *neighborhood = malloc(sizeof(ns_neighborhood)) - if neighborhood == NULL: - return NULL - - neighborhood.size = 0 - neighborhood.allocated_size = NEIGHBORHOOD_ALLOCATION_INCREMENT - neighborhood.beadids = malloc(NEIGHBORHOOD_ALLOCATION_INCREMENT * sizeof(ns_int)) - - if neighborhood.beadids == NULL: - free(neighborhood) - return NULL - - for zi in range(3): - for yi in range(3): - for xi in range(3): - # Calculate and/or reinitialize shifted coordinates - shifted_coord[XX] = current_coords[XX] + (xi - 1) * grid.cellsize[XX] - shifted_coord[YY] = current_coords[YY] + (yi - 1) * grid.cellsize[YY] - shifted_coord[ZZ] = current_coords[ZZ] + (zi - 1) * grid.cellsize[ZZ] - - # Make sure the shifted coordinates is inside the brick-shaped box - for m in range(DIM - 1, -1, -1): - - while shifted_coord[m] < 0: - for d in range(m+1): - shifted_coord[d] += box.c_pbcbox.box[m][d] - - - while shifted_coord[m] >= box.c_pbcbox.box[m][m]: - for d in range(m+1): - shifted_coord[d] -= box.c_pbcbox.box[m][d] - - # Get the cell index corresponding to the coord - cell_index = (shifted_coord[ZZ] / grid.cellsize[ZZ]) * (grid.ncells[XX] * grid.ncells[YY]) +\ - (shifted_coord[YY] / grid.cellsize[YY]) * grid.ncells[XX] + \ - (shifted_coord[XX] / grid.cellsize[XX]) - - # Just a safeguard - if cell_index >= grid.size: - continue - - # Check the cell index was not already selected - skip = False - for icheck in range(nchecked): - if already_checked[icheck] == cell_index: - skip = True - break - if skip: - continue - - # Search for neighbors inside this cell - for i_bead in range(grid.nbeads[cell_index]): - bid = grid.beadids[cell_index][i_bead] - - box.fast_pbc_dx(current_coords, &neighborcoords[bid, XX], dx) - - d2 = rvec_norm2(dx) - - if d2 < cutoff2: - if d2 < EPSILON: # Don't add the current bead as its own neighbor! - continue - - # Update neighbor lists - neighborhood.beadids[neighborhood.size] = bid - neighborhood.size += 1 - - if neighborhood.size >= neighborhood.allocated_size: - neighborhood.allocated_size += NEIGHBORHOOD_ALLOCATION_INCREMENT - neighborhood.beadids = realloc( neighborhood.beadids, neighborhood.allocated_size * sizeof(ns_int)) - - if neighborhood.beadids == NULL: - free(neighborhood) - return NULL - - # Register the cell as checked - already_checked[nchecked] = cell_index - nchecked += 1 - - return neighborhood - - -cdef ns_neighborhood_holder *ns_core(real[:, ::1] refcoords, - real[:, ::1] neighborcoords, - ns_grid *grid, - PBCBox box, - real cutoff) nogil: - cdef ns_int coordid, i, j - cdef ns_int ncoords = refcoords.shape[0] - cdef ns_int ncoords_neighbors = neighborcoords.shape[0] - cdef real cutoff2 = cutoff * cutoff - cdef ns_neighborhood_holder *holder - - cdef ns_int *neighbor_buf - cdef ns_int buf_size, ibuf - - holder = create_neighborhood_holder() - if holder == NULL: - return NULL - - holder.neighborhoods = malloc(sizeof(ns_neighborhood *) * ncoords) - if holder.neighborhoods == NULL: - free_neighborhood_holder(holder) - return NULL - - # Here starts the real core and the iteration over coordinates - for coordid in range(ncoords): - holder.neighborhoods[coordid] = retrieve_neighborhood(&refcoords[coordid, XX], - neighborcoords, - grid, - box, - cutoff2) - if holder.neighborhoods[coordid] == NULL: - free_neighborhood_holder(holder) - return NULL - - holder.neighborhoods[coordid].cutoff = cutoff - holder.size += 1 - - return holder - -cdef class NSResults(object): - """ - Class used to store results returned by `MDAnalysis.lib.grid.FastNS.search` - """ - cdef PBCBox box - cdef readonly real cutoff - cdef np.ndarray grid_coords - cdef np.ndarray ref_coords - cdef ns_int **nids - cdef ns_int *nsizes - cdef ns_int size - cdef list indices - cdef list coordinates - cdef list distances - - def __dealloc__(self): - if self.nids != NULL: - for i in range(self.size): - if self.nids[i] != NULL: - free(self.nids[i]) - free(self.nids) - - if self.nsizes != NULL: - free(self.nsizes) - - def __init__(self, PBCBox box, real cutoff): - self.box = box - self.cutoff = cutoff - - self.size = 0 - self.nids = NULL - self.nsizes = NULL - - self.grid_coords = None - self.ref_coords = None - - self.indices = None - self.coordinates = None - self.distances = None - - - cdef populate(self, ns_neighborhood_holder *holder, grid_coords, ref_coords): - cdef ns_int nid, i - cdef ns_neighborhood *neighborhood - - self.grid_coords = np.asarray(grid_coords) - self.ref_coords = np.asarray(ref_coords) - - # Allocate memory - self.nsizes = malloc(sizeof(ns_int) * holder.size) - if self.nsizes == NULL: - raise MemoryError("Could not allocate memory for NSResults") - - self.nids = malloc(sizeof(ns_int *) * holder.size) - if self.nids == NULL: - raise MemoryError("Could not allocate memory for NSResults") - - for nid in range(holder.size): - neighborhood = holder.neighborhoods[nid] - - self.nsizes[nid] = neighborhood.size - - self.nids[nid] = malloc(sizeof(ns_int *) * neighborhood.size) - if self.nids[nid] == NULL: - raise MemoryError("Could not allocate memory for NSResults") - - with nogil: - for nid in range(holder.size): - neighborhood = holder.neighborhoods[nid] - - for i in range(neighborhood.size): - self.nids[nid][i] = neighborhood.beadids[i] - - self.size = holder.size - - def get_indices(self): - """ - Return Neighbors indices. - - :return: list of indices - """ - cdef ns_int i, nid, size - - if self.indices is None: - indices = [] - - for nid in range(self.size): - size = self.nsizes[nid] - - tmp_incides = np.empty((size), dtype=np.int) - - for i in range(size): - tmp_incides[i] = self.nids[nid][i] - - indices.append(tmp_incides) - - self.indices = indices - - return self.indices - - - def get_coordinates(self): - """ - Return coordinates of neighbors. - - :return: list of coordinates - """ - cdef ns_int i, nid, size, beadid - - if self.coordinates is None: - coordinates = [] - - for nid in range(self.size): - size = self.nsizes[nid] - - tmp_values = np.empty((size, DIM), dtype=np.float32) - - for i in range(size): - beadid = self.nids[nid][i] - tmp_values[i] = self.grid_coords[beadid] - - coordinates.append(tmp_values) - - self.coordinates = coordinates - - return self.coordinates - - - def get_distances(self): - """ - Return coordinates of neighbors. - - :return: list of distances - """ - cdef ns_int i, nid, size, j, beadid - cdef rvec ref, other, dx - cdef real dist - cdef real[:, ::1] ref_coords = self.ref_coords - cdef real[:, ::1] grid_coords = self.grid_coords - - if self.distances is None: - distances = [] - - for nid in range(self.size): - size = self.nsizes[nid] - - tmp_values = np.empty((size), dtype=np.float32) - ref = &ref_coords[nid, 0] - - for i in range(size): - beadid = self.nids[nid][i] - other = &grid_coords[beadid, 0] - - tmp_values[i] = self.box.fast_distance(ref, other) - - distances.append(tmp_values) - - self.distances = distances - - return self.distances - - -# Python interface -cdef class FastNS(object): - cdef PBCBox box - cdef readonly real[:, ::1] coords - cdef real[:, ::1] coords_bbox - cdef readonly real cutoff - cdef bint prepared - cdef ns_grid grid - - def __init__(self, u): - import MDAnalysis as mda - from MDAnalysis.lib.mdamath import triclinic_vectors - - if not isinstance(u, mda.Universe): - raise TypeError("FastNS class must be initialized with a valid MDAnalysis.Universe instance") - box = triclinic_vectors(u.dimensions) - - if box.shape != (3, 3): - raise ValueError("Box must be provided as triclinic_dimensions (a 3x3 numpy.ndarray of unit cell vectors") - - self.box = PBCBox(box) - - self.coords = None - self.coords_bbox = None - - self.cutoff = -1 - - self.prepared = False - - self.grid.size = 0 - - - def __dealloc__(self): - cdef ns_int i - # Deallocate NS grid - if self.grid.nbeads != NULL: - free(self.grid.nbeads) - - for i in range(self.grid.size): - if self.grid.beadids[i] != NULL: - free(self.grid.beadids[i]) - free(self.grid.beadids) - - self.grid.size = 0 - - - def set_coords(self, real[:, ::1] coords): - self.coords = coords - - # Make sure atoms are inside the brick-shaped box - self.coords_bbox = self.box.fast_put_atoms_in_bbox(coords) - - self.prepared = False - - - def set_cutoff(self, real cutoff): - self.cutoff = cutoff - - self.prepared = False - - - def prepare(self, force=False): - cdef ns_int i, cellindex = -1 - cdef ns_int *allocated_size = NULL - cdef ns_int ncoords = self.coords.shape[0] - cdef ns_int allocation_guess - cdef rvec *coords = &self.coords_bbox[0, 0] - - if self.prepared and not force: - print("NS already prepared, nothing to do!") - - if self.coords is None: - raise ValueError("Coordinates must be set before NS preparation!") - - if self.cutoff < 0: - raise ValueError("Cutoff must be set before NS preparation!") - - with nogil: - - # Initializing grid - for i in range(DIM): - self.grid.ncells[i] = (self.box.c_pbcbox.box[i][i] / self.cutoff) - if self.grid.ncells[i] == 0: - self.grid.ncells[i] = 1 - self.grid.cellsize[i] = self.box.c_pbcbox.box[i][i] / self.grid.ncells[i] - self.grid.size = self.grid.ncells[XX] * self.grid.ncells[YY] * self.grid.ncells[ZZ] - - # This is just a guess on how much memory we might for each grid cell: - # we just assume an average bead density and we take four times this density just to be safe - allocation_guess = (4 * (ncoords / self.grid.size + 1)) - - # Allocate memory for the grid - self.grid.nbeads = malloc(sizeof(ns_int) * self.grid.size) - if self.grid.nbeads == NULL: - with gil: - raise MemoryError("Could not allocate memory for NS grid") - - # Allocate memory from temporary allocation counter - allocated_size = malloc(sizeof(ns_int) * self.grid.size) - if allocated_size == NULL: - # No need to free grid.nbeads as it will be freed by destroy_nsgrid called by __dealloc___ - with gil: - raise MemoryError("Could not allocate memory for allocation buffer") - - # Pre-allocate some memory for grid cells - for i in range(self.grid.size): - self.grid.nbeads[i] = 0 - allocated_size[i] = allocation_guess - - self.grid.beadids = malloc(sizeof(ns_int *) * self.grid.size) - if self.grid.beadids == NULL: - with gil: - raise MemoryError("Could not allocate memory for grid cells") - - for i in range(self.grid.size): - self.grid.beadids[i] = malloc(sizeof(ns_int) * allocated_size[i]) - if self.grid.beadids[i] == NULL: - with gil: - raise MemoryError("Could not allocate memory for grid cell") - - # Populate grid cells using the coordinates (ie do the heavy work) - for i in range(ncoords): - cellindex = (coords[i][ZZ] / self.grid.cellsize[ZZ]) * (self.grid.ncells[XX] * self.grid.ncells[YY]) +\ - (coords[i][YY] / self.grid.cellsize[YY]) * self.grid.ncells[XX] + \ - (coords[i][XX] / self.grid.cellsize[XX]) - - self.grid.beadids[cellindex][self.grid.nbeads[cellindex]] = i - self.grid.nbeads[cellindex] += 1 - - # We need to allocate more memory (simply double the amount of memory as - # 1. it should barely be needed - # 2. the size should stay fairly reasonable - if self.grid.nbeads[cellindex] >= allocated_size[cellindex]: - allocated_size[cellindex] *= 2 - self.grid.beadids[cellindex] = realloc( self.grid.beadids[cellindex], sizeof(ns_int) * allocated_size[cellindex]) - - # Now we can free the allocation buffer - free(allocated_size) - self.prepared = True - - - def search(self, search_coords): - cdef real[:, ::1] search_coords_bbox - cdef real[:, ::1] search_coords_view - cdef ns_int nid, i, j - cdef ns_neighborhood_holder *holder - cdef ns_neighborhood *neighborhood - - if not self.prepared: - self.prepare() - - # Check the shape of search_coords as a array of 3D coords if needed - shape = search_coords.shape - if len(shape) == 1: - if not shape[0] == 3: - raise ValueError("Coordinates must be 3D") - else: - search_coords_view = np.array([search_coords,], dtype=np.float32) - else: - search_coords_view = search_coords - - # Make sure atoms are inside the brick-shaped box - search_coords_bbox = self.box.fast_put_atoms_in_bbox(search_coords_view) - - with nogil: - holder = ns_core(search_coords_bbox, self.coords_bbox, &self.grid, self.box, self.cutoff) - - if holder == NULL: - raise MemoryError("Could not allocate memory to run NS core") - - results = NSResults(self.box, self.cutoff) - results.populate(holder, self.coords, search_coords_view) - - # Free memory allocated to holder - free_neighborhood_holder(holder) - - return results From 63b6f630a95543dac8e3cb515b7067656c03e309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Wed, 18 Jul 2018 17:29:27 +0200 Subject: [PATCH 24/30] Removed all references to c_gridsearch.pyx/grid module that reappeared after merge/rebase --- .../documentation_pages/lib_modules.rst | 4 +- package/setup.py | 6 +- .../MDAnalysisTests/lib/test_gridsearch.py | 140 ------------------ testsuite/MDAnalysisTests/lib/test_nsgrid.py | 4 +- 4 files changed, 7 insertions(+), 147 deletions(-) delete mode 100644 testsuite/MDAnalysisTests/lib/test_gridsearch.py diff --git a/package/doc/sphinx/source/documentation_pages/lib_modules.rst b/package/doc/sphinx/source/documentation_pages/lib_modules.rst index 83d7f81cfd9..f918f37137e 100644 --- a/package/doc/sphinx/source/documentation_pages/lib_modules.rst +++ b/package/doc/sphinx/source/documentation_pages/lib_modules.rst @@ -36,7 +36,7 @@ functions whereas mathematical functions are to be found in :mod:`MDAnalysis.lib.NeighborSearch` contains classes to do neighbor searches with MDAnalysis objects. -:mod:`MDAnalysis.lib.grid` contains a fast implementation of grid neighbor search. +:mod:`MDAnalysis.lib.nsgrid` contains a fast implementation of grid neighbor search. List of modules --------------- @@ -51,7 +51,7 @@ List of modules ./lib/transformations ./lib/qcprot ./lib/util - ./lib/grid + ./lib/nsgrid Low level file formats ---------------------- diff --git a/package/setup.py b/package/setup.py index 1532e0895d3..ced342972c4 100755 --- a/package/setup.py +++ b/package/setup.py @@ -374,8 +374,8 @@ def extensions(config): libraries=mathlib, define_macros=define_macros, extra_compile_args=extra_compile_args) - grid = MDAExtension('MDAnalysis.lib.grid', - ['MDAnalysis/lib/c_gridsearch' + source_suffix], + nsgrid = MDAExtension('MDAnalysis.lib.nsgrid', + ['MDAnalysis/lib/nsgrid' + source_suffix], include_dirs=include_dirs, libraries=mathlib + parallel_libraries, define_macros=define_macros + parallel_macros, @@ -383,7 +383,7 @@ def extensions(config): extra_link_args=parallel_args) pre_exts = [libdcd, distances, distances_omp, qcprot, transformation, libmdaxdr, util, encore_utils, - ap_clustering, spe_dimred, cutil, grid] + ap_clustering, spe_dimred, cutil, nsgrid] cython_generated = [] if use_cython: diff --git a/testsuite/MDAnalysisTests/lib/test_gridsearch.py b/testsuite/MDAnalysisTests/lib/test_gridsearch.py deleted file mode 100644 index 359d32544b7..00000000000 --- a/testsuite/MDAnalysisTests/lib/test_gridsearch.py +++ /dev/null @@ -1,140 +0,0 @@ -# -*- Mode: python; tab-width: 4; indent-tabs-mode:nil; coding:utf-8 -*- -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 fileencoding=utf-8 -# -# MDAnalysis --- https://www.mdanalysis.org -# Copyright (c) 2006-2018 The MDAnalysis Development Team and contributors -# (see the file AUTHORS for the full list of names) -# -# Released under the GNU Public Licence, v2 or any higher version -# -# Please cite your use of MDAnalysis in published work: -# -# R. J. Gowers, M. Linke, J. Barnoud, T. J. E. Reddy, M. N. Melo, S. L. Seyler, -# D. L. Dotson, J. Domanski, S. Buchoux, I. M. Kenney, and O. Beckstein. -# MDAnalysis: A Python package for the rapid analysis of molecular dynamics -# simulations. In S. Benthall and S. Rostrup editors, Proceedings of the 15th -# Python in Science Conference, pages 102-109, Austin, TX, 2016. SciPy. -# -# N. Michaud-Agrawal, E. J. Denning, T. B. Woolf, and O. Beckstein. -# MDAnalysis: A Toolkit for the Analysis of Molecular Dynamics Simulations. -# J. Comput. Chem. 32 (2011), 2319--2327, doi:10.1002/jcc.21787 -# - -from __future__ import print_function, absolute_import - -import pytest -from numpy.testing import assert_equal, assert_allclose -import numpy as np - -import MDAnalysis as mda -from MDAnalysis.lib import grid -from MDAnalysis.lib.pkdtree import PeriodicKDTree - -from MDAnalysisTests.datafiles import GRO - - -@pytest.fixture -def universe(): - u = mda.Universe(GRO) - return u - - -@pytest.fixture -def grid_results(): - u = mda.Universe(GRO) - cutoff = 2 - ref_pos = u.atoms.positions[13937] - return run_grid_search(u, ref_pos, cutoff) - - -def run_grid_search(u, ref_pos, cutoff): - coords = u.atoms.positions - - # Run grid search - searcher = grid.FastNS(u) - searcher.set_cutoff(cutoff) - searcher.set_coords(coords) - searcher.prepare() - - return searcher.search(ref_pos) - - -def run_search(universe, ref_id): - cutoff = 3 - coords = universe.atoms.positions - ref_pos = coords[ref_id] - - # Run pkdtree search - pkdt = PeriodicKDTree(universe.atoms.dimensions, bucket_size=10) - pkdt.set_coords(coords) - pkdt.search(ref_pos, cutoff) - - results_pkdtree = pkdt.get_indices() - results_pkdtree.remove(ref_id) - results_pkdtree = np.array(results_pkdtree) - results_pkdtree.sort() - - # Run grid search - results_grid = run_grid_search(universe, ref_pos, cutoff) - results_grid = results_grid.get_indices()[0] - results_grid.sort() - - return results_pkdtree, results_grid - - -def test_gridsearch(universe): - """Check that pkdtree and grid search return the same results (No PBC needed)""" - - ref_id = 0 - results_pkdtree, results_grid = run_search(universe, ref_id) - assert_equal(results_pkdtree, results_grid) - - -def test_gridsearch_PBC(universe): - """Check that pkdtree and grid search return the same results (PBC needed)""" - - ref_id = 13937 - results_pkdtree, results_grid = run_search(universe, ref_id) - assert_equal(results_pkdtree, results_grid) - - -def test_gridsearch_arraycoord(universe): - """Check the NS routine accepts a single bead coordinate as well as array of coordinates""" - cutoff = 2 - ref_pos = universe.atoms.positions[:5] - - results = [ - np.array([2, 1, 4, 3]), - np.array([2, 0, 3]), - np.array([0, 1, 3]), - np.array([ 2, 0, 1, 38341]), - np.array([ 6, 0, 5, 17]) - ] - - results_grid = run_grid_search(universe, ref_pos, cutoff).get_indices() - - assert_equal(results_grid, results) - - -def test_gridsearch_search_coordinates(grid_results): - """Check the NS routine can return coordinates instead of ids""" - - results = np.array( - [ - [40.32, 34.25, 55.9], - [0.61, 76.33, -0.56], - [0.48999998, 75.9, 0.19999999], - [-0.11, 76.19, 0.77] - ]) - - assert_allclose(grid_results.get_coordinates()[0], results) - - -def test_gridsearch_search_distances(grid_results): - """Check the NS routine can return PBC distances from neighbors""" - results = np.array([0.096, 0.096, 0.015, 0.179]) * 10 # These distances were obtained using gmx distance - results.sort() - - rounded_results = np.round(grid_results.get_distances()[0], 2) - - assert_allclose(sorted(rounded_results), results) \ No newline at end of file diff --git a/testsuite/MDAnalysisTests/lib/test_nsgrid.py b/testsuite/MDAnalysisTests/lib/test_nsgrid.py index 33f7f1c67fe..f692b7ca64c 100644 --- a/testsuite/MDAnalysisTests/lib/test_nsgrid.py +++ b/testsuite/MDAnalysisTests/lib/test_nsgrid.py @@ -79,8 +79,8 @@ def test_pbc_distances(): pbcbox.dx(a, bad) assert_equal(pbcbox.dx(a, b), dx) - assert pbcbox.distance(a, b) == np.sqrt(np.sum(dx*dx)) - assert pbcbox.distance2(a, b) == np.sum(dx*dx) + assert_allclose(pbcbox.distance(a, b), np.sqrt(np.sum(dx*dx)), atol=1e-5) + assert_allclose(pbcbox.distance2(a, b), np.sum(dx*dx), atol=1e-5) def test_pbc_put_in_bbox(): From 9e47943283ffac5eaa37ef5b4b87e4acf0a3f8fa Mon Sep 17 00:00:00 2001 From: Max Linke Date: Sat, 21 Jul 2018 14:30:23 +0200 Subject: [PATCH 25/30] remove unused code --- package/MDAnalysis/lib/nsgrid.pyx | 93 ++----------------------------- 1 file changed, 5 insertions(+), 88 deletions(-) diff --git a/package/MDAnalysis/lib/nsgrid.pyx b/package/MDAnalysis/lib/nsgrid.pyx index e4de849ffff..dc4017191c0 100644 --- a/package/MDAnalysis/lib/nsgrid.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -69,20 +69,15 @@ cdef void rvec_clear(rvec a) nogil: a[YY]=0.0 a[ZZ]=0.0 -######################################################################################################################## -# -# Utility class to handle PBC -# -######################################################################################################################## +############################### +# Utility class to handle PBC # +############################### cdef struct cPBCBox_t: matrix box rvec fbox_diag rvec hbox_diag rvec mhbox_diag real max_cutoff2 - ns_int ntric_vec - ns_int[DIM] tric_shift[MAX_NTRICVEC] - real[DIM] tric_vec[MAX_NTRICVEC] # Class to handle PBC calculations @@ -141,90 +136,12 @@ cdef class PBCBox(object): tmp += box[ZZ, YY] min_ss = min(box[XX, XX], min(tmp, box[ZZ, ZZ])) - self.c_pbcbox.max_cutoff2 = min(min_hv2, min_ss * min_ss) - # Update shift vectors - self.c_pbcbox.ntric_vec = 0 - - # We will only use single shifts - for kk in range(3): - k = order[kk] - - for jj in range(3): - j = order[jj] - - for ii in range(3): - i = order[ii] - - # A shift is only useful when it is trilinic - if j != 0 or k != 0: - d2old = 0 - d2new = 0 - - for d in range(DIM): - trial[d] = i*box[XX, d] + j*box[YY, d] + k*box[ZZ, d] - - # Choose the vector within the brick around 0,0,0 that - # will become the shortest due to shift try. - if trial[d] < 0: - pos[d] = min(self.c_pbcbox.hbox_diag[d], -trial[d]) - else: - pos[d] = max(-self.c_pbcbox.hbox_diag[d], -trial[d]) - - d2old += pos[d]**2 - d2new += (pos[d] + trial[d])**2 - - if BOX_MARGIN*d2new < d2old: - if not (j < -1 or j > 1 or k < -1 or k > 1): - use = True - - for dd in range(DIM): - if dd == 0: - shift = i - elif dd == 1: - shift = j - else: - shift = k - - if shift: - d2new_c = 0 - - for d in range(DIM): - d2new_c += (pos[d] + trial[d] - shift*box[dd, d])**2 - - if d2new_c <= BOX_MARGIN*d2new: - use = False - - if use: # Accept this shift vector. - if self.c_pbcbox.ntric_vec >= MAX_NTRICVEC: - with gil: - print("\nWARNING: Found more than %d triclinic " - "correction vectors, ignoring some." - % MAX_NTRICVEC) - print(" There is probably something wrong with " - "your box.") - print(np.array(box)) - - for i in range(self.c_pbcbox.ntric_vec): - print(" -> shift #{}: [{}, {}, {}]".format(i+1, - self.c_pbcbox.tric_shift[i][XX], - self.c_pbcbox.tric_shift[i][YY], - self.c_pbcbox.tric_shift[i][ZZ])) - else: - for d in range(DIM): - self.c_pbcbox.tric_vec[self.c_pbcbox.ntric_vec][d] = \ - trial[d] - self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][XX] = i - self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][YY] = j - self.c_pbcbox.tric_shift[self.c_pbcbox.ntric_vec][ZZ] = k - self.c_pbcbox.ntric_vec += 1 - - def update(self, real[:,::1] box): if box.shape[0] != DIM or box.shape[1] != DIM: - raise ValueError("Box must be a %i x %i matrix. (shape: %i x %i)" % - (DIM, DIM, box.shape[0], box.shape[1])) + raise ValueError("Box must be a {} x {} matrix. Got: {} x {})".format( + DIM, DIM, box.shape[0], box.shape[1])) if (box[XX, XX] == 0) or (box[YY, YY] == 0) or (box[ZZ, ZZ] == 0): raise ValueError("Box does not correspond to PBC=xyz") self.fast_update(box) From ced71af380253d3a4bd2c6829251daf6b6619453 Mon Sep 17 00:00:00 2001 From: Max Linke Date: Sat, 21 Jul 2018 14:30:43 +0200 Subject: [PATCH 26/30] make init_callable twice --- package/MDAnalysis/lib/nsgrid.pyx | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/package/MDAnalysis/lib/nsgrid.pyx b/package/MDAnalysis/lib/nsgrid.pyx index dc4017191c0..f051e7e413d 100644 --- a/package/MDAnalysis/lib/nsgrid.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -221,11 +221,9 @@ cdef class PBCBox(object): return np.asarray(self.fast_put_atoms_in_bbox(coords)) -######################################################################################################################## -# -# Neighbor Search Stuff -# -######################################################################################################################## +######################### +# Neighbor Search Stuff # +######################### cdef class NSResults(object): cdef readonly real cutoff cdef ns_int npairs @@ -246,7 +244,6 @@ cdef class NSResults(object): cdef np.ndarray pair_coordinates_buffer def __init__(self, real cutoff, real[:, ::1]coords, ns_int[:] search_ids, debug=False): - self.debug = debug self.cutoff = cutoff self.coords = coords @@ -254,12 +251,19 @@ cdef class NSResults(object): # Preallocate memory self.allocation_size = search_ids.shape[0] + 1 - self.pairs = PyMem_Malloc(sizeof(ipair) * self.allocation_size) - if not self.pairs: - MemoryError("Could not allocate memory for NSResults.pairs ({} bits requested)".format(sizeof(ipair) * self.allocation_size)) - self.pair_distances2 = PyMem_Malloc(sizeof(real) * self.allocation_size) - if not self.pair_distances2: - raise MemoryError("Could not allocate memory for NSResults.pair_distances2 ({} bits requested)".format(sizeof(real) * self.allocation_size)) + if not self.pairs and not self.pair_distances2: + self.pairs = PyMem_Malloc(sizeof(ipair) * self.allocation_size) + if not self.pairs: + MemoryError("Could not allocate memory for NSResults.pairs " + "({} bits requested)".format(sizeof(ipair) * self.allocation_size)) + self.pair_distances2 = PyMem_Malloc(sizeof(real) * self.allocation_size) + if not self.pair_distances2: + raise MemoryError("Could not allocate memory for NSResults.pair_distances2 " + "({} bits requested)".format(sizeof(real) * self.allocation_size)) + else: + if self.resize(self.allocation_size + (self.allocation_size * 0.5 + 1)) == 0: + raise MemoryError("foo") + self.npairs = 0 # Buffer From c8bac83b1aa3069f247ba6baa1f82b5dc4afcff3 Mon Sep 17 00:00:00 2001 From: Max Linke Date: Sat, 21 Jul 2018 15:34:44 +0200 Subject: [PATCH 27/30] use OK and ERROR as return codes --- package/MDAnalysis/lib/nsgrid.pyx | 32 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/package/MDAnalysis/lib/nsgrid.pyx b/package/MDAnalysis/lib/nsgrid.pyx index f051e7e413d..01bf67e0c4c 100644 --- a/package/MDAnalysis/lib/nsgrid.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -46,6 +46,9 @@ DEF EPSILON = 1e-5 DEF BOX_MARGIN=1.0010 DEF MAX_NTRICVEC=12 +DEF OK=0 +DEF ERROR=1 + # Used to handle memory allocation from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free from libc.math cimport sqrt @@ -261,7 +264,7 @@ cdef class NSResults(object): raise MemoryError("Could not allocate memory for NSResults.pair_distances2 " "({} bits requested)".format(sizeof(real) * self.allocation_size)) else: - if self.resize(self.allocation_size + (self.allocation_size * 0.5 + 1)) == 0: + if self.resize(self.allocation_size) != OK: raise MemoryError("foo") self.npairs = 0 @@ -278,13 +281,13 @@ cdef class NSResults(object): PyMem_Free(self.pair_distances2) cdef int add_neighbors(self, ns_int beadid_i, ns_int beadid_j, real distance2) nogil: - # Important: If this function returns 0, it means that memory allocation failed + # Important: If this function returns ERROR, it means that memory allocation failed # Reallocate memory if needed if self.npairs >= self.allocation_size: # We need to reallocate memory - if self.resize(self.allocation_size + (self.allocation_size * 0.5 + 1)) == 0: - return 0 + if self.resize(self.allocation_size + (self.allocation_size * 0.5 + 1)) != OK: + return ERROR # Actually store pair and distance squared if beadid_i < beadid_j: @@ -296,20 +299,20 @@ cdef class NSResults(object): self.pair_distances2[self.npairs] = distance2 self.npairs += 1 - return self.npairs + return OK cdef int resize(self, ns_int new_size) nogil: # Important: If this function returns 0, it means that memory allocation failed if new_size < self.npairs: # Silently ignored the request - return 1 + return OK if self.allocation_size >= new_size: if self.debug: with gil: print("NSresults: Reallocation requested but not needed ({} requested but {} already allocated)".format(new_size, self.allocation_size)) - return 1 + return OK self.allocation_size = new_size @@ -323,12 +326,12 @@ cdef class NSResults(object): self.pair_distances2 = PyMem_Realloc(self.pair_distances2, sizeof(real) * self.allocation_size) if not self.pairs: - return 0 + return ERROR if not self.pair_distances2: - return 0 + return ERROR - return 1 + return OK def get_pairs(self): cdef ns_int i @@ -823,14 +826,7 @@ cdef class FastNS(object): if d2 < EPSILON: continue - - # if self.debug and debug: - # self.grid.cellid2cellxyz(cellindex, debug_cellxyz) - # with gil: - # self.box.fast_pbc_dx(&self.coords[current_beadid, XX], &self.coords[bid, XX], dx) - # dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) - # print("FastNS: \_ Neighbor found: bead#{} (cell[{},{},{}]) -> dx={} -> d={:.3f}".format(bid, debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], dx_py, np.sqrt(d2))) - if results.add_neighbors(current_beadid, bid, d2) == 0: + elif results.add_neighbors(current_beadid, bid, d2) != OK: memory_error = True break npairs += 1 From 0cc8b5cd44f14554f84e26c068d1510ce053f835 Mon Sep 17 00:00:00 2001 From: Max Linke Date: Sat, 21 Jul 2018 15:34:59 +0200 Subject: [PATCH 28/30] remove dead code --- package/MDAnalysis/lib/nsgrid.pyx | 171 +----------------------------- 1 file changed, 4 insertions(+), 167 deletions(-) diff --git a/package/MDAnalysis/lib/nsgrid.pyx b/package/MDAnalysis/lib/nsgrid.pyx index 01bf67e0c4c..df6d130ef14 100644 --- a/package/MDAnalysis/lib/nsgrid.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -615,14 +615,11 @@ cdef class FastNS(object): cdef rvec shifted_coord, probe, dx cdef ns_int nchecked = 0 + cdef ns_int[:] checked = np.zeros(size, dtype=np.int) cdef real cutoff2 = self.cutoff * self.cutoff - cdef ns_int[:] checked cdef ns_int npairs = 0 - #cdef bint debug=False - - if not self.prepared: self.prepare() @@ -636,134 +633,44 @@ cdef class FastNS(object): search_ids_view = search_ids size_search = search_ids.shape[0] - checked = np.zeros(size, dtype=np.int) - results = NSResults(self.cutoff, self.coords, search_ids, self.debug) cdef bint memory_error = False - # if self.debug and debug: - # print("FastNS: Debug flag is set to True for FastNS.search()") - with nogil: for i in range(size_search): if memory_error: break current_beadid = search_ids_view[i] - cellindex = self.grid.cellids[current_beadid] self.grid.cellid2cellxyz(cellindex, cellxyz) - - # if self.debug and debug: - # with gil: - # print("FastNS: Checking neighbors for bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]:" .format( - # current_beadid, - # self.coords[current_beadid, XX], self.coords[current_beadid, YY], self.coords[current_beadid, ZZ], - # self.coords_bbox[current_beadid, XX], self.coords_bbox[current_beadid, YY], self.coords_bbox[current_beadid, ZZ], - # cellxyz[XX], cellxyz[YY], cellxyz[ZZ])) - - for xi in range(DIM): if memory_error: break - - if not self.box.is_triclinic: - # If box is not triclinic (ie rect), when can already check if the shift can be skipped (ie cutoff inside the cell) - if xi == 0: - if self.coords_bbox[current_beadid, XX] - self.cutoff > self.grid.cellsize[XX] * cellxyz[XX]: - # if self.debug and debug: - # with gil: - # print("FastNS: -> Bead X={:.3f}, Cell X={:.3f}, cutoff={:.3f} -> -X shift ignored".format( - # self.coords_bbox[current_beadid, XX], - # self.grid.cellsize[XX] * cellxyz[XX], - # self.cutoff - # )) - continue - - if xi == 2: - if self.coords_bbox[current_beadid, XX] + self.cutoff < self.grid.cellsize[XX] * (cellxyz[XX] + 1): - # if self.debug and debug: - # with gil: - # print( - # "FastNS: -> Bead X={:.3f}, Next cell X={:.3f}, cutoff={:.3f} -> +X shift ignored".format( - # self.coords_bbox[current_beadid, XX], - # self.grid.cellsize[XX] * (cellxyz[XX] + 1), - # self.cutoff - # )) - continue - for yi in range(DIM): if memory_error: break - - if not self.box.is_triclinic: - if yi == 0: - if self.coords_bbox[current_beadid, YY] - self.cutoff > self.grid.cellsize[YY] * cellxyz[YY]: - # if self.debug and debug: - # with gil: - # print("FastNS: -> Bead Y={:.3f}, Cell Y={:.3f}, cutoff={:.3f} -> -Y shift is ignored".format( - # self.coords_bbox[current_beadid, YY], - # self.grid.cellsize[YY] * cellxyz[YY], - # self.cutoff, - # )) - continue - - if yi == 2: - if self.coords_bbox[current_beadid, YY] + self.cutoff < self.grid.cellsize[YY] * (cellxyz[YY] + 1): - # if self.debug and debug: - # with gil: - # print("FastNS: -> Bead Y={:.3f}, Next cell Y={:.3f}, cutoff={:.3f} -> +Y shift is ignored".format( - # self.coords_bbox[current_beadid, YY], - # self.grid.cellsize[YY] * (cellxyz[YY] +1), - # self.cutoff, - # )) - continue - - for zi in range(DIM): - if not self.box.is_triclinic: - if zi == 0: - if self.coords_bbox[current_beadid, ZZ] - self.cutoff > self.grid.cellsize[ZZ] * cellxyz[ZZ]: - if self.coords_bbox[current_beadid, ZZ] - self.cutoff > 0: - # if self.debug and debug: - # with gil: - # print("FastNS: -> Bead Z={:.3f}, Cell Z={:.3f}, cutoff={:.3f} -> -Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[ZZ] * cellxyz[ZZ], self.cutoff)) - continue - - if zi == 2: - if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.grid.cellsize[ZZ] * (cellxyz[ZZ] + 1): - if self.coords_bbox[current_beadid, ZZ] + self.cutoff < self.box.c_pbcbox.box[ZZ][ZZ]: - # if self.debug and debug: - # with gil: - # print("FastNS: -> Bead Z={:.3f}, Next cell Z={:.3f}, cutoff={:.3f} -> +Z shift is ignored".format(self.coords_bbox[current_beadid, ZZ], self.grid.cellsize[XX] * (cellxyz[ZZ] + 1), self.cutoff)) - continue - + if memory_error: + break # Calculate and/or reinitialize shifted coordinates shifted_coord[XX] = self.coords[current_beadid, XX] + (xi - 1) * self.grid.cellsize[XX] shifted_coord[YY] = self.coords[current_beadid, YY] + (yi - 1) * self.grid.cellsize[YY] shifted_coord[ZZ] = self.coords[current_beadid, ZZ] + (zi - 1) * self.grid.cellsize[ZZ] - probe[XX] = self.coords[current_beadid, XX] + (xi - 1) * self.cutoff probe[YY] = self.coords[current_beadid, YY] + (yi - 1) * self.cutoff probe[ZZ] = self.coords[current_beadid, ZZ] + (zi - 1) * self.cutoff - # Make sure the shifted coordinates is inside the brick-shaped box for m in range(DIM - 1, -1, -1): - while shifted_coord[m] < 0: for d in range(m+1): shifted_coord[d] += self.box.c_pbcbox.box[m][d] - - while shifted_coord[m] >= self.box.c_pbcbox.box[m][m]: for d in range(m+1): shifted_coord[d] -= self.box.c_pbcbox.box[m][d] - while probe[m] < 0: for d in range(m+1): probe[d] += self.box.c_pbcbox.box[m][d] - - while probe[m] >= self.box.c_pbcbox.box[m][m]: for d in range(m+1): probe[d] -= self.box.c_pbcbox.box[m][d] @@ -772,58 +679,12 @@ cdef class FastNS(object): cellindex_adjacent = self.grid.coord2cellid(shifted_coord) cellindex_probe = self.grid.coord2cellid(probe) - if cellindex == cellindex_probe and xi != 1 and yi != 1 and zi != 1: - # if self.debug and debug: - # with gil: - # print("FastNS: Grid shift [{}][{}][{}]: Cutoff is inside current cell -> This shift is ignored".format( - # xi - 1, - # yi -1, - # zi -1 - # )) - continue - - # if self.debug and debug: - # self.grid.cellid2cellxyz(cellindex_adjacent, debug_cellxyz) - # with gil: - # dist_shift = self.box.fast_distance(&self.coords[current_beadid, XX], shifted_coord) - # grid_shift = np.array([(xi - 1) * self.grid.cellsize[XX], - # (yi - 1) * self.grid.cellsize[YY], - # (zi - 1) * self.grid.cellsize[ZZ]]) - # print("FastNS: -> Checking cell#{} ({},{},{}) for neighbors (dshift={:.3f}, grid_shift=({:.3f},{:.3f},{:.3f}->{:.3f})".format( - # cellindex, - # debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], - # dist_shift, - # grid_shift[XX], grid_shift[YY], grid_shift[ZZ], - # np.sqrt(np.sum(grid_shift**2)) - # )) - - for j in range(self.grid.nbeads[cellindex_adjacent]): bid = self.grid.beadids[cellindex_adjacent * self.grid.nbeads_per_cell + j] - if checked[bid] != 0: continue - d2 = self.box.fast_distance2(&self.coords_bbox[current_beadid, XX], &self.coords_bbox[bid, XX]) - - # if self.debug: - # self.grid.cellid2cellxyz(cellindex, debug_cellxyz) - # with gil: - # print( - # "Beads #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) and #{} (cell[{},{},{}]-coords[{:.3f},{:.3f},{:.3f}]) are tested (d2={:.3f})".format( - # current_beadid, - # cellxyz[XX], cellxyz[YY], cellxyz[ZZ], - # self.coords_bbox[current_beadid, XX], - # self.coords_bbox[current_beadid, YY], - # self.coords_bbox[current_beadid, ZZ], - # bid, - # debug_cellxyz[XX], debug_cellxyz[YY], debug_cellxyz[ZZ], - # self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], - # self.coords_bbox[bid, ZZ], - # d2)) - if d2 < cutoff2: - if d2 < EPSILON: continue elif results.add_neighbors(current_beadid, bid, d2) != OK: @@ -831,30 +692,6 @@ cdef class FastNS(object): break npairs += 1 checked[current_beadid] = 1 - if memory_error: raise MemoryError("Could not allocate memory to store NS results") - - - if self.debug: - print("Total number of pairs={}".format(npairs)) - - # ref_bead = 13937 - # beads = np.array([4398, 4401, 13939, 13940, 13941, 17987, 23518, 23519, 23521, 23734, 47451]) - 1 - # for bid in beads: - # self.box.fast_pbc_dx(&self.coords[ref_bead, XX], &self.coords[bid, XX], dx) - # dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) - # self.box.fast_pbc_dx(&self.coords_bbox[ref_bead, XX], &self.coords_bbox[bid, XX], dx) - # rect_dx_py = np.array([dx[XX], dx[YY], dx[ZZ]]) - # self.grid.cellid2cellxyz(self.grid.coord2cellid(&self.coords_bbox[bid, XX]), cellxyz) - # print("Bead #{} ({:.3f},{:.3f},{:.3f})->rect({:.3f},{:.3f},{:.3f}) - cell[{},{},{}]: dx=[{:.3f},{:.3f},{:.3f}] -> dist: {:.3f} ({})".format( - # bid, - # self.coords[bid, XX], self.coords[bid, YY], self.coords[bid,ZZ], - # self.coords_bbox[bid, XX], self.coords_bbox[bid, YY], self.coords_bbox[bid,ZZ], - # cellxyz[XX], cellxyz[YY], cellxyz[ZZ], - # dx[XX], dx[YY], dx[ZZ], - # np.sqrt(np.sum(dx_py**2)), - # self.box.fast_distance(&self.coords[ref_bead, XX], &self.coords[bid, XX]) <= self.cutoff, - # )) - - return results \ No newline at end of file + return results From 17ad02349e0c44075c5f8f1aea16386b3c9d7f5e Mon Sep 17 00:00:00 2001 From: Max Linke Date: Sat, 21 Jul 2018 20:42:25 +0200 Subject: [PATCH 29/30] remove possible memory leak --- package/MDAnalysis/lib/nsgrid.pyx | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/package/MDAnalysis/lib/nsgrid.pyx b/package/MDAnalysis/lib/nsgrid.pyx index df6d130ef14..6dff29a6bab 100644 --- a/package/MDAnalysis/lib/nsgrid.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -506,12 +506,7 @@ cdef class NSGrid(object): cdef fill_grid(self, real[:, ::1] coords): cdef ns_int i, cellindex = -1 cdef ns_int ncoords = coords.shape[0] - cdef ns_int *beadcounts = NULL - - # Allocate memory - beadcounts = PyMem_Malloc(sizeof(ns_int) * self.size) - if not beadcounts: - raise MemoryError("Could not allocate memory for bead count buffer ({} bits requested)".format(sizeof(ns_int) * self.size)) + cdef ns_int[:] beadcounts = np.empty(self.size, dtype=np.int) with nogil: # Initialize buffers @@ -528,12 +523,12 @@ cdef class NSGrid(object): if self.nbeads[cellindex] > self.nbeads_per_cell: self.nbeads_per_cell = self.nbeads[cellindex] - # Allocate memory - with gil: - self.beadids = PyMem_Malloc(sizeof(ns_int) * self.size * self.nbeads_per_cell) #np.empty((self.size, nbeads_max), dtype=np.int) - if not self.beadids: - raise MemoryError("Could not allocate memory for NSGrid.beadids ({} bits requested)".format(sizeof(ns_int) * self.size * self.nbeads_per_cell)) + # Allocate memory + self.beadids = PyMem_Malloc(sizeof(ns_int) * self.size * self.nbeads_per_cell) #np.empty((self.size, nbeads_max), dtype=np.int) + if not self.beadids: + raise MemoryError("Could not allocate memory for NSGrid.beadids ({} bits requested)".format(sizeof(ns_int) * self.size * self.nbeads_per_cell)) + with nogil: # Second loop: fill grid for i in range(ncoords): @@ -542,8 +537,6 @@ cdef class NSGrid(object): self.beadids[cellindex * self.nbeads_per_cell + beadcounts[cellindex]] = i beadcounts[cellindex] += 1 - # Now we can free the allocation buffer - PyMem_Free(beadcounts) cdef class FastNS(object): From 607aadff62cb58fc079b2481d20fcddfb91fbba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Buchoux?= Date: Thu, 2 Aug 2018 23:08:04 +0200 Subject: [PATCH 30/30] started documentation for FastNS --- package/MDAnalysis/lib/nsgrid.pyx | 103 +++++++++++++++++- .../source/documentation_pages/lib/grid.rst | 2 - .../source/documentation_pages/lib/nsgrid.rst | 2 + 3 files changed, 102 insertions(+), 5 deletions(-) delete mode 100644 package/doc/sphinx/source/documentation_pages/lib/grid.rst create mode 100644 package/doc/sphinx/source/documentation_pages/lib/nsgrid.rst diff --git a/package/MDAnalysis/lib/nsgrid.pyx b/package/MDAnalysis/lib/nsgrid.pyx index 6dff29a6bab..181828424f4 100644 --- a/package/MDAnalysis/lib/nsgrid.pyx +++ b/package/MDAnalysis/lib/nsgrid.pyx @@ -26,12 +26,37 @@ # cython: cdivision=True # cython: boundscheck=False # cython: initializedcheck=False +# cython: embedsignature=True """ -Neighbor search library --- :mod:`MDAnalysis.lib.grid` -====================================================== +Neighbor search library --- :mod:`MDAnalysis.lib.nsgrid` +======================================================== + +About the code +-------------- + +This Neighbor search library is a serialized Cython version greatly inspired by the NS grid search implemented in +`GROMACS `_ . + +GROMACS 4.x code (more precisely `nsgrid.c `_ and +`ns.c `_ ) was used as reference to write this file. + +GROMACS 4.x code is released under the GNU Public Licence v2. + + +About the algorithm +------------------- + +The neighbor search implemented here is based on `cell lists `_ which allow +computation of pairs [#]_ with a cost of :math:`O(N)`, instead of :math:`O(N^2)`. +The basic algorithm is described in the following references:` + +Examples +-------- + + +.. [#] a pair correspond to two particles that are considered as neighbors . -This Neighbor search library is a serialized Cython port of the NS grid search implemented in GROMACS. """ @@ -85,12 +110,38 @@ cdef struct cPBCBox_t: # Class to handle PBC calculations cdef class PBCBox(object): + """ + Cython implementation of `PBC-related `_ operations. + This class is used by classes :class:`FastNS` and :class:`NSGrid` to put all particles inside a brick-shaped box + and to compute PBC-aware distance. + + .. warning:: + + This class is not meant to be used by end users. + + .. warning:: + + Even if MD triclinic boxes can be handle by this class, internal optimization is made based on the + assumption that particles are inside a brick-shaped box. When this is not the case, calculated distances are not + warranted to be exact. + + """ cdef cPBCBox_t c_pbcbox cdef rvec center cdef rvec bbox_center cdef bint is_triclinic def __init__(self, real[:,::1] box): + """ + + Parameters + ---------- + + box : :class:`numpy.ndarray` + the MD box vectors as returned by + :func:`MDAnalysis.lib.mdamath.triclinic_vectors`. dtype must be :class:`numpy.float32` + + """ self.update(box) cdef void fast_update(self, real[:,::1] box) nogil: @@ -142,6 +193,24 @@ cdef class PBCBox(object): self.c_pbcbox.max_cutoff2 = min(min_hv2, min_ss * min_ss) def update(self, real[:,::1] box): + """ + + Updates internal MD box representation and parameters used for calculations. + + .. note:: + + Call to this method is only needed when the MD box is changed as it always called when class is + instantiated. + + Parameters + ---------- + + box : a :class:`numpy.ndarray` that describes the MD box vectors as returned by + :func:`MDAnalysis.lib.mdamath.triclinic_vectors`. + `dtype` must be :class:`numpy.float32` + + """ + if box.shape[0] != DIM or box.shape[1] != DIM: raise ValueError("Box must be a {} x {} matrix. Got: {} x {})".format( DIM, DIM, box.shape[0], box.shape[1])) @@ -167,6 +236,21 @@ cdef class PBCBox(object): dx[j] += self.c_pbcbox.box[i][j] def dx(self, real[:] a, real[:] b): + """ + + + Returns the distance vector :math:`dx` between two coordinates :math:`a` and :math:`b` . + if the minimum image convention is respected by :math:`a` and :math:`b` then :math:`dx = ab` . + if not, :math:`dx` is modified so it is always compliant with the minimum image convention. + + Parameters + ---------- + a : :class:`numpy.ndarray` + coordinates with a `shape` of 3 and a `dtype` of :class:`numpy.float32` + b : :class:`numpy.ndarray` + coordinates with a `shape` of 3 and a `dtype` of :class:`numpy.float32` + + """ cdef rvec dx if a.shape[0] != DIM or b.shape[0] != DIM: raise ValueError("Not 3 D coordinates") @@ -182,6 +266,19 @@ cdef class PBCBox(object): return rvec_norm2(dx) def distance2(self, real[:] a, real[:] b): + """ + Returns the distance vector :math:`dx` between two coordinates :math:`a` and :math:`b` . + if the minimum image convention is respected by :math:`a` and :math:`b` then :math:`dx = ab` . + if not, :math:`dx` is modified so it is always compliant with the minimum image convention. + + Parameters + ---------- + a : :class:`numpy.ndarray` + coordinates with a `shape` of 3 and a `dtype` of :class:`numpy.float32` + b : :class:`numpy.ndarray` + coordinates with a `shape` of 3 and a `dtype` of :class:`numpy.float32` + + """ if a.shape[0] != DIM or b.shape[0] != DIM: raise ValueError("Not 3 D coordinates") return self.fast_distance2(&a[XX], &b[XX]) diff --git a/package/doc/sphinx/source/documentation_pages/lib/grid.rst b/package/doc/sphinx/source/documentation_pages/lib/grid.rst deleted file mode 100644 index a6143e9ca38..00000000000 --- a/package/doc/sphinx/source/documentation_pages/lib/grid.rst +++ /dev/null @@ -1,2 +0,0 @@ -.. automodule:: MDAnalysis.lib.grid - :members: \ No newline at end of file diff --git a/package/doc/sphinx/source/documentation_pages/lib/nsgrid.rst b/package/doc/sphinx/source/documentation_pages/lib/nsgrid.rst new file mode 100644 index 00000000000..f36f2480b0b --- /dev/null +++ b/package/doc/sphinx/source/documentation_pages/lib/nsgrid.rst @@ -0,0 +1,2 @@ +.. automodule:: MDAnalysis.lib.nsgrid + :members: \ No newline at end of file