potential fix for issue 218 (circular references preventing garbage collection)

Changelog

@@ -1,6 +1,10 @@
  version 1.0.9 (not yet released)
  ================================
 
+ * use weak references to group instances when creating Dimension and Variable
+   objects. This prevents cyclic references messing up garbage collection (issue
+   218, pull request 219).
+
  * accessing values from a 0-dimensional Variable now returns a 0-dimensional
    numpy array, not a 1-dimensional array (issue 220). To write code
    compatible with both the old and new (fixed) behavior, wrap values accessed

README → README.md

@@ -16,4 +16,4 @@ Quick Start
 * To run all the tests, execute 'cd test; python run_all.py'.
 
 More detailed documentation is available at docs/index.html, or
-http://netcdf4-python.googlecode.com/svn/trunk/docs/index.html.
+http://unidata.github.io/netcdf4-python.

netCDF4.pyx

@@ -804,6 +804,7 @@ __version__ = "1.0.9"
 import posixpath
 import netcdftime
 import numpy
+import weakref
 import sys
 import warnings
 from glob import glob
@@ -1353,6 +1354,7 @@ netcdf C library version >= 4.3.1, otherwise will always return C{UNDEFINED}.
 the L{Dataset} in a unix directory format (the names of groups in the
 hierarchy separated by backslashes). A L{Dataset}, instance is the root
 group, so the path is simply C{'/'}."""
+    cdef object __weakref__
     cdef public int _grpid
     cdef public int _isopen
     cdef public groups, dimensions, variables, disk_format, path, parent,\
@@ -1434,9 +1436,9 @@ group, so the path is simply C{'/'}."""
         self.file_format = self.data_model
         self.disk_format = _get_full_format(grpid)
         # diskless read access only works with NETCDF_CLASSIC (for now)
-        ncopen = mode.startswith('a') or mode.startswith('r')
-        if diskless and self.data_model != 'NETCDF3_CLASSIC' and ncopen:
-            raise ValueError("diskless access only supported for NETCDF3_CLASSIC format")
+        #ncopen = mode.startswith('a') or mode.startswith('r')
+        #if diskless and self.data_model != 'NETCDF3_CLASSIC' and ncopen:
+        #    raise ValueError("diskless access only supported for NETCDF3_CLASSIC format")
         self._grpid = grpid
         self._isopen = 1
         self.path = '/'
@@ -2036,7 +2038,8 @@ determine if the dimension is unlimited"""
         cdef char *dimname
         cdef size_t lendim
         self._grpid = grp._grpid
-        self._grp = grp
+        # make a weakref to group to avoid circular ref (issue 218)
+        self._grp = weakref.proxy(grp)
         self._data_model = grp.data_model
         self._name = name
         if 'id' in kwargs:
@@ -2261,7 +2264,8 @@ instance. If C{None}, the data is not truncated. """
         if type(dimensions) == str or type(dimensions) == bytes or type(dimensions) == unicode:
             dimensions = dimensions,
         self._grpid = grp._grpid
-        self._grp = grp
+        # make a weakref to group to avoid circular ref (issue 218)
+        self._grp = weakref.proxy(grp)
         # convert to a real numpy datatype object if necessary.
         if (not isinstance(datatype, CompoundType) and \
             not isinstance(datatype, VLType)) and \
@@ -2494,19 +2498,20 @@ instance. If C{None}, the data is not truncated. """
         ierr = nc_inq_var_fill(self._grpid,self._varid,&no_fill,NULL)
         if ierr != NC_NOERR:
             raise RuntimeError((<char *>nc_strerror(ierr)).decode('ascii'))
-        if no_fill != 1:
-            try:
-                fillval = self._FillValue
-                msg = 'filling on'
-            except AttributeError:
-                fillval = default_fillvals[self.dtype.str[1:]]
-                if self.dtype.str[1:] in ['u1','i1']:
-                    msg = 'filling on, default _FillValue of %s ignored\n' % fillval
-                else:
-                    msg = 'filling on, default _FillValue of %s used\n' % fillval
-            ncdump_var.append(msg)
-        else:
-            ncdump_var.append('filling off\n')
+        if self._isprimitive:
+            if no_fill != 1:
+                try:
+                    fillval = self._FillValue
+                    msg = 'filling on'
+                except AttributeError:
+                    fillval = default_fillvals[self.dtype.str[1:]]
+                    if self.dtype.str[1:] in ['u1','i1']:
+                        msg = 'filling on, default _FillValue of %s ignored\n' % fillval
+                    else:
+                        msg = 'filling on, default _FillValue of %s used\n' % fillval
+                ncdump_var.append(msg)
+            else:
+                ncdump_var.append('filling off\n')
 
 
         return ''.join(ncdump_var)

setup.cfg.template

@@ -3,10 +3,15 @@
 [options]
 # if true, the nc-config script (installed with netcdf 4.1.2 and higher)
 # will be used to determine the locations of required libraries.
-#use_ncconfig=True
-# path to nc-config script.
+# Usually, nothing else is needed.
+use_ncconfig=True
+# path to nc-config script (use if not found in unix PATH).
 #ncconfig=/usr/local/bin/nc-config 
 [directories]
+#
+# If nc-config doesn't do the trick, you can specify the locations
+# of the libraries and headers manually below
+#
 # uncomment and set to netCDF install location.
 # Include files should be located in netCDF4_dir/include and
 # the library should be located in netCDF4_dir/lib.

setup.py

@@ -336,7 +336,7 @@ def getnetcdfvers(libdirs):
   long_description = "netCDF version 4 has many features not found in earlier versions of the library, such as hierarchical groups, zlib compression, multiple unlimited dimensions, and new data types.  It is implemented on top of HDF5.  This module implements most of the new features, and can read and write netCDF files compatible with older versions of the library.  The API is modelled after Scientific.IO.NetCDF, and should be familiar to users of that module.\n\nThis project has a `Subversion repository <http://code.google.com/p/netcdf4-python/source>`_ where you may access the most up-to-date source.",
   author            = "Jeff Whitaker",
   author_email      = "jeffrey.s.whitaker@noaa.gov",
-  url               = "http://netcdf4-python.googlecode.com/svn/trunk/docs/netCDF4-module.html",
+  url               = "http://github.com/Unidata/netcdf4-python",
   download_url      = "http://python.org/pypi/netCDF4",
   scripts           = ['utils/nc3tonc4','utils/nc4tonc3','utils/ncinfo'],
   platforms         = ["any"],

test/tst_refcount.py

@@ -0,0 +1,28 @@
+import unittest, netCDF4, tempfile, os
+
+file_name = tempfile.mktemp(".nc")
+
+class RefCountTestCase(unittest.TestCase):
+
+    def setUp(self):
+        nc = netCDF4.Dataset(file_name, mode='w', format='NETCDF4')
+        d = nc.createDimension('fred', 2000)
+        v = nc.createVariable('frank','f',('fred',))
+        self.file = file_name
+        self.nc = nc
+
+    def tearDown(self):
+        # Remove the temporary files
+        os.remove(self.file)
+
+    def runTest(self):
+        """testing garbage collection (issue 218)"""
+        # this should trigger garbage collection (__dealloc__ method)
+        del self.nc
+        # if __dealloc__ not called to close file, then this
+        # will fail with "Permission denied" error (since you can't
+        # open a file 'w' that is already open for writing).
+        nc = netCDF4.Dataset(self.file, mode='w', format='NETCDF4')
+
+if __name__ == '__main__':
+    unittest.main()