Improve the health of this package

.landscape.yml

@@ -0,0 +1,22 @@
+pylint:
+    disable:
+        # We use this a lot (e.g. via document._meta)
+        - protected-access
+
+    options:
+        additional-builtins:
+            # add xrange and long as valid built-ins. In Python 3, xrange is
+            # translated into range and long is translated into int via 2to3 (see
+            # "use_2to3" in setup.py). This should be removed when we drop Python
+            # 2 support (which probably won't happen any time soon).
+            - xrange
+            - long
+
+pyflakes:
+    disable:
+        # undefined variables are already covered by pylint (and exclude
+        # xrange & long)
+        - F821
+
+ignore-paths:
+    - benchmark.py

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 
 python:
-- '2.6'  # TODO remove in v0.11.0
 - '2.7'
 - '3.3'
 - '3.4'
@@ -43,7 +42,11 @@ before_script:
 script:
 - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage
 
-after_script: coveralls --verbose
+# For now only submit coveralls for Python v2.7. Python v3.x currently shows
+# 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible
+# code in a separate dir and runs tests on that.
+after_script:
+- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coveralls --verbose; fi
 
 notifications:
   irc: irc.freenode.org#mongoengine

benchmark.py

@@ -1,118 +1,41 @@
 #!/usr/bin/env python
 
-import timeit
-
-
-def cprofile_main():
-    from pymongo import Connection
-    connection = Connection()
-    connection.drop_database('timeit_test')
-    connection.disconnect()
-
-    from mongoengine import Document, DictField, connect
-    connect("timeit_test")
-
-    class Noddy(Document):
-        fields = DictField()
+"""
+Simple benchmark comparing PyMongo and MongoEngine.
+
+Sample run on a mid 2015 MacBook Pro (commit b282511):
+
+Benchmarking...
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries - Pymongo
+2.58979988098
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries - Pymongo write_concern={"w": 0}
+1.26657605171
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries - MongoEngine
+8.4351580143
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries without continual assign - MongoEngine
+7.20191693306
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade = True
+6.31104588509
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True
+6.07083487511
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False
+5.97704291344
+----------------------------------------------------------------------------------------------------
+Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False
+5.9111430645
+"""
 
-    for i in range(1):
-        noddy = Noddy()
-        for j in range(20):
-            noddy.fields["key" + str(j)] = "value " + str(j)
-        noddy.save()
+import timeit
 
 
 def main():
-    """
-    0.4 Performance Figures ...
-
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - Pymongo
-    3.86744189262
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine
-    6.23374891281
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
-    5.33027005196
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
-    pass - No Cascade
-
-    0.5.X
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - Pymongo
-    3.89597702026
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine
-    21.7735359669
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
-    19.8670389652
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
-    pass - No Cascade
-
-    0.6.X
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - Pymongo
-    3.81559205055
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine
-    10.0446798801
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
-    9.51354718208
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
-    9.02567505836
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, force=True
-    8.44933390617
-
-    0.7.X
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - Pymongo
-    3.78801012039
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine
-    9.73050498962
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False
-    8.33456707001
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, safe=False, validate=False, cascade=False
-    8.37778115273
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, force=True
-    8.36906409264
-    0.8.X
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - Pymongo
-    3.69964408875
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - Pymongo write_concern={"w": 0}
-    3.5526599884
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine
-    7.00959801674
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries without continual assign - MongoEngine
-    5.60943293571
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine - write_concern={"w": 0}, cascade=True
-    6.715102911
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False, cascade=True
-    5.50644683838
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, write_concern={"w": 0}, validate=False
-    4.69851183891
-    ----------------------------------------------------------------------------------------------------
-    Creating 10000 dictionaries - MongoEngine, force_insert=True, write_concern={"w": 0}, validate=False
-    4.68946313858
-    ----------------------------------------------------------------------------------------------------
-    """
     print("Benchmarking...")
 
     setup = """
@@ -131,7 +54,7 @@ def main():
 for i in range(10000):
     example = {'fields': {}}
     for j in range(20):
-        example['fields']["key"+str(j)] = "value "+str(j)
+        example['fields']['key' + str(j)] = 'value ' + str(j)
 
     noddy.save(example)
 
@@ -146,17 +69,18 @@ def main():
 
     stmt = """
 from pymongo import MongoClient
+from pymongo.write_concern import WriteConcern
 connection = MongoClient()
 
-db = connection.timeit_test
+db = connection.get_database('timeit_test', write_concern=WriteConcern(w=0))
 noddy = db.noddy
 
 for i in range(10000):
     example = {'fields': {}}
     for j in range(20):
         example['fields']["key"+str(j)] = "value "+str(j)
 
-    noddy.save(example, write_concern={"w": 0})
+    noddy.save(example)
 
 myNoddys = noddy.find()
 [n for n in myNoddys] # iterate
@@ -171,10 +95,10 @@ def main():
 from pymongo import MongoClient
 connection = MongoClient()
 connection.drop_database('timeit_test')
-connection.disconnect()
+connection.close()
 
 from mongoengine import Document, DictField, connect
-connect("timeit_test")
+connect('timeit_test')
 
 class Noddy(Document):
     fields = DictField()

docs/changelog.rst

@@ -4,6 +4,13 @@ Changelog
 
 Development
 ===========
+- (Fill this out as you fix issues and develop you features).
+
+Changes in 0.11.0
+=================
+- BREAKING CHANGE: Renamed `ConnectionError` to `MongoEngineConnectionError` since the former is a built-in exception name in Python v3.x. #1428
+- BREAKING CHANGE: Dropped Python 2.6 support. #1428
+- BREAKING CHANGE: `from mongoengine.base import ErrorClass` won't work anymore for any error from `mongoengine.errors` (e.g. `ValidationError`). Use `from mongoengine.errors import ErrorClass instead`. #1428
 - Fixed absent rounding for DecimalField when `force_string` is set. #1103
 
 Changes in 0.10.8

docs/upgrade.rst

@@ -2,6 +2,32 @@
 Upgrading
 #########
 
+0.11.0
+******
+This release includes a major rehaul of MongoEngine's code quality and
+introduces a few breaking changes. It also touches many different parts of
+the package and although all the changes have been tested and scrutinized,
+you're encouraged to thorougly test the upgrade.
+
+First breaking change involves renaming `ConnectionError` to `MongoEngineConnectionError`.
+If you import or catch this exception, you'll need to rename it in your code.
+
+Second breaking change drops Python v2.6 support. If you run MongoEngine on
+that Python version, you'll need to upgrade it first.
+
+Third breaking change drops an old backward compatibility measure where
+`from mongoengine.base import ErrorClass` would work on top of
+`from mongoengine.errors import ErrorClass` (where `ErrorClass` is e.g.
+`ValidationError`). If you import any exceptions from `mongoengine.base`,
+change it to `mongoengine.errors`.
+
+0.10.8
+******
+This version fixed an issue where specifying a MongoDB URI host would override
+more information than it should. These changes are minor, but they still
+subtly modify the connection logic and thus you're encouraged to test your
+MongoDB connection before shipping v0.10.8 in production.
+
 0.10.7
 ******
 

mongoengine/__init__.py

@@ -1,25 +1,35 @@
-import connection
-from connection import *
-import document
-from document import *
-import errors
-from errors import *
-import fields
-from fields import *
-import queryset
-from queryset import *
-import signals
-from signals import *
-
-__all__ = (list(document.__all__) + fields.__all__ + connection.__all__ +
-           list(queryset.__all__) + signals.__all__ + list(errors.__all__))
+# Import submodules so that we can expose their __all__
+from mongoengine import connection
+from mongoengine import document
+from mongoengine import errors
+from mongoengine import fields
+from mongoengine import queryset
+from mongoengine import signals
+
+# Import everything from each submodule so that it can be accessed via
+# mongoengine, e.g. instead of `from mongoengine.connection import connect`,
+# users can simply use `from mongoengine import connect`, or even
+# `from mongoengine import *` and then `connect('testdb')`.
+from mongoengine.connection import *
+from mongoengine.document import *
+from mongoengine.errors import *
+from mongoengine.fields import *
+from mongoengine.queryset import *
+from mongoengine.signals import *
+
+
+__all__ = (list(document.__all__) + list(fields.__all__) +
+           list(connection.__all__) + list(queryset.__all__) +
+           list(signals.__all__) + list(errors.__all__))
+
 
 VERSION = (0, 10, 9)
 
 
 def get_version():
-    if isinstance(VERSION[-1], basestring):
-        return '.'.join(map(str, VERSION[:-1])) + VERSION[-1]
+    """Return the VERSION as a string, e.g. for VERSION == (0, 10, 7),
+    return '0.10.7'.
+    """
     return '.'.join(map(str, VERSION))
 
 

mongoengine/base/__init__.py

@@ -1,8 +1,28 @@
+# Base module is split into several files for convenience. Files inside of
+# this module should import from a specific submodule (e.g.
+# `from mongoengine.base.document import BaseDocument`), but all of the
+# other modules should import directly from the top-level module (e.g.
+# `from mongoengine.base import BaseDocument`). This approach is cleaner and
+# also helps with cyclical import errors.
 from mongoengine.base.common import *
 from mongoengine.base.datastructures import *
 from mongoengine.base.document import *
 from mongoengine.base.fields import *
 from mongoengine.base.metaclasses import *
 
-# Help with backwards compatibility
-from mongoengine.errors import *
+__all__ = (
+    # common
+    'UPDATE_OPERATORS', '_document_registry', 'get_document',
+
+    # datastructures
+    'BaseDict', 'BaseList', 'EmbeddedDocumentList',
+
+    # document
+    'BaseDocument',
+
+    # fields
+    'BaseField', 'ComplexBaseField', 'ObjectIdField', 'GeoJsonBaseField',
+
+    # metaclasses
+    'DocumentMetaclass', 'TopLevelDocumentMetaclass'
+)

mongoengine/base/common.py

@@ -1,13 +1,18 @@
 from mongoengine.errors import NotRegistered
 
-__all__ = ('ALLOW_INHERITANCE', 'get_document', '_document_registry')
+__all__ = ('UPDATE_OPERATORS', 'get_document', '_document_registry')
+
+
+UPDATE_OPERATORS = set(['set', 'unset', 'inc', 'dec', 'pop', 'push',
+                        'push_all', 'pull', 'pull_all', 'add_to_set',
+                        'set_on_insert', 'min', 'max'])
 
-ALLOW_INHERITANCE = False
 
 _document_registry = {}
 
 
 def get_document(name):
+    """Get a document class by name."""
     doc = _document_registry.get(name, None)
     if not doc:
         # Possible old style name