diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 66533c13d..d75bdb1a0 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -21,6 +21,7 @@ env: PYMONGO_3_9: 3.9 PYMONGO_3_11: 3.11 PYMONGO_3_12: 3.12 + PYMONGO_4_0: 4.0 MAIN_PYTHON_VERSION: 3.7 @@ -61,6 +62,9 @@ jobs: - python-version: 3.7 MONGODB: $MONGODB_4_4 PYMONGO: $PYMONGO_3_12 + - python-version: 3.9 + MONGODB: $MONGODB_4_4 + PYMONGO: $PYMONGO_4_0 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/AUTHORS b/AUTHORS index 60663940a..40508b532 100644 --- a/AUTHORS +++ b/AUTHORS @@ -263,3 +263,4 @@ that much better: * Timothé Perez (https://github.com/AchilleAsh) * oleksandr-l5 (https://github.com/oleksandr-l5) * Ido Shraga (https://github.com/idoshr) + * Terence Honles (https://github.com/terencehonles) diff --git a/docs/changelog.rst b/docs/changelog.rst index c49de79ff..32cfdfdb9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,11 +7,41 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). -- EnumField improvements: now `choices` limits the values of an enum to allow +- EnumField improvements: now ``choices`` limits the values of an enum to allow - Fix bug that prevented instance queryset from using custom queryset_class #2589 - Fix deepcopy of EmbeddedDocument #2202 - Fix error when using precision=0 with DecimalField #2535 - Add support for regex and whole word text search query #2568 +- BREAKING CHANGE: Updates to support pymongo 4.0. Where possible deprecated + functionality has been migrated, but additional care should be taken when + migrating to pymongo 4.0 as existing code may have been using deprecated + features which have now been removed #2614. + + For the pymongo migration guide see: + https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html. + + In addition to the changes in the migration guide, the following is a high + level overview of the changes made to MongoEngine when using pymongo 4.0: + + - limited support of geohaystack indexes has been removed + - ``QuerySet.map_reduce`` has been migrated from ``Collection.map_reduce`` + and ``Collection.inline_map_reduce`` to use + ``db.command({mapReduce: ..., ...})`` and support between the two may need + additional verification. + - UUIDs are encoded with the ``pythonLegacy`` encoding by default instead of + the newer and cross platform ``standard`` encoding. Existing UUIDs will + need to be migrated before changing the encoding, and this should be done + explicitly by the user rather than switching to a new default by + MongoEngine. This default will change at a later date, but to allow + specifying and then migrating to the new format a default ``json_options`` + has been provided. + - ``Queryset.count`` has been using ``Collection.count_documents`` and + transparently falling back to ``Collection.count`` when using features that + are not supported by ``Collection.count_documents``. ``Collection.count`` + has been removed and no automatic fallback is possible. The migration guide + documents the extended functionality which is no longer supported. Rewrite + the unsupported queries or fetch the whole result set and perform the count + locally. Changes in 0.23.1 =========== diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 46935c1b8..7a1de50e3 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -1,5 +1,6 @@ import copy import numbers +import warnings from functools import partial import pymongo @@ -23,11 +24,17 @@ OperationError, ValidationError, ) +from mongoengine.pymongo_support import LEGACY_JSON_OPTIONS __all__ = ("BaseDocument", "NON_FIELD_ERRORS") NON_FIELD_ERRORS = "__all__" +try: + GEOHAYSTACK = pymongo.GEOHAYSTACK +except AttributeError: + GEOHAYSTACK = None + class BaseDocument: # TODO simplify how `_changed_fields` is used. @@ -439,10 +446,19 @@ def to_json(self, *args, **kwargs): Defaults to True. """ use_db_field = kwargs.pop("use_db_field", True) + if "json_options" not in kwargs: + warnings.warn( + "No 'json_options' are specified! Falling back to " + "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " + "For use with other MongoDB drivers specify the UUID " + "representation to use.", + DeprecationWarning, + ) + kwargs["json_options"] = LEGACY_JSON_OPTIONS return json_util.dumps(self.to_mongo(use_db_field), *args, **kwargs) @classmethod - def from_json(cls, json_data, created=False): + def from_json(cls, json_data, created=False, **kwargs): """Converts json data to a Document instance :param str json_data: The json data to load into the Document @@ -460,7 +476,16 @@ def from_json(cls, json_data, created=False): # TODO should `created` default to False? If the object already exists # in the DB, you would likely retrieve it from MongoDB itself through # a query, not load it from JSON data. - return cls._from_son(json_util.loads(json_data), created=created) + if "json_options" not in kwargs: + warnings.warn( + "No 'json_options' are specified! Falling back to " + "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " + "For use with other MongoDB drivers specify the UUID " + "representation to use.", + DeprecationWarning, + ) + kwargs["json_options"] = LEGACY_JSON_OPTIONS + return cls._from_son(json_util.loads(json_data, **kwargs), created=created) def __expand_dynamic_values(self, name, value): """Expand any dynamic values to their correct types / values.""" @@ -898,7 +923,10 @@ def _build_index_spec(cls, spec): elif key.startswith("("): direction = pymongo.GEOSPHERE elif key.startswith(")"): - direction = pymongo.GEOHAYSTACK + try: + direction = pymongo.GEOHAYSTACK + except AttributeError: + raise NotImplementedError elif key.startswith("*"): direction = pymongo.GEO2D if key.startswith(("+", "-", "*", "$", "#", "(", ")")): @@ -923,10 +951,10 @@ def _build_index_spec(cls, spec): index_list.append((key, direction)) # Don't add cls to a geo index - if include_cls and direction not in ( - pymongo.GEO2D, - pymongo.GEOHAYSTACK, - pymongo.GEOSPHERE, + if ( + include_cls + and direction not in (pymongo.GEO2D, pymongo.GEOSPHERE) + and (GEOHAYSTACK is None or direction != GEOHAYSTACK) ): index_list.insert(0, ("_cls", 1)) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index e9078a2e7..bbc98f17d 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,6 +1,10 @@ +import warnings + from pymongo import MongoClient, ReadPreference, uri_parser from pymongo.database import _check_name +from mongoengine.pymongo_support import PYMONGO_VERSION + __all__ = [ "DEFAULT_CONNECTION_NAME", "DEFAULT_DATABASE_NAME", @@ -162,6 +166,18 @@ def _get_connection_settings( kwargs.pop("slaves", None) kwargs.pop("is_slave", None) + if "uuidRepresentation" not in kwargs: + warnings.warn( + "No uuidRepresentation is specified! Falling back to " + "'pythonLegacy' which is the default for pymongo 3.x. " + "For compatibility with other MongoDB drivers this should be " + "specified as 'standard' or '{java,csharp}Legacy' to work with " + "older drivers in those languages. This will be changed to " + "'standard' in a future release.", + DeprecationWarning, + ) + kwargs["uuidRepresentation"] = "pythonLegacy" + conn_settings.update(kwargs) return conn_settings @@ -263,15 +279,25 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): raise ConnectionFailure(msg) def _clean_settings(settings_dict): - irrelevant_fields_set = { - "name", - "username", - "password", - "authentication_source", - "authentication_mechanism", - } + if PYMONGO_VERSION < (4,): + irrelevant_fields_set = { + "name", + "username", + "password", + "authentication_source", + "authentication_mechanism", + } + rename_fields = {} + else: + irrelevant_fields_set = {"name"} + rename_fields = { + "authentication_source": "authSource", + "authentication_mechanism": "authMechanism", + } return { - k: v for k, v in settings_dict.items() if k not in irrelevant_fields_set + rename_fields.get(k, k): v + for k, v in settings_dict.items() + if k not in irrelevant_fields_set and v is not None } raw_conn_settings = _connection_settings[alias].copy() @@ -351,14 +377,18 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn = get_connection(alias) conn_settings = _connection_settings[alias] db = conn[conn_settings["name"]] - auth_kwargs = {"source": conn_settings["authentication_source"]} - if conn_settings["authentication_mechanism"] is not None: - auth_kwargs["mechanism"] = conn_settings["authentication_mechanism"] # Authenticate if necessary - if conn_settings["username"] and ( - conn_settings["password"] - or conn_settings["authentication_mechanism"] == "MONGODB-X509" + if ( + PYMONGO_VERSION < (4,) + and conn_settings["username"] + and ( + conn_settings["password"] + or conn_settings["authentication_mechanism"] == "MONGODB-X509" + ) ): + auth_kwargs = {"source": conn_settings["authentication_source"]} + if conn_settings["authentication_mechanism"] is not None: + auth_kwargs["mechanism"] = conn_settings["authentication_mechanism"] db.authenticate( conn_settings["username"], conn_settings["password"], **auth_kwargs ) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 0ca2622c5..eb9c99622 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -210,13 +210,14 @@ def __init__(self, alias=DEFAULT_CONNECTION_NAME): } def _turn_on_profiling(self): - self.initial_profiling_level = self.db.profiling_level() - self.db.set_profiling_level(0) + profile_update_res = self.db.command({"profile": 0}) + self.initial_profiling_level = profile_update_res["was"] + self.db.system.profile.drop() - self.db.set_profiling_level(2) + self.db.command({"profile": 2}) def _resets_profiling(self): - self.db.set_profiling_level(self.initial_profiling_level) + self.db.command({"profile": self.initial_profiling_level}) def __enter__(self): self._turn_on_profiling() diff --git a/mongoengine/document.py b/mongoengine/document.py index e56a9e7c2..5d6a47a0e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -888,10 +888,6 @@ def ensure_indexes(cls): index_cls = cls._meta.get("index_cls", True) collection = cls._get_collection() - # 746: when connection is via mongos, the read preference is not necessarily an indication that - # this code runs on a secondary - if not collection.is_mongos and collection.read_preference > 1: - return # determine if an index which we are creating includes # _cls as its first field; if so, we can avoid creating diff --git a/mongoengine/pymongo_support.py b/mongoengine/pymongo_support.py index dc7aaa6bc..fcddbf7b2 100644 --- a/mongoengine/pymongo_support.py +++ b/mongoengine/pymongo_support.py @@ -1,14 +1,18 @@ """ -Helper functions, constants, and types to aid with PyMongo v2.7 - v3.x support. +Helper functions, constants, and types to aid with PyMongo support. """ import pymongo +from bson import binary, json_util from pymongo.errors import OperationFailure -_PYMONGO_37 = (3, 7) - PYMONGO_VERSION = tuple(pymongo.version_tuple[:2]) -IS_PYMONGO_GTE_37 = PYMONGO_VERSION >= _PYMONGO_37 +if PYMONGO_VERSION >= (4,): + LEGACY_JSON_OPTIONS = json_util.LEGACY_JSON_OPTIONS.with_options( + uuid_representation=binary.UuidRepresentation.PYTHON_LEGACY, + ) +else: + LEGACY_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS def count_documents( @@ -29,15 +33,28 @@ def count_documents( kwargs["collation"] = collation # count_documents appeared in pymongo 3.7 - if IS_PYMONGO_GTE_37: + if PYMONGO_VERSION >= (3, 7): try: return collection.count_documents(filter=filter, **kwargs) - except OperationFailure: + except OperationFailure as err: + if PYMONGO_VERSION >= (4,): + raise + # OperationFailure - accounts for some operators that used to work # with .count but are no longer working with count_documents (i.e $geoNear, $near, and $nearSphere) # fallback to deprecated Cursor.count # Keeping this should be reevaluated the day pymongo removes .count entirely - pass + message = str(err) + if not ( + "not allowed in this context" in message + and ( + "$where" in message + or "$geoNear" in message + or "$near" in message + or "$nearSphere" in message + ) + ): + raise cursor = collection.find(filter) for option, option_value in kwargs.items(): @@ -49,7 +66,7 @@ def count_documents( def list_collection_names(db, include_system_collections=False): """Pymongo>3.7 deprecates collection_names in favour of list_collection_names""" - if IS_PYMONGO_GTE_37: + if PYMONGO_VERSION >= (3, 7): collections = db.list_collection_names() else: collections = db.collection_names() diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 98b4b66e4..d3efe3f74 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -28,7 +28,10 @@ NotUniqueError, OperationError, ) -from mongoengine.pymongo_support import count_documents +from mongoengine.pymongo_support import ( + LEGACY_JSON_OPTIONS, + count_documents, +) from mongoengine.queryset import transform from mongoengine.queryset.field_list import QueryFieldList from mongoengine.queryset.visitor import Q, QNode @@ -1266,6 +1269,15 @@ def max_time_ms(self, ms): def to_json(self, *args, **kwargs): """Converts a queryset to JSON""" + if "json_options" not in kwargs: + warnings.warn( + "No 'json_options' are specified! Falling back to " + "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " + "For use with other MongoDB drivers specify the UUID " + "representation to use.", + DeprecationWarning, + ) + kwargs["json_options"] = LEGACY_JSON_OPTIONS return json_util.dumps(self.as_pymongo(), *args, **kwargs) def from_json(self, json_data): @@ -1335,9 +1347,8 @@ def map_reduce( :param map_f: map function, as :class:`~bson.code.Code` or string :param reduce_f: reduce function, as :class:`~bson.code.Code` or string - :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options + :param output: output collection name, if set to 'inline' will return + the results inline. This can also be a dictionary containing output options see: http://docs.mongodb.org/manual/reference/command/mapReduce/#dbcmd.mapReduce :param finalize_f: finalize function, an optional function that performs any post-reduction processing. @@ -1347,12 +1358,6 @@ def map_reduce( Returns an iterator yielding :class:`~mongoengine.document.MapReduceDocument`. - - .. note:: - - Map/Reduce changed in server version **>= 1.7.4**. The PyMongo - :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.11**. """ queryset = self.clone() @@ -1389,10 +1394,10 @@ def map_reduce( mr_args["limit"] = limit if output == "inline" and not queryset._ordering: - map_reduce_function = "inline_map_reduce" + inline = True + mr_args["out"] = {"inline": 1} else: - map_reduce_function = "map_reduce" - + inline = False if isinstance(output, str): mr_args["out"] = output @@ -1422,17 +1427,29 @@ def map_reduce( mr_args["out"] = SON(ordered_output) - results = getattr(queryset._collection, map_reduce_function)( - map_f, reduce_f, **mr_args + db = queryset._document._get_db() + result = db.command( + { + "mapReduce": queryset._document._get_collection_name(), + "map": map_f, + "reduce": reduce_f, + **mr_args, + } ) - if map_reduce_function == "map_reduce": - results = results.find() + if inline: + docs = result["results"] + else: + if isinstance(result["result"], str): + docs = db[result["result"]].find() + else: + info = result["result"] + docs = db.client[info["db"]][info["collection"]].find() if queryset._ordering: - results = results.sort(queryset._ordering) + docs = docs.sort(queryset._ordering) - for doc in results: + for doc in docs: yield MapReduceDocument( queryset._document, queryset._collection, doc["_id"], doc["value"] ) @@ -1476,7 +1493,7 @@ def exec_js(self, code, *fields, **options): code = Code(code, scope=scope) db = queryset._document._get_db() - return db.eval(code, *fields) + return db.command("eval", code, args=fields).get("retval") def where(self, where_clause): """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript diff --git a/setup.py b/setup.py index bb777da0e..78e20331b 100644 --- a/setup.py +++ b/setup.py @@ -142,7 +142,7 @@ def run_tests(self): platforms=["any"], classifiers=CLASSIFIERS, python_requires=">=3.6", - install_requires=["pymongo>=3.4, <4.0"], + install_requires=["pymongo>=3.4,<=4.0"], cmdclass={"test": PyTest}, **extra_opts ) diff --git a/tests/document/test_dynamic.py b/tests/document/test_dynamic.py index 909c6f796..170b2ea3d 100644 --- a/tests/document/test_dynamic.py +++ b/tests/document/test_dynamic.py @@ -28,9 +28,9 @@ def test_simple_dynamic_document(self): p.age = 34 assert p.to_mongo() == {"_cls": "Person", "name": "James", "age": 34} - assert p.to_mongo().keys() == ["_cls", "name", "age"] + assert sorted(p.to_mongo().keys()) == ["_cls", "age", "name"] p.save() - assert p.to_mongo().keys() == ["_id", "_cls", "name", "age"] + assert sorted(p.to_mongo().keys()) == ["_cls", "_id", "age", "name"] assert self.Person.objects.first().age == 34 diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index e308d12a9..4d56f8553 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -11,6 +11,7 @@ MONGODB_42, get_mongodb_version, ) +from mongoengine.pymongo_support import PYMONGO_VERSION class TestIndexes(unittest.TestCase): @@ -247,10 +248,9 @@ class Place(Document): def test_explicit_geohaystack_index(self): """Ensure that geohaystack indexes work when created via meta[indexes]""" - pytest.skip( - "GeoHaystack index creation is not supported for now" - "from meta, as it requires a bucketSize parameter." - ) + # This test can be removed when pymongo 3.x is no longer supported + if PYMONGO_VERSION >= (4,): + pytest.skip("GEOHAYSTACK has been removed in pymongo 4.0") class Place(Document): location = DictField() @@ -261,10 +261,13 @@ class Place(Document): {"fields": [("location.point", "geoHaystack"), ("name", 1)]} ] == Place._meta["index_specs"] - Place.ensure_indexes() - info = Place._get_collection().index_information() - info = [value["key"] for key, value in info.items()] - assert [("location.point", "geoHaystack")] in info + # GeoHaystack index creation is not supported for now from meta, as it + # requires a bucketSize parameter. + if False: + Place.ensure_indexes() + info = Place._get_collection().index_information() + info = [value["key"] for key, value in info.items()] + assert [("location.point", "geoHaystack")] in info def test_create_geohaystack_index(self): """Ensure that geohaystack indexes can be created""" @@ -273,10 +276,25 @@ class Place(Document): location = DictField() name = StringField() - Place.create_index({"fields": (")location.point", "name")}, bucketSize=10) - info = Place._get_collection().index_information() - info = [value["key"] for key, value in info.items()] - assert [("location.point", "geoHaystack"), ("name", 1)] in info + if PYMONGO_VERSION >= (4,): + expected_error = NotImplementedError + elif get_mongodb_version() >= (4, 9): + expected_error = OperationFailure + else: + expected_error = None + + # This test can be removed when pymongo 3.x is no longer supported + if expected_error: + with pytest.raises(expected_error): + Place.create_index( + {"fields": (")location.point", "name")}, + bucketSize=10, + ) + else: + Place.create_index({"fields": (")location.point", "name")}, bucketSize=10) + info = Place._get_collection().index_information() + info = [value["key"] for key, value in info.items()] + assert [("location.point", "geoHaystack"), ("name", 1)] in info def test_dictionary_indexes(self): """Ensure that indexes are used when meta[indexes] contains @@ -549,7 +567,9 @@ class BlogPost(Document): incorrect_collation = {"arndom": "wrdo"} with pytest.raises(OperationFailure) as exc_info: BlogPost.objects.collation(incorrect_collation).count() - assert "Missing expected field" in str(exc_info.value) + assert "Missing expected field" in str( + exc_info.value + ) or "unknown field" in str(exc_info.value) query_result = BlogPost.objects.collation({}).order_by("name") assert [x.name for x in query_result] == sorted(names) diff --git a/tests/document/test_inheritance.py b/tests/document/test_inheritance.py index 550a4bdf3..09a207d5a 100644 --- a/tests/document/test_inheritance.py +++ b/tests/document/test_inheritance.py @@ -246,11 +246,15 @@ class Employee(Person): assert ["_cls", "age", "id", "name", "salary"] == sorted( Employee._fields.keys() ) - assert Person(name="Bob", age=35).to_mongo().keys() == ["_cls", "name", "age"] - assert Employee(name="Bob", age=35, salary=0).to_mongo().keys() == [ + assert sorted(Person(name="Bob", age=35).to_mongo().keys()) == [ "_cls", + "age", "name", + ] + assert sorted(Employee(name="Bob", age=35, salary=0).to_mongo().keys()) == [ + "_cls", "age", + "name", "salary", ] assert Employee._get_collection_name() == Person._get_collection_name() @@ -334,7 +338,7 @@ class Dog(Animal): # Check that _cls etc aren't present on simple documents dog = Animal(name="dog").save() - assert dog.to_mongo().keys() == ["_id", "name"] + assert sorted(dog.to_mongo().keys()) == ["_id", "name"] collection = self.db[Animal._get_collection_name()] obj = collection.find_one() diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index cd8e9b1b0..1f9a76b47 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -28,7 +28,10 @@ MONGODB_36, get_mongodb_version, ) -from mongoengine.pymongo_support import list_collection_names +from mongoengine.pymongo_support import ( + PYMONGO_VERSION, + list_collection_names, +) from mongoengine.queryset import NULLIFY, Q from tests import fixtures from tests.fixtures import ( @@ -704,11 +707,15 @@ class Person(EmbeddedDocument): class Employee(Person): salary = IntField() - assert Person(name="Bob", age=35).to_mongo().keys() == ["_cls", "name", "age"] - assert Employee(name="Bob", age=35, salary=0).to_mongo().keys() == [ + assert sorted(Person(name="Bob", age=35).to_mongo().keys()) == [ "_cls", + "age", "name", + ] + assert sorted(Employee(name="Bob", age=35, salary=0).to_mongo().keys()) == [ + "_cls", "age", + "name", "salary", ] @@ -717,7 +724,7 @@ class SubDoc(EmbeddedDocument): id = StringField(required=True) sub_doc = SubDoc(id="abc") - assert sub_doc.to_mongo().keys() == ["id"] + assert list(sub_doc.to_mongo().keys()) == ["id"] def test_embedded_document(self): """Ensure that embedded documents are set up correctly.""" @@ -2751,17 +2758,17 @@ class Person(Document): from pymongo.collection import Collection - orig_update = Collection.update + orig_update_one = Collection.update_one try: - def fake_update(*args, **kwargs): + def fake_update_one(*args, **kwargs): self.fail("Unexpected update for %s" % args[0].name) - return orig_update(*args, **kwargs) + return orig_update_one(*args, **kwargs) - Collection.update = fake_update + Collection.update_one = fake_update_one person.save() finally: - Collection.update = orig_update + Collection.update_one = orig_update_one def test_db_alias_tests(self): """DB Alias tests.""" @@ -2939,7 +2946,11 @@ def __str__(self): } ) assert [str(b) for b in custom_qs] == ["1", "2"] - assert custom_qs.count() == 2 + + # count only will work with this raw query before pymongo 4.x, but + # the length is also implicitly checked above + if PYMONGO_VERSION < (4,): + assert custom_qs.count() == 2 def test_switch_db_instance(self): register_connection("testdb-1", "mongoenginetest2") diff --git a/tests/queryset/test_geo.py b/tests/queryset/test_geo.py index 0fe3af97a..670cf38db 100644 --- a/tests/queryset/test_geo.py +++ b/tests/queryset/test_geo.py @@ -2,6 +2,7 @@ import unittest from mongoengine import * +from mongoengine.pymongo_support import PYMONGO_VERSION from tests.utils import MongoDBTestCase @@ -47,13 +48,15 @@ def test_near(self): # note that "near" will show the san francisco event, too, # although it sorts to last. events = self.Event.objects(location__near=[-87.67892, 41.9120459]) - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event1, event3, event2] # ensure ordering is respected by "near" events = self.Event.objects(location__near=[-87.67892, 41.9120459]) events = events.order_by("-date") - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event3, event1, event2] def test_near_and_max_distance(self): @@ -65,8 +68,9 @@ def test_near_and_max_distance(self): # find events within 10 degrees of san francisco point = [-122.415579, 37.7566023] events = self.Event.objects(location__near=point, location__max_distance=10) - assert events.count() == 1 - assert events[0] == event2 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event2] def test_near_and_min_distance(self): """Ensure the "min_distance" operator works alongside the "near" @@ -77,7 +81,9 @@ def test_near_and_min_distance(self): # find events at least 10 degrees away of san francisco point = [-122.415579, 37.7566023] events = self.Event.objects(location__near=point, location__min_distance=10) - assert events.count() == 2 + if PYMONGO_VERSION < (4,): + assert events.count() == 2 + assert list(events) == [event3, event1] def test_within_distance(self): """Make sure the "within_distance" operator works.""" @@ -153,13 +159,15 @@ def test_2dsphere_near(self): # note that "near" will show the san francisco event, too, # although it sorts to last. events = self.Event.objects(location__near=[-87.67892, 41.9120459]) - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event1, event3, event2] # ensure ordering is respected by "near" events = self.Event.objects(location__near=[-87.67892, 41.9120459]) events = events.order_by("-date") - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event3, event1, event2] def test_2dsphere_near_and_max_distance(self): @@ -171,21 +179,25 @@ def test_2dsphere_near_and_max_distance(self): # find events within 10km of san francisco point = [-122.415579, 37.7566023] events = self.Event.objects(location__near=point, location__max_distance=10000) - assert events.count() == 1 - assert events[0] == event2 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event2] # find events within 1km of greenpoint, broolyn, nyc, ny events = self.Event.objects( location__near=[-73.9509714, 40.7237134], location__max_distance=1000 ) - assert events.count() == 0 + if PYMONGO_VERSION < (4,): + assert events.count() == 0 + assert list(events) == [] # ensure ordering is respected by "near" events = self.Event.objects( location__near=[-87.67892, 41.9120459], location__max_distance=10000 ).order_by("-date") - assert events.count() == 2 - assert events[0] == event3 + if PYMONGO_VERSION < (4,): + assert events.count() == 2 + assert list(events) == [event3, event1] def test_2dsphere_geo_within_box(self): """Ensure the "geo_within_box" operator works with a 2dsphere @@ -236,15 +248,17 @@ def test_2dsphere_near_and_min_max_distance(self): location__min_distance=1000, location__max_distance=10000, ).order_by("-date") - assert events.count() == 1 - assert events[0] == event3 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event3] # ensure ordering is respected by "near" with "min_distance" events = self.Event.objects( location__near=[-87.67892, 41.9120459], location__min_distance=10000 ).order_by("-date") - assert events.count() == 1 - assert events[0] == event2 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event2] def test_2dsphere_geo_within_center(self): """Make sure the "geo_within_center" operator works with a @@ -289,7 +303,8 @@ class Event(Document): # note that "near" will show the san francisco event, too, # although it sorts to last. events = Event.objects(venue__location__near=[-87.67892, 41.9120459]) - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event1, event3, event2] def test_geo_spatial_embedded(self): @@ -318,7 +333,9 @@ class Point(Document): # Finds both points because they are within 60 km of the reference # point equidistant between them. points = Point.objects(location__near_sphere=[-122, 37.5]) - assert points.count() == 2 + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [north_point, south_point] # Same behavior for _within_spherical_distance points = Point.objects( @@ -329,36 +346,42 @@ class Point(Document): points = Point.objects( location__near_sphere=[-122, 37.5], location__max_distance=60 / earth_radius ) - assert points.count() == 2 + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [north_point, south_point] # Test query works with max_distance, being farer from one point points = Point.objects( location__near_sphere=[-122, 37.8], location__max_distance=60 / earth_radius ) close_point = points.first() - assert points.count() == 1 + if PYMONGO_VERSION < (4,): + assert points.count() == 1 + assert list(points) == [north_point] # Test query works with min_distance, being farer from one point points = Point.objects( location__near_sphere=[-122, 37.8], location__min_distance=60 / earth_radius ) - assert points.count() == 1 + if PYMONGO_VERSION < (4,): + assert points.count() == 1 far_point = points.first() + assert list(points) == [south_point] assert close_point != far_point # Finds both points, but orders the north point first because it's # closer to the reference point to the north. points = Point.objects(location__near_sphere=[-122, 38.5]) - assert points.count() == 2 - assert points[0].id == north_point.id - assert points[1].id == south_point.id + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [north_point, south_point] # Finds both points, but orders the south point first because it's # closer to the reference point to the south. points = Point.objects(location__near_sphere=[-122, 36.5]) - assert points.count() == 2 - assert points[0].id == south_point.id - assert points[1].id == north_point.id + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [south_point, north_point] # Finds only one point because only the first point is within 60km of # the reference point to the south. @@ -375,56 +398,72 @@ class Road(Document): Road.drop_collection() - Road(name="66", line=[[40, 5], [41, 6]]).save() + road = Road(name="66", line=[[40, 5], [41, 6]]) + road.save() # near point = {"type": "Point", "coordinates": [40, 5]} - roads = Road.objects.filter(line__near=point["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__near=point["coordinates"]) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__near=point).count() - assert 1 == roads + roads = Road.objects.filter(line__near=point) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__near={"$geometry": point}).count() - assert 1 == roads + roads = Road.objects.filter(line__near={"$geometry": point}) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] # Within polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(line__geo_within=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_within=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_within=polygon).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_within=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_within={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_within={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] # Intersects line = {"type": "LineString", "coordinates": [[40, 5], [40, 6]]} - roads = Road.objects.filter(line__geo_intersects=line["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=line["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects=line).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=line) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects={"$geometry": line}).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects={"$geometry": line}) + assert roads.count() == 1 + assert list(roads) == [road] polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(line__geo_intersects=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects=polygon).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] def test_polygon(self): class Road(Document): @@ -433,56 +472,72 @@ class Road(Document): Road.drop_collection() - Road(name="66", poly=[[[40, 5], [40, 6], [41, 6], [40, 5]]]).save() + road = Road(name="66", poly=[[[40, 5], [40, 6], [41, 6], [40, 5]]]) + road.save() # near point = {"type": "Point", "coordinates": [40, 5]} - roads = Road.objects.filter(poly__near=point["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__near=point["coordinates"]) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__near=point).count() - assert 1 == roads + roads = Road.objects.filter(poly__near=point) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__near={"$geometry": point}).count() - assert 1 == roads + roads = Road.objects.filter(poly__near={"$geometry": point}) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] # Within polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(poly__geo_within=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_within=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_within=polygon).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_within=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_within={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_within={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] # Intersects line = {"type": "LineString", "coordinates": [[40, 5], [41, 6]]} - roads = Road.objects.filter(poly__geo_intersects=line["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=line["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects=line).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=line) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects={"$geometry": line}).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects={"$geometry": line}) + assert roads.count() == 1 + assert list(roads) == [road] polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(poly__geo_intersects=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects=polygon).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] def test_aspymongo_with_only(self): """Ensure as_pymongo works with only""" diff --git a/tests/test_connection.py b/tests/test_connection.py index 2b7d46a78..a9c56d63b 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -23,6 +23,7 @@ get_connection, get_db, ) +from mongoengine.pymongo_support import PYMONGO_VERSION def get_tz_awareness(connection): @@ -350,8 +351,14 @@ def test_connect_uri(self): c.mongoenginetest.system.users.delete_many({}) c.admin.command("createUser", "admin", pwd="password", roles=["root"]) - c.admin.authenticate("admin", "password") - c.admin.command("createUser", "username", pwd="password", roles=["dbOwner"]) + + adminadmin_settings = mongoengine.connection._connection_settings[ + "adminadmin" + ] = mongoengine.connection._connection_settings["admin"].copy() + adminadmin_settings["username"] = "admin" + adminadmin_settings["password"] = "password" + ca = connect(db="mongoenginetest", alias="adminadmin") + ca.admin.command("createUser", "username", pwd="password", roles=["dbOwner"]) connect( "testdb_uri", host="mongodb://username:password@localhost/mongoenginetest" @@ -404,8 +411,14 @@ def test_uri_without_credentials_doesnt_override_conn_settings(self): # OperationFailure means that mongoengine attempted authentication # w/ the provided username/password and failed - that's the desired # behavior. If the MongoDB URI would override the credentials - with pytest.raises(OperationFailure): - get_db() + if PYMONGO_VERSION >= (4,): + with pytest.raises(OperationFailure): + db = get_db() + # pymongo 4.x does not call db.authenticate and needs to perform an operation to trigger the failure + db.list_collection_names() + else: + with pytest.raises(OperationFailure): + db = get_db() def test_connect_uri_with_authsource(self): """Ensure that the connect() method works well with `authSource` @@ -482,7 +495,10 @@ def test_connection_pool_via_kwarg(self): conn = connect( "mongoenginetest", alias="max_pool_size_via_kwarg", **pool_size_kwargs ) - assert conn.max_pool_size == 100 + if PYMONGO_VERSION >= (4,): + assert conn.options.pool_options.max_pool_size == 100 + else: + assert conn.max_pool_size == 100 def test_connection_pool_via_uri(self): """Ensure we can specify a max connection pool size using @@ -492,7 +508,10 @@ def test_connection_pool_via_uri(self): host="mongodb://localhost/test?maxpoolsize=100", alias="max_pool_size_via_uri", ) - assert conn.max_pool_size == 100 + if PYMONGO_VERSION >= (4,): + assert conn.options.pool_options.max_pool_size == 100 + else: + assert conn.max_pool_size == 100 def test_write_concern(self): """Ensure write concern can be specified in connect() via @@ -567,8 +586,8 @@ def test_connect_2_databases_uses_same_client_if_only_dbname_differs(self): assert c1 is c2 def test_connect_2_databases_uses_different_client_if_different_parameters(self): - c1 = connect(alias="testdb1", db="testdb1", username="u1") - c2 = connect(alias="testdb2", db="testdb2", username="u2") + c1 = connect(alias="testdb1", db="testdb1", username="u1", password="pass") + c2 = connect(alias="testdb2", db="testdb2", username="u2", password="pass") assert c1 is not c2 diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index 4ea7fa5ab..68dc8f4be 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -269,17 +269,23 @@ def test_query_counter_temporarily_modifies_profiling_level(self): connect("mongoenginetest") db = get_db() - initial_profiling_level = db.profiling_level() + def _current_profiling_level(): + return db.command({"profile": -1})["was"] + + def _set_profiling_level(lvl): + db.command({"profile": lvl}) + + initial_profiling_level = _current_profiling_level() try: new_level = 1 - db.set_profiling_level(new_level) - assert db.profiling_level() == new_level + _set_profiling_level(new_level) + assert _current_profiling_level() == new_level with query_counter(): - assert db.profiling_level() == 2 - assert db.profiling_level() == new_level + assert _current_profiling_level() == 2 + assert _current_profiling_level() == new_level except Exception: - db.set_profiling_level( + _set_profiling_level( initial_profiling_level ) # Ensures it gets reseted no matter the outcome of the test raise diff --git a/tox.ini b/tox.ini index 9925787b2..783e9f9ec 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pypy3-{mg34,mg36,mg39,mg311,mg312} +envlist = pypy3-{mg34,mg36,mg39,mg311,mg312,mg4} [testenv] commands = @@ -10,5 +10,6 @@ deps = mg39: pymongo>=3.9,<3.10 mg311: pymongo>=3.11,<3.12 mg312: pymongo>=3.12,<3.13 + mg4: pymongo>=4.0,<4.1 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs