From 9711df0d4b601981bf639e53a2c893557b7c0db7 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 1 Dec 2021 22:53:29 +0100 Subject: [PATCH 01/17] initial work for supporting recent pymongo 4.0 --- mongoengine/document.py | 4 ---- setup.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mongoengine/document.py b/mongoengine/document.py index e56a9e7c2..5d6a47a0e 100644 --- a/mongoengine/document.py +++ b/mongoengine/document.py @@ -888,10 +888,6 @@ def ensure_indexes(cls): index_cls = cls._meta.get("index_cls", True) collection = cls._get_collection() - # 746: when connection is via mongos, the read preference is not necessarily an indication that - # this code runs on a secondary - if not collection.is_mongos and collection.read_preference > 1: - return # determine if an index which we are creating includes # _cls as its first field; if so, we can avoid creating diff --git a/setup.py b/setup.py index bb777da0e..78e20331b 100644 --- a/setup.py +++ b/setup.py @@ -142,7 +142,7 @@ def run_tests(self): platforms=["any"], classifiers=CLASSIFIERS, python_requires=">=3.6", - install_requires=["pymongo>=3.4, <4.0"], + install_requires=["pymongo>=3.4,<=4.0"], cmdclass={"test": PyTest}, **extra_opts ) From 6067dde61d6104ec602966acae745a7854388be3 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 1 Dec 2021 22:56:58 +0100 Subject: [PATCH 02/17] update ci --- .github/workflows/github-actions.yml | 4 ++++ tox.ini | 1 + 2 files changed, 5 insertions(+) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 66533c13d..d75bdb1a0 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -21,6 +21,7 @@ env: PYMONGO_3_9: 3.9 PYMONGO_3_11: 3.11 PYMONGO_3_12: 3.12 + PYMONGO_4_0: 4.0 MAIN_PYTHON_VERSION: 3.7 @@ -61,6 +62,9 @@ jobs: - python-version: 3.7 MONGODB: $MONGODB_4_4 PYMONGO: $PYMONGO_3_12 + - python-version: 3.9 + MONGODB: $MONGODB_4_4 + PYMONGO: $PYMONGO_4_0 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} diff --git a/tox.ini b/tox.ini index 9925787b2..98c367b44 100644 --- a/tox.ini +++ b/tox.ini @@ -10,5 +10,6 @@ deps = mg39: pymongo>=3.9,<3.10 mg311: pymongo>=3.11,<3.12 mg312: pymongo>=3.12,<3.13 + mg4: pymongo>=4.0,<4.1 setenv = PYTHON_EGG_CACHE = {envdir}/python-eggs From 1f5d2f3a9f88e397808fac560b6c69fe92ff9613 Mon Sep 17 00:00:00 2001 From: Bastien Gerard Date: Wed, 1 Dec 2021 23:24:55 +0100 Subject: [PATCH 03/17] fix set_profiling_level with db.command() --- mongoengine/context_managers.py | 9 +++++---- tests/test_context_managers.py | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mongoengine/context_managers.py b/mongoengine/context_managers.py index 0ca2622c5..eb9c99622 100644 --- a/mongoengine/context_managers.py +++ b/mongoengine/context_managers.py @@ -210,13 +210,14 @@ def __init__(self, alias=DEFAULT_CONNECTION_NAME): } def _turn_on_profiling(self): - self.initial_profiling_level = self.db.profiling_level() - self.db.set_profiling_level(0) + profile_update_res = self.db.command({"profile": 0}) + self.initial_profiling_level = profile_update_res["was"] + self.db.system.profile.drop() - self.db.set_profiling_level(2) + self.db.command({"profile": 2}) def _resets_profiling(self): - self.db.set_profiling_level(self.initial_profiling_level) + self.db.command({"profile": self.initial_profiling_level}) def __enter__(self): self._turn_on_profiling() diff --git a/tests/test_context_managers.py b/tests/test_context_managers.py index 4ea7fa5ab..68dc8f4be 100644 --- a/tests/test_context_managers.py +++ b/tests/test_context_managers.py @@ -269,17 +269,23 @@ def test_query_counter_temporarily_modifies_profiling_level(self): connect("mongoenginetest") db = get_db() - initial_profiling_level = db.profiling_level() + def _current_profiling_level(): + return db.command({"profile": -1})["was"] + + def _set_profiling_level(lvl): + db.command({"profile": lvl}) + + initial_profiling_level = _current_profiling_level() try: new_level = 1 - db.set_profiling_level(new_level) - assert db.profiling_level() == new_level + _set_profiling_level(new_level) + assert _current_profiling_level() == new_level with query_counter(): - assert db.profiling_level() == 2 - assert db.profiling_level() == new_level + assert _current_profiling_level() == 2 + assert _current_profiling_level() == new_level except Exception: - db.set_profiling_level( + _set_profiling_level( initial_profiling_level ) # Ensures it gets reseted no matter the outcome of the test raise From eddaf51fde8b2f85fb58c1f7da16b05bbbe768a4 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Wed, 5 Jan 2022 16:55:56 -0800 Subject: [PATCH 04/17] feat: handle geohaystack removal --- mongoengine/base/document.py | 18 ++++++++++++----- mongoengine/pymongo_support.py | 8 ++------ tests/document/test_indexes.py | 37 +++++++++++++++++++++------------- tox.ini | 2 +- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 46935c1b8..88386510c 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -28,6 +28,11 @@ NON_FIELD_ERRORS = "__all__" +try: + GEOHAYSTACK = pymongo.GEOHAYSTACK +except AttributeError: + GEOHAYSTACK = None + class BaseDocument: # TODO simplify how `_changed_fields` is used. @@ -898,7 +903,10 @@ def _build_index_spec(cls, spec): elif key.startswith("("): direction = pymongo.GEOSPHERE elif key.startswith(")"): - direction = pymongo.GEOHAYSTACK + try: + direction = pymongo.GEOHAYSTACK + except AttributeError: + raise NotImplementedError elif key.startswith("*"): direction = pymongo.GEO2D if key.startswith(("+", "-", "*", "$", "#", "(", ")")): @@ -923,10 +931,10 @@ def _build_index_spec(cls, spec): index_list.append((key, direction)) # Don't add cls to a geo index - if include_cls and direction not in ( - pymongo.GEO2D, - pymongo.GEOHAYSTACK, - pymongo.GEOSPHERE, + if ( + include_cls + and direction not in (pymongo.GEO2D, pymongo.GEOSPHERE) + and (GEOHAYSTACK is None or direction != GEOHAYSTACK) ): index_list.insert(0, ("_cls", 1)) diff --git a/mongoengine/pymongo_support.py b/mongoengine/pymongo_support.py index dc7aaa6bc..05c9e094d 100644 --- a/mongoengine/pymongo_support.py +++ b/mongoengine/pymongo_support.py @@ -4,12 +4,8 @@ import pymongo from pymongo.errors import OperationFailure -_PYMONGO_37 = (3, 7) - PYMONGO_VERSION = tuple(pymongo.version_tuple[:2]) -IS_PYMONGO_GTE_37 = PYMONGO_VERSION >= _PYMONGO_37 - def count_documents( collection, filter, skip=None, limit=None, hint=None, collation=None @@ -29,7 +25,7 @@ def count_documents( kwargs["collation"] = collation # count_documents appeared in pymongo 3.7 - if IS_PYMONGO_GTE_37: + if PYMONGO_VERSION >= (3, 7): try: return collection.count_documents(filter=filter, **kwargs) except OperationFailure: @@ -49,7 +45,7 @@ def count_documents( def list_collection_names(db, include_system_collections=False): """Pymongo>3.7 deprecates collection_names in favour of list_collection_names""" - if IS_PYMONGO_GTE_37: + if PYMONGO_VERSION >= (3, 7): collections = db.list_collection_names() else: collections = db.collection_names() diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index e308d12a9..452aa9a4a 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -11,6 +11,7 @@ MONGODB_42, get_mongodb_version, ) +from mongoengine.pymongo_support import PYMONGO_VERSION class TestIndexes(unittest.TestCase): @@ -247,11 +248,9 @@ class Place(Document): def test_explicit_geohaystack_index(self): """Ensure that geohaystack indexes work when created via meta[indexes]""" - pytest.skip( - "GeoHaystack index creation is not supported for now" - "from meta, as it requires a bucketSize parameter." - ) - + # This test can be removed when pymongo 3.x is no longer supported + if PYMONGO_VERSION >= (4,): + pytest.skip('GEOHAYSTACK has been removed in pymongo 4.0') class Place(Document): location = DictField() name = StringField() @@ -261,22 +260,32 @@ class Place(Document): {"fields": [("location.point", "geoHaystack"), ("name", 1)]} ] == Place._meta["index_specs"] - Place.ensure_indexes() - info = Place._get_collection().index_information() - info = [value["key"] for key, value in info.items()] - assert [("location.point", "geoHaystack")] in info + # GeoHaystack index creation is not supported for now from meta, as it + # requires a bucketSize parameter. + if False: + Place.ensure_indexes() + info = Place._get_collection().index_information() + info = [value["key"] for key, value in info.items()] + assert [("location.point", "geoHaystack")] in info def test_create_geohaystack_index(self): """Ensure that geohaystack indexes can be created""" - class Place(Document): location = DictField() name = StringField() - Place.create_index({"fields": (")location.point", "name")}, bucketSize=10) - info = Place._get_collection().index_information() - info = [value["key"] for key, value in info.items()] - assert [("location.point", "geoHaystack"), ("name", 1)] in info + # This test can be removed when pymongo 3.x is no longer supported + if PYMONGO_VERSION >= (4,): + with pytest.raises(NotImplementedError): + Place.create_index( + {"fields": (")location.point", "name")}, + bucketSize=10, + ) + else: + Place.create_index({"fields": (")location.point", "name")}, bucketSize=10) + info = Place._get_collection().index_information() + info = [value["key"] for key, value in info.items()] + assert [("location.point", "geoHaystack"), ("name", 1)] in info def test_dictionary_indexes(self): """Ensure that indexes are used when meta[indexes] contains diff --git a/tox.ini b/tox.ini index 98c367b44..783e9f9ec 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pypy3-{mg34,mg36,mg39,mg311,mg312} +envlist = pypy3-{mg34,mg36,mg39,mg311,mg312,mg4} [testenv] commands = From da4488eb008bb81b96d62a10307aa2426f6bd4b3 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Wed, 5 Jan 2022 17:28:46 -0800 Subject: [PATCH 05/17] feat: handle testing test_indexes failures with MongoDB > 4.4 --- tests/document/test_indexes.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/document/test_indexes.py b/tests/document/test_indexes.py index 452aa9a4a..4d56f8553 100644 --- a/tests/document/test_indexes.py +++ b/tests/document/test_indexes.py @@ -250,7 +250,8 @@ def test_explicit_geohaystack_index(self): """Ensure that geohaystack indexes work when created via meta[indexes]""" # This test can be removed when pymongo 3.x is no longer supported if PYMONGO_VERSION >= (4,): - pytest.skip('GEOHAYSTACK has been removed in pymongo 4.0') + pytest.skip("GEOHAYSTACK has been removed in pymongo 4.0") + class Place(Document): location = DictField() name = StringField() @@ -270,13 +271,21 @@ class Place(Document): def test_create_geohaystack_index(self): """Ensure that geohaystack indexes can be created""" + class Place(Document): location = DictField() name = StringField() - # This test can be removed when pymongo 3.x is no longer supported if PYMONGO_VERSION >= (4,): - with pytest.raises(NotImplementedError): + expected_error = NotImplementedError + elif get_mongodb_version() >= (4, 9): + expected_error = OperationFailure + else: + expected_error = None + + # This test can be removed when pymongo 3.x is no longer supported + if expected_error: + with pytest.raises(expected_error): Place.create_index( {"fields": (")location.point", "name")}, bucketSize=10, @@ -558,7 +567,9 @@ class BlogPost(Document): incorrect_collation = {"arndom": "wrdo"} with pytest.raises(OperationFailure) as exc_info: BlogPost.objects.collation(incorrect_collation).count() - assert "Missing expected field" in str(exc_info.value) + assert "Missing expected field" in str( + exc_info.value + ) or "unknown field" in str(exc_info.value) query_result = BlogPost.objects.collation({}).order_by("name") assert [x.name for x in query_result] == sorted(names) From 8790fd885d3efd3af1cd3ed632c89ec92e2f7e0b Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Wed, 5 Jan 2022 17:52:03 -0800 Subject: [PATCH 06/17] tests: update max_pool_size moving to options.pool_options --- tests/test_connection.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 2b7d46a78..c2034518a 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -23,6 +23,7 @@ get_connection, get_db, ) +from mongoengine.pymongo_support import PYMONGO_VERSION def get_tz_awareness(connection): @@ -482,7 +483,10 @@ def test_connection_pool_via_kwarg(self): conn = connect( "mongoenginetest", alias="max_pool_size_via_kwarg", **pool_size_kwargs ) - assert conn.max_pool_size == 100 + if PYMONGO_VERSION >= (4,): + assert conn.options.pool_options.max_pool_size == 100 + else: + assert conn.max_pool_size == 100 def test_connection_pool_via_uri(self): """Ensure we can specify a max connection pool size using @@ -492,7 +496,10 @@ def test_connection_pool_via_uri(self): host="mongodb://localhost/test?maxpoolsize=100", alias="max_pool_size_via_uri", ) - assert conn.max_pool_size == 100 + if PYMONGO_VERSION >= (4,): + assert conn.options.pool_options.max_pool_size == 100 + else: + assert conn.max_pool_size == 100 def test_write_concern(self): """Ensure write concern can be specified in connect() via From db51d8a03ee48fa27f026e9b1f96008485461979 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 01:19:22 -0800 Subject: [PATCH 07/17] feat: update map_reduce to use db.command --- mongoengine/queryset/base.py | 41 ++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 98b4b66e4..bf05ea6c7 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1335,9 +1335,8 @@ def map_reduce( :param map_f: map function, as :class:`~bson.code.Code` or string :param reduce_f: reduce function, as :class:`~bson.code.Code` or string - :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options + :param output: output collection name, if set to 'inline' will return + the results inline. This can also be a dictionary containing output options see: http://docs.mongodb.org/manual/reference/command/mapReduce/#dbcmd.mapReduce :param finalize_f: finalize function, an optional function that performs any post-reduction processing. @@ -1347,12 +1346,6 @@ def map_reduce( Returns an iterator yielding :class:`~mongoengine.document.MapReduceDocument`. - - .. note:: - - Map/Reduce changed in server version **>= 1.7.4**. The PyMongo - :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.11**. """ queryset = self.clone() @@ -1389,10 +1382,10 @@ def map_reduce( mr_args["limit"] = limit if output == "inline" and not queryset._ordering: - map_reduce_function = "inline_map_reduce" + inline = True + mr_args["out"] = {"inline": 1} else: - map_reduce_function = "map_reduce" - + inline = False if isinstance(output, str): mr_args["out"] = output @@ -1422,17 +1415,29 @@ def map_reduce( mr_args["out"] = SON(ordered_output) - results = getattr(queryset._collection, map_reduce_function)( - map_f, reduce_f, **mr_args + db = queryset._document._get_db() + result = db.command( + { + "mapReduce": queryset._document._get_collection_name(), + "map": map_f, + "reduce": reduce_f, + **mr_args, + } ) - if map_reduce_function == "map_reduce": - results = results.find() + if inline: + docs = result["results"] + else: + if isinstance(result["result"], str): + docs = db[result["result"]].find() + else: + info = result["result"] + docs = db.client[info["db"]][info["collection"]].find() if queryset._ordering: - results = results.sort(queryset._ordering) + docs = docs.sort(queryset._ordering) - for doc in results: + for doc in docs: yield MapReduceDocument( queryset._document, queryset._collection, doc["_id"], doc["value"] ) From 56bba1afc6b7ac39277c23019e3edf99123d4117 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 10:29:56 -0800 Subject: [PATCH 08/17] feat: handle eval removal --- mongoengine/queryset/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index bf05ea6c7..1b1dba8f0 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -1481,7 +1481,7 @@ def exec_js(self, code, *fields, **options): code = Code(code, scope=scope) db = queryset._document._get_db() - return db.eval(code, *fields) + return db.command("eval", code, args=fields).get("retval") def where(self, where_clause): """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript From f930aabe223b6d794956ee32ff70a98e1e15ed48 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 11:34:45 -0800 Subject: [PATCH 09/17] feat: use legacy python UUID encoding by default if unspecified --- mongoengine/base/document.py | 24 ++++++++++++++++++++++-- mongoengine/connection.py | 14 ++++++++++++++ mongoengine/pymongo_support.py | 7 ++++++- mongoengine/queryset/base.py | 14 +++++++++++++- 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py index 88386510c..7a1de50e3 100644 --- a/mongoengine/base/document.py +++ b/mongoengine/base/document.py @@ -1,5 +1,6 @@ import copy import numbers +import warnings from functools import partial import pymongo @@ -23,6 +24,7 @@ OperationError, ValidationError, ) +from mongoengine.pymongo_support import LEGACY_JSON_OPTIONS __all__ = ("BaseDocument", "NON_FIELD_ERRORS") @@ -444,10 +446,19 @@ def to_json(self, *args, **kwargs): Defaults to True. """ use_db_field = kwargs.pop("use_db_field", True) + if "json_options" not in kwargs: + warnings.warn( + "No 'json_options' are specified! Falling back to " + "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " + "For use with other MongoDB drivers specify the UUID " + "representation to use.", + DeprecationWarning, + ) + kwargs["json_options"] = LEGACY_JSON_OPTIONS return json_util.dumps(self.to_mongo(use_db_field), *args, **kwargs) @classmethod - def from_json(cls, json_data, created=False): + def from_json(cls, json_data, created=False, **kwargs): """Converts json data to a Document instance :param str json_data: The json data to load into the Document @@ -465,7 +476,16 @@ def from_json(cls, json_data, created=False): # TODO should `created` default to False? If the object already exists # in the DB, you would likely retrieve it from MongoDB itself through # a query, not load it from JSON data. - return cls._from_son(json_util.loads(json_data), created=created) + if "json_options" not in kwargs: + warnings.warn( + "No 'json_options' are specified! Falling back to " + "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " + "For use with other MongoDB drivers specify the UUID " + "representation to use.", + DeprecationWarning, + ) + kwargs["json_options"] = LEGACY_JSON_OPTIONS + return cls._from_son(json_util.loads(json_data, **kwargs), created=created) def __expand_dynamic_values(self, name, value): """Expand any dynamic values to their correct types / values.""" diff --git a/mongoengine/connection.py b/mongoengine/connection.py index e9078a2e7..522826165 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -1,3 +1,5 @@ +import warnings + from pymongo import MongoClient, ReadPreference, uri_parser from pymongo.database import _check_name @@ -162,6 +164,18 @@ def _get_connection_settings( kwargs.pop("slaves", None) kwargs.pop("is_slave", None) + if "uuidRepresentation" not in kwargs: + warnings.warn( + "No uuidRepresentation is specified! Falling back to " + "'pythonLegacy' which is the default for pymongo 3.x. " + "For compatibility with other MongoDB drivers this should be " + "specified as 'standard' or '{java,csharp}Legacy' to work with " + "older drivers in those languages. This will be changed to " + "'standard' in a future release.", + DeprecationWarning, + ) + kwargs["uuidRepresentation"] = "pythonLegacy" + conn_settings.update(kwargs) return conn_settings diff --git a/mongoengine/pymongo_support.py b/mongoengine/pymongo_support.py index 05c9e094d..f3f9dea79 100644 --- a/mongoengine/pymongo_support.py +++ b/mongoengine/pymongo_support.py @@ -1,11 +1,16 @@ """ -Helper functions, constants, and types to aid with PyMongo v2.7 - v3.x support. +Helper functions, constants, and types to aid with PyMongo support. """ import pymongo +from bson import binary, json_util from pymongo.errors import OperationFailure PYMONGO_VERSION = tuple(pymongo.version_tuple[:2]) +LEGACY_JSON_OPTIONS = json_util.LEGACY_JSON_OPTIONS.with_options( + uuid_representation=binary.UuidRepresentation.PYTHON_LEGACY, +) + def count_documents( collection, filter, skip=None, limit=None, hint=None, collation=None diff --git a/mongoengine/queryset/base.py b/mongoengine/queryset/base.py index 1b1dba8f0..d3efe3f74 100644 --- a/mongoengine/queryset/base.py +++ b/mongoengine/queryset/base.py @@ -28,7 +28,10 @@ NotUniqueError, OperationError, ) -from mongoengine.pymongo_support import count_documents +from mongoengine.pymongo_support import ( + LEGACY_JSON_OPTIONS, + count_documents, +) from mongoengine.queryset import transform from mongoengine.queryset.field_list import QueryFieldList from mongoengine.queryset.visitor import Q, QNode @@ -1266,6 +1269,15 @@ def max_time_ms(self, ms): def to_json(self, *args, **kwargs): """Converts a queryset to JSON""" + if "json_options" not in kwargs: + warnings.warn( + "No 'json_options' are specified! Falling back to " + "LEGACY_JSON_OPTIONS with uuid_representation=PYTHON_LEGACY. " + "For use with other MongoDB drivers specify the UUID " + "representation to use.", + DeprecationWarning, + ) + kwargs["json_options"] = LEGACY_JSON_OPTIONS return json_util.dumps(self.as_pymongo(), *args, **kwargs) def from_json(self, json_data): From e071bd19fefebafc5f60769124120d4c3426c6d8 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 12:22:32 -0800 Subject: [PATCH 10/17] tests: update for SON.keys() no longer returning lists --- tests/document/test_dynamic.py | 4 ++-- tests/document/test_inheritance.py | 10 +++++++--- tests/document/test_instance.py | 10 +++++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/document/test_dynamic.py b/tests/document/test_dynamic.py index 909c6f796..170b2ea3d 100644 --- a/tests/document/test_dynamic.py +++ b/tests/document/test_dynamic.py @@ -28,9 +28,9 @@ def test_simple_dynamic_document(self): p.age = 34 assert p.to_mongo() == {"_cls": "Person", "name": "James", "age": 34} - assert p.to_mongo().keys() == ["_cls", "name", "age"] + assert sorted(p.to_mongo().keys()) == ["_cls", "age", "name"] p.save() - assert p.to_mongo().keys() == ["_id", "_cls", "name", "age"] + assert sorted(p.to_mongo().keys()) == ["_cls", "_id", "age", "name"] assert self.Person.objects.first().age == 34 diff --git a/tests/document/test_inheritance.py b/tests/document/test_inheritance.py index 550a4bdf3..09a207d5a 100644 --- a/tests/document/test_inheritance.py +++ b/tests/document/test_inheritance.py @@ -246,11 +246,15 @@ class Employee(Person): assert ["_cls", "age", "id", "name", "salary"] == sorted( Employee._fields.keys() ) - assert Person(name="Bob", age=35).to_mongo().keys() == ["_cls", "name", "age"] - assert Employee(name="Bob", age=35, salary=0).to_mongo().keys() == [ + assert sorted(Person(name="Bob", age=35).to_mongo().keys()) == [ "_cls", + "age", "name", + ] + assert sorted(Employee(name="Bob", age=35, salary=0).to_mongo().keys()) == [ + "_cls", "age", + "name", "salary", ] assert Employee._get_collection_name() == Person._get_collection_name() @@ -334,7 +338,7 @@ class Dog(Animal): # Check that _cls etc aren't present on simple documents dog = Animal(name="dog").save() - assert dog.to_mongo().keys() == ["_id", "name"] + assert sorted(dog.to_mongo().keys()) == ["_id", "name"] collection = self.db[Animal._get_collection_name()] obj = collection.find_one() diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index cd8e9b1b0..3dabaabca 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -704,11 +704,15 @@ class Person(EmbeddedDocument): class Employee(Person): salary = IntField() - assert Person(name="Bob", age=35).to_mongo().keys() == ["_cls", "name", "age"] - assert Employee(name="Bob", age=35, salary=0).to_mongo().keys() == [ + assert sorted(Person(name="Bob", age=35).to_mongo().keys()) == [ "_cls", + "age", "name", + ] + assert sorted(Employee(name="Bob", age=35, salary=0).to_mongo().keys()) == [ + "_cls", "age", + "name", "salary", ] @@ -717,7 +721,7 @@ class SubDoc(EmbeddedDocument): id = StringField(required=True) sub_doc = SubDoc(id="abc") - assert sub_doc.to_mongo().keys() == ["id"] + assert list(sub_doc.to_mongo().keys()) == ["id"] def test_embedded_document(self): """Ensure that embedded documents are set up correctly.""" From f31199a773ba549d0ca7e68aad035356f5c163ba Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 12:37:28 -0800 Subject: [PATCH 11/17] tests: fix monkeypatching removed collection.update --- tests/document/test_instance.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 3dabaabca..84a507461 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -2755,17 +2755,17 @@ class Person(Document): from pymongo.collection import Collection - orig_update = Collection.update + orig_update_one = Collection.update_one try: - def fake_update(*args, **kwargs): + def fake_update_one(*args, **kwargs): self.fail("Unexpected update for %s" % args[0].name) - return orig_update(*args, **kwargs) + return orig_update_one(*args, **kwargs) - Collection.update = fake_update + Collection.update_one = fake_update_one person.save() finally: - Collection.update = orig_update + Collection.update_one = orig_update_one def test_db_alias_tests(self): """DB Alias tests.""" From 316e0bfbe06587ef8f2801574d1913440ab10119 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 12:53:44 -0800 Subject: [PATCH 12/17] feat: narrow count_documents fall back to deprecated count --- mongoengine/pymongo_support.py | 17 +++++++++++++++-- tests/document/test_instance.py | 11 +++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mongoengine/pymongo_support.py b/mongoengine/pymongo_support.py index f3f9dea79..44c2bb5d3 100644 --- a/mongoengine/pymongo_support.py +++ b/mongoengine/pymongo_support.py @@ -33,12 +33,25 @@ def count_documents( if PYMONGO_VERSION >= (3, 7): try: return collection.count_documents(filter=filter, **kwargs) - except OperationFailure: + except OperationFailure as err: + if PYMONGO_VERSION >= (4,): + raise + # OperationFailure - accounts for some operators that used to work # with .count but are no longer working with count_documents (i.e $geoNear, $near, and $nearSphere) # fallback to deprecated Cursor.count # Keeping this should be reevaluated the day pymongo removes .count entirely - pass + message = str(err) + if not ( + "not allowed in this context" in message + and ( + "$where" in message + or "$geoNear" in message + or "$near" in message + or "$nearSphere" in message + ) + ): + raise cursor = collection.find(filter) for option, option_value in kwargs.items(): diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index 84a507461..1f9a76b47 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -28,7 +28,10 @@ MONGODB_36, get_mongodb_version, ) -from mongoengine.pymongo_support import list_collection_names +from mongoengine.pymongo_support import ( + PYMONGO_VERSION, + list_collection_names, +) from mongoengine.queryset import NULLIFY, Q from tests import fixtures from tests.fixtures import ( @@ -2943,7 +2946,11 @@ def __str__(self): } ) assert [str(b) for b in custom_qs] == ["1", "2"] - assert custom_qs.count() == 2 + + # count only will work with this raw query before pymongo 4.x, but + # the length is also implicitly checked above + if PYMONGO_VERSION < (4,): + assert custom_qs.count() == 2 def test_switch_db_instance(self): register_connection("testdb-1", "mongoenginetest2") From 3e84beec8467834cc8129f72634c6a3ab055ec20 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 16:18:57 -0800 Subject: [PATCH 13/17] tests: update geo tests to not use .count() on queries that cannot use count_documents --- tests/queryset/test_geo.py | 209 +++++++++++++++++++++++-------------- 1 file changed, 132 insertions(+), 77 deletions(-) diff --git a/tests/queryset/test_geo.py b/tests/queryset/test_geo.py index 0fe3af97a..670cf38db 100644 --- a/tests/queryset/test_geo.py +++ b/tests/queryset/test_geo.py @@ -2,6 +2,7 @@ import unittest from mongoengine import * +from mongoengine.pymongo_support import PYMONGO_VERSION from tests.utils import MongoDBTestCase @@ -47,13 +48,15 @@ def test_near(self): # note that "near" will show the san francisco event, too, # although it sorts to last. events = self.Event.objects(location__near=[-87.67892, 41.9120459]) - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event1, event3, event2] # ensure ordering is respected by "near" events = self.Event.objects(location__near=[-87.67892, 41.9120459]) events = events.order_by("-date") - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event3, event1, event2] def test_near_and_max_distance(self): @@ -65,8 +68,9 @@ def test_near_and_max_distance(self): # find events within 10 degrees of san francisco point = [-122.415579, 37.7566023] events = self.Event.objects(location__near=point, location__max_distance=10) - assert events.count() == 1 - assert events[0] == event2 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event2] def test_near_and_min_distance(self): """Ensure the "min_distance" operator works alongside the "near" @@ -77,7 +81,9 @@ def test_near_and_min_distance(self): # find events at least 10 degrees away of san francisco point = [-122.415579, 37.7566023] events = self.Event.objects(location__near=point, location__min_distance=10) - assert events.count() == 2 + if PYMONGO_VERSION < (4,): + assert events.count() == 2 + assert list(events) == [event3, event1] def test_within_distance(self): """Make sure the "within_distance" operator works.""" @@ -153,13 +159,15 @@ def test_2dsphere_near(self): # note that "near" will show the san francisco event, too, # although it sorts to last. events = self.Event.objects(location__near=[-87.67892, 41.9120459]) - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event1, event3, event2] # ensure ordering is respected by "near" events = self.Event.objects(location__near=[-87.67892, 41.9120459]) events = events.order_by("-date") - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event3, event1, event2] def test_2dsphere_near_and_max_distance(self): @@ -171,21 +179,25 @@ def test_2dsphere_near_and_max_distance(self): # find events within 10km of san francisco point = [-122.415579, 37.7566023] events = self.Event.objects(location__near=point, location__max_distance=10000) - assert events.count() == 1 - assert events[0] == event2 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event2] # find events within 1km of greenpoint, broolyn, nyc, ny events = self.Event.objects( location__near=[-73.9509714, 40.7237134], location__max_distance=1000 ) - assert events.count() == 0 + if PYMONGO_VERSION < (4,): + assert events.count() == 0 + assert list(events) == [] # ensure ordering is respected by "near" events = self.Event.objects( location__near=[-87.67892, 41.9120459], location__max_distance=10000 ).order_by("-date") - assert events.count() == 2 - assert events[0] == event3 + if PYMONGO_VERSION < (4,): + assert events.count() == 2 + assert list(events) == [event3, event1] def test_2dsphere_geo_within_box(self): """Ensure the "geo_within_box" operator works with a 2dsphere @@ -236,15 +248,17 @@ def test_2dsphere_near_and_min_max_distance(self): location__min_distance=1000, location__max_distance=10000, ).order_by("-date") - assert events.count() == 1 - assert events[0] == event3 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event3] # ensure ordering is respected by "near" with "min_distance" events = self.Event.objects( location__near=[-87.67892, 41.9120459], location__min_distance=10000 ).order_by("-date") - assert events.count() == 1 - assert events[0] == event2 + if PYMONGO_VERSION < (4,): + assert events.count() == 1 + assert list(events) == [event2] def test_2dsphere_geo_within_center(self): """Make sure the "geo_within_center" operator works with a @@ -289,7 +303,8 @@ class Event(Document): # note that "near" will show the san francisco event, too, # although it sorts to last. events = Event.objects(venue__location__near=[-87.67892, 41.9120459]) - assert events.count() == 3 + if PYMONGO_VERSION < (4,): + assert events.count() == 3 assert list(events) == [event1, event3, event2] def test_geo_spatial_embedded(self): @@ -318,7 +333,9 @@ class Point(Document): # Finds both points because they are within 60 km of the reference # point equidistant between them. points = Point.objects(location__near_sphere=[-122, 37.5]) - assert points.count() == 2 + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [north_point, south_point] # Same behavior for _within_spherical_distance points = Point.objects( @@ -329,36 +346,42 @@ class Point(Document): points = Point.objects( location__near_sphere=[-122, 37.5], location__max_distance=60 / earth_radius ) - assert points.count() == 2 + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [north_point, south_point] # Test query works with max_distance, being farer from one point points = Point.objects( location__near_sphere=[-122, 37.8], location__max_distance=60 / earth_radius ) close_point = points.first() - assert points.count() == 1 + if PYMONGO_VERSION < (4,): + assert points.count() == 1 + assert list(points) == [north_point] # Test query works with min_distance, being farer from one point points = Point.objects( location__near_sphere=[-122, 37.8], location__min_distance=60 / earth_radius ) - assert points.count() == 1 + if PYMONGO_VERSION < (4,): + assert points.count() == 1 far_point = points.first() + assert list(points) == [south_point] assert close_point != far_point # Finds both points, but orders the north point first because it's # closer to the reference point to the north. points = Point.objects(location__near_sphere=[-122, 38.5]) - assert points.count() == 2 - assert points[0].id == north_point.id - assert points[1].id == south_point.id + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [north_point, south_point] # Finds both points, but orders the south point first because it's # closer to the reference point to the south. points = Point.objects(location__near_sphere=[-122, 36.5]) - assert points.count() == 2 - assert points[0].id == south_point.id - assert points[1].id == north_point.id + if PYMONGO_VERSION < (4,): + assert points.count() == 2 + assert list(points) == [south_point, north_point] # Finds only one point because only the first point is within 60km of # the reference point to the south. @@ -375,56 +398,72 @@ class Road(Document): Road.drop_collection() - Road(name="66", line=[[40, 5], [41, 6]]).save() + road = Road(name="66", line=[[40, 5], [41, 6]]) + road.save() # near point = {"type": "Point", "coordinates": [40, 5]} - roads = Road.objects.filter(line__near=point["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__near=point["coordinates"]) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__near=point).count() - assert 1 == roads + roads = Road.objects.filter(line__near=point) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__near={"$geometry": point}).count() - assert 1 == roads + roads = Road.objects.filter(line__near={"$geometry": point}) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] # Within polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(line__geo_within=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_within=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_within=polygon).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_within=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_within={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_within={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] # Intersects line = {"type": "LineString", "coordinates": [[40, 5], [40, 6]]} - roads = Road.objects.filter(line__geo_intersects=line["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=line["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects=line).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=line) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects={"$geometry": line}).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects={"$geometry": line}) + assert roads.count() == 1 + assert list(roads) == [road] polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(line__geo_intersects=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects=polygon).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(line__geo_intersects={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(line__geo_intersects={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] def test_polygon(self): class Road(Document): @@ -433,56 +472,72 @@ class Road(Document): Road.drop_collection() - Road(name="66", poly=[[[40, 5], [40, 6], [41, 6], [40, 5]]]).save() + road = Road(name="66", poly=[[[40, 5], [40, 6], [41, 6], [40, 5]]]) + road.save() # near point = {"type": "Point", "coordinates": [40, 5]} - roads = Road.objects.filter(poly__near=point["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__near=point["coordinates"]) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__near=point).count() - assert 1 == roads + roads = Road.objects.filter(poly__near=point) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__near={"$geometry": point}).count() - assert 1 == roads + roads = Road.objects.filter(poly__near={"$geometry": point}) + if PYMONGO_VERSION < (4,): + assert roads.count() == 1 + assert list(roads) == [road] # Within polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(poly__geo_within=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_within=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_within=polygon).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_within=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_within={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_within={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] # Intersects line = {"type": "LineString", "coordinates": [[40, 5], [41, 6]]} - roads = Road.objects.filter(poly__geo_intersects=line["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=line["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects=line).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=line) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects={"$geometry": line}).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects={"$geometry": line}) + assert roads.count() == 1 + assert list(roads) == [road] polygon = { "type": "Polygon", "coordinates": [[[40, 5], [40, 6], [41, 6], [41, 5], [40, 5]]], } - roads = Road.objects.filter(poly__geo_intersects=polygon["coordinates"]).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=polygon["coordinates"]) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects=polygon).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects=polygon) + assert roads.count() == 1 + assert list(roads) == [road] - roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}).count() - assert 1 == roads + roads = Road.objects.filter(poly__geo_intersects={"$geometry": polygon}) + assert roads.count() == 1 + assert list(roads) == [road] def test_aspymongo_with_only(self): """Ensure as_pymongo works with only""" From 389fe1ef62efb1edfd03aa393b10aa05797a7e72 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 17:05:03 -0800 Subject: [PATCH 14/17] feat: handle db.authenticate removal --- mongoengine/connection.py | 44 ++++++++++++++++++++++++++------------- tests/test_connection.py | 19 ++++++++++++----- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/mongoengine/connection.py b/mongoengine/connection.py index 522826165..bbc98f17d 100644 --- a/mongoengine/connection.py +++ b/mongoengine/connection.py @@ -3,6 +3,8 @@ from pymongo import MongoClient, ReadPreference, uri_parser from pymongo.database import _check_name +from mongoengine.pymongo_support import PYMONGO_VERSION + __all__ = [ "DEFAULT_CONNECTION_NAME", "DEFAULT_DATABASE_NAME", @@ -277,15 +279,25 @@ def get_connection(alias=DEFAULT_CONNECTION_NAME, reconnect=False): raise ConnectionFailure(msg) def _clean_settings(settings_dict): - irrelevant_fields_set = { - "name", - "username", - "password", - "authentication_source", - "authentication_mechanism", - } + if PYMONGO_VERSION < (4,): + irrelevant_fields_set = { + "name", + "username", + "password", + "authentication_source", + "authentication_mechanism", + } + rename_fields = {} + else: + irrelevant_fields_set = {"name"} + rename_fields = { + "authentication_source": "authSource", + "authentication_mechanism": "authMechanism", + } return { - k: v for k, v in settings_dict.items() if k not in irrelevant_fields_set + rename_fields.get(k, k): v + for k, v in settings_dict.items() + if k not in irrelevant_fields_set and v is not None } raw_conn_settings = _connection_settings[alias].copy() @@ -365,14 +377,18 @@ def get_db(alias=DEFAULT_CONNECTION_NAME, reconnect=False): conn = get_connection(alias) conn_settings = _connection_settings[alias] db = conn[conn_settings["name"]] - auth_kwargs = {"source": conn_settings["authentication_source"]} - if conn_settings["authentication_mechanism"] is not None: - auth_kwargs["mechanism"] = conn_settings["authentication_mechanism"] # Authenticate if necessary - if conn_settings["username"] and ( - conn_settings["password"] - or conn_settings["authentication_mechanism"] == "MONGODB-X509" + if ( + PYMONGO_VERSION < (4,) + and conn_settings["username"] + and ( + conn_settings["password"] + or conn_settings["authentication_mechanism"] == "MONGODB-X509" + ) ): + auth_kwargs = {"source": conn_settings["authentication_source"]} + if conn_settings["authentication_mechanism"] is not None: + auth_kwargs["mechanism"] = conn_settings["authentication_mechanism"] db.authenticate( conn_settings["username"], conn_settings["password"], **auth_kwargs ) diff --git a/tests/test_connection.py b/tests/test_connection.py index c2034518a..4fb147561 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -351,8 +351,14 @@ def test_connect_uri(self): c.mongoenginetest.system.users.delete_many({}) c.admin.command("createUser", "admin", pwd="password", roles=["root"]) - c.admin.authenticate("admin", "password") - c.admin.command("createUser", "username", pwd="password", roles=["dbOwner"]) + + adminadmin_settings = mongoengine.connection._connection_settings[ + "adminadmin" + ] = mongoengine.connection._connection_settings["admin"].copy() + adminadmin_settings["username"] = "admin" + adminadmin_settings["password"] = "password" + ca = connect(db="mongoenginetest", alias="adminadmin") + ca.admin.command("createUser", "username", pwd="password", roles=["dbOwner"]) connect( "testdb_uri", host="mongodb://username:password@localhost/mongoenginetest" @@ -406,7 +412,10 @@ def test_uri_without_credentials_doesnt_override_conn_settings(self): # w/ the provided username/password and failed - that's the desired # behavior. If the MongoDB URI would override the credentials with pytest.raises(OperationFailure): - get_db() + db = get_db() + # pymongo 4.x does not call db.authenticate and needs to perform an operation to trigger the failure + if PYMONGO_VERSION >= (4,): + db.list_collection_names() def test_connect_uri_with_authsource(self): """Ensure that the connect() method works well with `authSource` @@ -574,8 +583,8 @@ def test_connect_2_databases_uses_same_client_if_only_dbname_differs(self): assert c1 is c2 def test_connect_2_databases_uses_different_client_if_different_parameters(self): - c1 = connect(alias="testdb1", db="testdb1", username="u1") - c2 = connect(alias="testdb2", db="testdb2", username="u2") + c1 = connect(alias="testdb1", db="testdb1", username="u1", password="pass") + c2 = connect(alias="testdb2", db="testdb2", username="u2", password="pass") assert c1 is not c2 From 1e231a3317dbff8a369f743e6e470b01839ec1ef Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Thu, 6 Jan 2022 17:13:13 -0800 Subject: [PATCH 15/17] fix: fix json_options for pymongo 3.9 --- mongoengine/pymongo_support.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mongoengine/pymongo_support.py b/mongoengine/pymongo_support.py index 44c2bb5d3..fcddbf7b2 100644 --- a/mongoengine/pymongo_support.py +++ b/mongoengine/pymongo_support.py @@ -7,9 +7,12 @@ PYMONGO_VERSION = tuple(pymongo.version_tuple[:2]) -LEGACY_JSON_OPTIONS = json_util.LEGACY_JSON_OPTIONS.with_options( - uuid_representation=binary.UuidRepresentation.PYTHON_LEGACY, -) +if PYMONGO_VERSION >= (4,): + LEGACY_JSON_OPTIONS = json_util.LEGACY_JSON_OPTIONS.with_options( + uuid_representation=binary.UuidRepresentation.PYTHON_LEGACY, + ) +else: + LEGACY_JSON_OPTIONS = json_util.DEFAULT_JSON_OPTIONS def count_documents( From d86dabaa60bf82c762e407ec42930e932a036e78 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Mon, 24 Jan 2022 09:49:39 -0800 Subject: [PATCH 16/17] tests: update connection test to be more specific based on PR review --- tests/test_connection.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/test_connection.py b/tests/test_connection.py index 4fb147561..a9c56d63b 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -411,11 +411,14 @@ def test_uri_without_credentials_doesnt_override_conn_settings(self): # OperationFailure means that mongoengine attempted authentication # w/ the provided username/password and failed - that's the desired # behavior. If the MongoDB URI would override the credentials - with pytest.raises(OperationFailure): - db = get_db() - # pymongo 4.x does not call db.authenticate and needs to perform an operation to trigger the failure - if PYMONGO_VERSION >= (4,): + if PYMONGO_VERSION >= (4,): + with pytest.raises(OperationFailure): + db = get_db() + # pymongo 4.x does not call db.authenticate and needs to perform an operation to trigger the failure db.list_collection_names() + else: + with pytest.raises(OperationFailure): + db = get_db() def test_connect_uri_with_authsource(self): """Ensure that the connect() method works well with `authSource` From 2f29d6912fcf80cc5f3707062969311473311f36 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Mon, 24 Jan 2022 13:09:53 -0800 Subject: [PATCH 17/17] docs: update documentation based on changes --- AUTHORS | 1 + docs/changelog.rst | 32 +++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 60663940a..40508b532 100644 --- a/AUTHORS +++ b/AUTHORS @@ -263,3 +263,4 @@ that much better: * Timothé Perez (https://github.com/AchilleAsh) * oleksandr-l5 (https://github.com/oleksandr-l5) * Ido Shraga (https://github.com/idoshr) + * Terence Honles (https://github.com/terencehonles) diff --git a/docs/changelog.rst b/docs/changelog.rst index c49de79ff..32cfdfdb9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,11 +7,41 @@ Changelog Development =========== - (Fill this out as you fix issues and develop your features). -- EnumField improvements: now `choices` limits the values of an enum to allow +- EnumField improvements: now ``choices`` limits the values of an enum to allow - Fix bug that prevented instance queryset from using custom queryset_class #2589 - Fix deepcopy of EmbeddedDocument #2202 - Fix error when using precision=0 with DecimalField #2535 - Add support for regex and whole word text search query #2568 +- BREAKING CHANGE: Updates to support pymongo 4.0. Where possible deprecated + functionality has been migrated, but additional care should be taken when + migrating to pymongo 4.0 as existing code may have been using deprecated + features which have now been removed #2614. + + For the pymongo migration guide see: + https://pymongo.readthedocs.io/en/stable/migrate-to-pymongo4.html. + + In addition to the changes in the migration guide, the following is a high + level overview of the changes made to MongoEngine when using pymongo 4.0: + + - limited support of geohaystack indexes has been removed + - ``QuerySet.map_reduce`` has been migrated from ``Collection.map_reduce`` + and ``Collection.inline_map_reduce`` to use + ``db.command({mapReduce: ..., ...})`` and support between the two may need + additional verification. + - UUIDs are encoded with the ``pythonLegacy`` encoding by default instead of + the newer and cross platform ``standard`` encoding. Existing UUIDs will + need to be migrated before changing the encoding, and this should be done + explicitly by the user rather than switching to a new default by + MongoEngine. This default will change at a later date, but to allow + specifying and then migrating to the new format a default ``json_options`` + has been provided. + - ``Queryset.count`` has been using ``Collection.count_documents`` and + transparently falling back to ``Collection.count`` when using features that + are not supported by ``Collection.count_documents``. ``Collection.count`` + has been removed and no automatic fallback is possible. The migration guide + documents the extended functionality which is no longer supported. Rewrite + the unsupported queries or fetch the whole result set and perform the count + locally. Changes in 0.23.1 ===========