From 78ba4837c894aa7ecc4d888035955e55ae9df101 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 14 Jan 2025 14:53:34 +0100 Subject: [PATCH 1/7] Run tests on MongoDB v5.0.23 instead of v3.2.19. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9bb57a69d..af9698eb2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,9 +23,9 @@ jobs: <<: *defaults docker: - image: circleci/python:3.10 - - image: mongo:3.2.19 + - image: mongo:5.0.23 test-3.11: <<: *defaults docker: - image: cimg/python:3.11 - - image: mongo:3.2.19 + - image: mongo:5.0.23 From 34ca4485f5467e71d928f27201dab93a781cff42 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 14 Jan 2025 15:13:15 +0100 Subject: [PATCH 2/7] Remove `item_frequencies`. This is an obscure piece of functionality that's hasn't been working for quite some time and is not needed for us going forward. The only remaining references to `item_frequencies` are in the old upgrade doc and in a changelog that's unmaintained in this repo: ``` (vevn) wojcikstefan@stefans-mbp mongoengine % rg item_frequencies docs/changelog.rst 233:- Added support for null / zero / false values in item_frequencies 432:- Updated item_frequencies to handle embedded document lookups 443:- Added map_reduce method item_frequencies and set as default (as db.eval doesn't work in sharded environments) 464:- Fixed item_frequencies when using name thats the same as a native js function docs/upgrade.rst 388:item_frequencies 392:item_frequencies. A side effect was to return keys in the value they are 449: * :meth:`~mongoengine.queryset.QuerySet.item_frequencies` ``` --- docs/guide/querying.rst | 15 --- mongoengine/queryset/queryset.py | 139 -------------------- tests/queryset/test_queryset.py | 216 ------------------------------- 3 files changed, 370 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 1350130e8..b1f518ebe 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -357,21 +357,6 @@ To get the average (mean) of a field on a collection of documents, use mean_age = User.objects.average('age') -As MongoDB provides native lists, MongoEngine provides a helper method to get a -dictionary of the frequencies of items in lists across an entire collection -- -:meth:`~mongoengine.queryset.QuerySet.item_frequencies`. An example of its use -would be generating "tag-clouds":: - - class Article(Document): - tag = ListField(StringField()) - - # After adding some tagged articles... - tag_freqs = Article.objects.item_frequencies('tag', normalize=True) - - from operator import itemgetter - top_tags = sorted(tag_freqs.items(), key=itemgetter(1), reverse=True)[:10] - - Query efficiency and performance ================================ diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 4f92b0a1e..1721976fe 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1212,33 +1212,6 @@ def average(self, field): return result[0]['total'] return 0 - def item_frequencies(self, field, normalize=False, map_reduce=True): - """Returns a dictionary of all items present in a field across - the whole queried set of documents, and their corresponding frequency. - This is useful for generating tag clouds, or searching documents. - - .. note:: - - Can only do direct simple mappings and cannot map across - :class:`~mongoengine.fields.ReferenceField` or - :class:`~mongoengine.fields.GenericReferenceField` for more complex - counting a manual map reduce call would is required. - - If the field is a :class:`~mongoengine.fields.ListField`, the items within - each list will be counted individually. - - :param field: the field to use - :param normalize: normalize the results so they add to 1.0 - :param map_reduce: Use map_reduce over exec_js - - .. versionchanged:: 0.5 defaults to map_reduce and can handle embedded - document lookups - """ - if map_reduce: - return self._item_frequencies_map_reduce(field, - normalize=normalize) - return self._item_frequencies_exec_js(field, normalize=normalize) - # Iterator helpers def __next__(self): @@ -1370,118 +1343,6 @@ def no_dereference(self): # Helper Functions - def _item_frequencies_map_reduce(self, field, normalize=False): - map_func = """ - function() { - var path = '{{~%(field)s}}'.split('.'); - var field = this; - - for (p in path) { - if (typeof field != 'undefined') - field = field[path[p]]; - else - break; - } - if (field && field.constructor == Array) { - field.forEach(function(item) { - emit(item, 1); - }); - } else if (typeof field != 'undefined') { - emit(field, 1); - } else { - emit(null, 1); - } - } - """ % dict(field=field) - reduce_func = """ - function(key, values) { - var total = 0; - var valuesSize = values.length; - for (var i=0; i < valuesSize; i++) { - total += parseInt(values[i], 10); - } - return total; - } - """ - values = self.map_reduce(map_func, reduce_func, 'inline') - frequencies = {} - for f in values: - key = f.key - if isinstance(key, float): - if int(key) == key: - key = int(key) - frequencies[key] = int(f.value) - - if normalize: - count = sum(frequencies.values()) - frequencies = dict([(k, float(v) / count) - for k, v in list(frequencies.items())]) - - return frequencies - - def _item_frequencies_exec_js(self, field, normalize=False): - """Uses exec_js to execute""" - freq_func = """ - function(path) { - var path = path.split('.'); - - var total = 0.0; - db[collection].find(query).forEach(function(doc) { - var field = doc; - for (p in path) { - if (field) - field = field[path[p]]; - else - break; - } - if (field && field.constructor == Array) { - total += field.length; - } else { - total++; - } - }); - - var frequencies = {}; - var types = {}; - var inc = 1.0; - - db[collection].find(query).forEach(function(doc) { - field = doc; - for (p in path) { - if (field) - field = field[path[p]]; - else - break; - } - if (field && field.constructor == Array) { - field.forEach(function(item) { - frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); - }); - } else { - var item = field; - types[item] = item; - frequencies[item] = inc + (isNaN(frequencies[item]) ? 0: frequencies[item]); - } - }); - return [total, frequencies, types]; - } - """ - total, data, types = self.exec_js(freq_func, field) - values = dict([(types.get(k), int(v)) for k, v in data.items()]) - - if normalize: - values = dict([(k, float(v) / total) for k, v in list(values.items())]) - - frequencies = {} - for k, v in values.items(): - if isinstance(k, float): - if int(k) == k: - k = int(k) - - frequencies[k] = v - - return frequencies - def _fields_to_dbfields(self, fields): """Translate fields paths to its db equivalents""" ret = [] diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 922ec7f3c..a90ca8219 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -1927,222 +1927,6 @@ class Link(Document): Link.drop_collection() - def test_item_frequencies(self): - """Ensure that item frequencies are properly generated from lists. - """ - class BlogPost(Document): - hits = IntField() - tags = ListField(StringField(), db_field='blogTags') - - BlogPost.drop_collection() - - BlogPost(hits=1, tags=['music', 'film', 'actors', 'watch']).save() - BlogPost(hits=2, tags=['music', 'watch']).save() - BlogPost(hits=2, tags=['music', 'actors']).save() - - def test_assertions(f): - f = dict((key, int(val)) for key, val in list(f.items())) - self.assertEqual(set(['music', 'film', 'actors', 'watch']), set(f.keys())) - self.assertEqual(f['music'], 3) - self.assertEqual(f['actors'], 2) - self.assertEqual(f['watch'], 2) - self.assertEqual(f['film'], 1) - - exec_js = BlogPost.objects.item_frequencies('tags') - map_reduce = BlogPost.objects.item_frequencies('tags', map_reduce=True) - test_assertions(exec_js) - test_assertions(map_reduce) - - # Ensure query is taken into account - def test_assertions(f): - f = dict((key, int(val)) for key, val in list(f.items())) - self.assertEqual(set(['music', 'actors', 'watch']), set(f.keys())) - self.assertEqual(f['music'], 2) - self.assertEqual(f['actors'], 1) - self.assertEqual(f['watch'], 1) - - exec_js = BlogPost.objects(hits__gt=1).item_frequencies('tags') - map_reduce = BlogPost.objects(hits__gt=1).item_frequencies('tags', map_reduce=True) - test_assertions(exec_js) - test_assertions(map_reduce) - - # Check that normalization works - def test_assertions(f): - self.assertAlmostEqual(f['music'], 3.0/8.0) - self.assertAlmostEqual(f['actors'], 2.0/8.0) - self.assertAlmostEqual(f['watch'], 2.0/8.0) - self.assertAlmostEqual(f['film'], 1.0/8.0) - - exec_js = BlogPost.objects.item_frequencies('tags', normalize=True) - map_reduce = BlogPost.objects.item_frequencies('tags', normalize=True, map_reduce=True) - test_assertions(exec_js) - test_assertions(map_reduce) - - # Check item_frequencies works for non-list fields - def test_assertions(f): - self.assertEqual(set([1, 2]), set(f.keys())) - self.assertEqual(f[1], 1) - self.assertEqual(f[2], 2) - - exec_js = BlogPost.objects.item_frequencies('hits') - map_reduce = BlogPost.objects.item_frequencies('hits', map_reduce=True) - test_assertions(exec_js) - test_assertions(map_reduce) - - BlogPost.drop_collection() - - def test_item_frequencies_on_embedded(self): - """Ensure that item frequencies are properly generated from lists. - """ - - class Phone(EmbeddedDocument): - number = StringField() - - class Person(Document): - name = StringField() - phone = EmbeddedDocumentField(Phone) - - Person.drop_collection() - - doc = Person(name="Guido") - doc.phone = Phone(number='62-3331-1656') - doc.save() - - doc = Person(name="Marr") - doc.phone = Phone(number='62-3331-1656') - doc.save() - - doc = Person(name="WP Junior") - doc.phone = Phone(number='62-3332-1656') - doc.save() - - def test_assertions(f): - f = dict((key, int(val)) for key, val in list(f.items())) - self.assertEqual(set(['62-3331-1656', '62-3332-1656']), set(f.keys())) - self.assertEqual(f['62-3331-1656'], 2) - self.assertEqual(f['62-3332-1656'], 1) - - exec_js = Person.objects.item_frequencies('phone.number') - map_reduce = Person.objects.item_frequencies('phone.number', map_reduce=True) - test_assertions(exec_js) - test_assertions(map_reduce) - - # Ensure query is taken into account - def test_assertions(f): - f = dict((key, int(val)) for key, val in list(f.items())) - self.assertEqual(set(['62-3331-1656']), set(f.keys())) - self.assertEqual(f['62-3331-1656'], 2) - - exec_js = Person.objects(phone__number='62-3331-1656').item_frequencies('phone.number') - map_reduce = Person.objects(phone__number='62-3331-1656').item_frequencies('phone.number', map_reduce=True) - test_assertions(exec_js) - test_assertions(map_reduce) - - # Check that normalization works - def test_assertions(f): - self.assertEqual(f['62-3331-1656'], 2.0/3.0) - self.assertEqual(f['62-3332-1656'], 1.0/3.0) - - exec_js = Person.objects.item_frequencies('phone.number', normalize=True) - map_reduce = Person.objects.item_frequencies('phone.number', normalize=True, map_reduce=True) - test_assertions(exec_js) - test_assertions(map_reduce) - - def test_item_frequencies_null_values(self): - - class Person(Document): - name = StringField() - city = StringField() - - Person.drop_collection() - - Person(name="Wilson Snr", city="CRB").save() - Person(name="Wilson Jr").save() - - freq = Person.objects.item_frequencies('city') - self.assertEqual(freq, {'CRB': 1.0, None: 1.0}) - freq = Person.objects.item_frequencies('city', normalize=True) - self.assertEqual(freq, {'CRB': 0.5, None: 0.5}) - - freq = Person.objects.item_frequencies('city', map_reduce=True) - self.assertEqual(freq, {'CRB': 1.0, None: 1.0}) - freq = Person.objects.item_frequencies('city', normalize=True, map_reduce=True) - self.assertEqual(freq, {'CRB': 0.5, None: 0.5}) - - def test_item_frequencies_with_null_embedded(self): - class Data(EmbeddedDocument): - name = StringField() - - class Extra(EmbeddedDocument): - tag = StringField() - - class Person(Document): - data = EmbeddedDocumentField(Data, required=True) - extra = EmbeddedDocumentField(Extra) - - Person.drop_collection() - - p = Person() - p.data = Data(name="Wilson Jr") - p.save() - - p = Person() - p.data = Data(name="Wesley") - p.extra = Extra(tag="friend") - p.save() - - ot = Person.objects.item_frequencies('extra.tag', map_reduce=False) - self.assertEqual(ot, {None: 1.0, 'friend': 1.0}) - - ot = Person.objects.item_frequencies('extra.tag', map_reduce=True) - self.assertEqual(ot, {None: 1.0, 'friend': 1.0}) - - def test_item_frequencies_with_0_values(self): - class Test(Document): - val = IntField() - - Test.drop_collection() - t = Test() - t.val = 0 - t.save() - - ot = Test.objects.item_frequencies('val', map_reduce=True) - self.assertEqual(ot, {0: 1}) - ot = Test.objects.item_frequencies('val', map_reduce=False) - self.assertEqual(ot, {0: 1}) - - def test_item_frequencies_with_False_values(self): - class Test(Document): - val = BooleanField() - - Test.drop_collection() - t = Test() - t.val = False - t.save() - - ot = Test.objects.item_frequencies('val', map_reduce=True) - self.assertEqual(ot, {False: 1}) - ot = Test.objects.item_frequencies('val', map_reduce=False) - self.assertEqual(ot, {False: 1}) - - def test_item_frequencies_normalize(self): - class Test(Document): - val = IntField() - - Test.drop_collection() - - for i in range(50): - Test(val=1).save() - - for i in range(20): - Test(val=2).save() - - freqs = Test.objects.item_frequencies('val', map_reduce=False, normalize=True) - self.assertEqual(freqs, {1: 50.0/70, 2: 20.0/70}) - - freqs = Test.objects.item_frequencies('val', map_reduce=True, normalize=True) - self.assertEqual(freqs, {1: 50.0/70, 2: 20.0/70}) - def test_average(self): """Ensure that field can be averaged correctly.""" ages = [0, 23, 54, 12, 94, 27] From 70806f9f7b7276f3a8c0becb8b5be8a473d3632f Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 14 Jan 2025 15:19:09 +0100 Subject: [PATCH 3/7] Remove `map_reduce`. This is an obscure piece of functionality that's hasn't been working for quite some time and is not needed for us going forward. The only remaining references to `map_reduce` are in the old upgrade doc and in a changelog that's unmaintained in this repo: ``` (vevn) wojcikstefan@stefans-mbp mongoengine % rg map_reduce docs/changelog.rst 418:- Updated sum / average to use map_reduce as db.eval doesn't work in sharded environments 443:- Added map_reduce method item_frequencies and set as default (as db.eval doesn't work in sharded environments) 444:- Added inline_map_reduce option to map_reduce 459: map_reduce now requires an output. docs/upgrade.rst 428:main areas of changed are: choices in fields, map_reduce and collection names. 444:More methods now use map_reduce as db.eval is not supported for sharding as ``` --- mongoengine/queryset/queryset.py | 95 -------------- tests/queryset/test_queryset.py | 205 ------------------------------- 2 files changed, 300 deletions(-) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index 1721976fe..f7653adda 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -1030,101 +1030,6 @@ def from_json(self, json_data): # JS functionality - def map_reduce(self, map_f, reduce_f, output, finalize_f=None, limit=None, - scope=None): - """Perform a map/reduce query using the current query spec - and ordering. While ``map_reduce`` respects ``QuerySet`` chaining, - it must be the last call made, as it does not return a maleable - ``QuerySet``. - - See the :meth:`~mongoengine.tests.QuerySetTest.test_map_reduce` - and :meth:`~mongoengine.tests.QuerySetTest.test_map_advanced` - tests in ``tests.queryset.QuerySetTest`` for usage examples. - - :param map_f: map function, as :class:`~bson.code.Code` or string - :param reduce_f: reduce function, as - :class:`~bson.code.Code` or string - :param output: output collection name, if set to 'inline' will try to - use :class:`~pymongo.collection.Collection.inline_map_reduce` - This can also be a dictionary containing output options - see: http://docs.mongodb.org/manual/reference/commands/#mapReduce - :param finalize_f: finalize function, an optional function that - performs any post-reduction processing. - :param scope: values to insert into map/reduce global scope. Optional. - :param limit: number of objects from current query to provide - to map/reduce method - - Returns an iterator yielding - :class:`~mongoengine.document.MapReduceDocument`. - - .. note:: - - Map/Reduce changed in server version **>= 1.7.4**. The PyMongo - :meth:`~pymongo.collection.Collection.map_reduce` helper requires - PyMongo version **>= 1.11**. - - .. versionchanged:: 0.5 - - removed ``keep_temp`` keyword argument, which was only relevant - for MongoDB server versions older than 1.7.4 - - .. versionadded:: 0.3 - """ - queryset = self.clone() - - MapReduceDocument = _import_class('MapReduceDocument') - - if not hasattr(self._collection, "map_reduce"): - raise NotImplementedError("Requires MongoDB >= 1.7.1") - - map_f_scope = {} - if isinstance(map_f, Code): - map_f_scope = map_f.scope - map_f = str(map_f) - map_f = Code(queryset._sub_js_fields(map_f), map_f_scope) - - reduce_f_scope = {} - if isinstance(reduce_f, Code): - reduce_f_scope = reduce_f.scope - reduce_f = str(reduce_f) - reduce_f_code = queryset._sub_js_fields(reduce_f) - reduce_f = Code(reduce_f_code, reduce_f_scope) - - mr_args = {'query': queryset._query} - - if finalize_f: - finalize_f_scope = {} - if isinstance(finalize_f, Code): - finalize_f_scope = finalize_f.scope - finalize_f = str(finalize_f) - finalize_f_code = queryset._sub_js_fields(finalize_f) - finalize_f = Code(finalize_f_code, finalize_f_scope) - mr_args['finalize'] = finalize_f - - if scope: - mr_args['scope'] = scope - - if limit: - mr_args['limit'] = limit - - if output == 'inline' and not queryset._ordering: - map_reduce_function = 'inline_map_reduce' - else: - map_reduce_function = 'map_reduce' - mr_args['out'] = output - - results = getattr(queryset._collection, map_reduce_function)( - map_f, reduce_f, **mr_args) - - if map_reduce_function == 'map_reduce': - results = results.find() - - if queryset._ordering: - results = results.sort(queryset._ordering) - - for doc in results: - yield MapReduceDocument(queryset._document, queryset._collection, - doc['_id'], doc['value']) - def exec_js(self, code, *fields, **options): """Execute a Javascript function on the server. A list of fields may be provided, which will be translated to their correct names and supplied diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index a90ca8219..ffc7e9567 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -1722,211 +1722,6 @@ class Author(Document): names = [a.author.name for a in Author.objects.order_by('-author__age')] self.assertEqual(names, ['User A', 'User B', 'User C']) - def test_map_reduce(self): - """Ensure map/reduce is both mapping and reducing. - """ - class BlogPost(Document): - title = StringField() - tags = ListField(StringField(), db_field='post-tag-list') - - BlogPost.drop_collection() - - BlogPost(title="Post #1", tags=['music', 'film', 'print']).save() - BlogPost(title="Post #2", tags=['music', 'film']).save() - BlogPost(title="Post #3", tags=['film', 'photography']).save() - - map_f = """ - function() { - this[~tags].forEach(function(tag) { - emit(tag, 1); - }); - } - """ - - reduce_f = """ - function(key, values) { - var total = 0; - for(var i=0; i 0) { - y = 1; - } else if (x = 0) { - y = 0; - } else { - y = -1; - } - - // calculate 'Z', the maximal value - if(Math.abs(x) >= 1) { - z = Math.abs(x); - } else { - z = 1; - } - - return {x: x, y: y, z: z, t_s: sec_since_epoch}; - } - """ - - finalize_f = """ - function(key, value) { - // f(sec_since_epoch,y,z) = - // log10(z) + ((y*sec_since_epoch) / 45000) - z_10 = Math.log(value.z) / Math.log(10); - weight = z_10 + ((value.y * value.t_s) / 45000); - return weight; - } - """ - - # provide the reddit epoch (used for ranking) as a variable available - # to all phases of the map/reduce operation: map, reduce, and finalize. - reddit_epoch = mktime(datetime(2005, 12, 8, 7, 46, 43).timetuple()) - scope = {'reddit_epoch': reddit_epoch} - - # run a map/reduce operation across all links. ordering is set - # to "-value", which orders the "weight" value returned from - # "finalize_f" in descending order. - results = Link.objects.order_by("-value") - results = results.map_reduce(map_f, - reduce_f, - "myresults", - finalize_f=finalize_f, - scope=scope) - results = list(results) - - # assert troublesome Buzz article is ranked 1st - self.assertTrue(results[0].object.title.startswith("Google Buzz")) - - # assert laser vision is ranked last - self.assertTrue(results[-1].object.title.startswith("How to see")) - - Link.drop_collection() - def test_average(self): """Ensure that field can be averaged correctly.""" ages = [0, 23, 54, 12, 94, 27] From 88222425cfa274891d431b4387cbcc1c31e990fb Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 14 Jan 2025 15:27:12 +0100 Subject: [PATCH 4/7] Remove `QuerySet.exec_js` and `QuerySet.where`. These weren't really working well already and we don't need them. It's also dangerous to run arbitrary and possibly unvalidated / unsanitized JS code from Python. The only remaining mention of `exec_js` is in the unmaintained changelog: ``` (vevn) wojcikstefan@stefans-mbp mongoengine % rg exec_js docs/changelog.rst 535:- Fixed bug where ``QuerySet.exec_js`` ignored ``Q`` objects ``` --- docs/guide/querying.rst | 103 ----------------------- mongoengine/queryset/queryset.py | 70 +--------------- tests/queryset/test_queryset.py | 136 ------------------------------- 3 files changed, 2 insertions(+), 307 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index b1f518ebe..5e7ecffd7 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -526,106 +526,3 @@ cannot use the `$` syntax in keyword arguments it has been mapped to `S`:: Currently only top level lists are handled, future versions of mongodb / pymongo plan to support nested positional operators. See `The $ positional operator `_. - -Server-side javascript execution -================================ -Javascript functions may be written and sent to the server for execution. The -result of this is the return value of the Javascript function. This -functionality is accessed through the -:meth:`~mongoengine.queryset.QuerySet.exec_js` method on -:meth:`~mongoengine.queryset.QuerySet` objects. Pass in a string containing a -Javascript function as the first argument. - -The remaining positional arguments are names of fields that will be passed into -you Javascript function as its arguments. This allows functions to be written -that may be executed on any field in a collection (e.g. the -:meth:`~mongoengine.queryset.QuerySet.sum` method, which accepts the name of -the field to sum over as its argument). Note that field names passed in in this -manner are automatically translated to the names used on the database (set -using the :attr:`name` keyword argument to a field constructor). - -Keyword arguments to :meth:`~mongoengine.queryset.QuerySet.exec_js` are -combined into an object called :attr:`options`, which is available in the -Javascript function. This may be used for defining specific parameters for your -function. - -Some variables are made available in the scope of the Javascript function: - -* ``collection`` -- the name of the collection that corresponds to the - :class:`~mongoengine.Document` class that is being used; this should be - used to get the :class:`Collection` object from :attr:`db` in Javascript - code -* ``query`` -- the query that has been generated by the - :class:`~mongoengine.queryset.QuerySet` object; this may be passed into - the :meth:`find` method on a :class:`Collection` object in the Javascript - function -* ``options`` -- an object containing the keyword arguments passed into - :meth:`~mongoengine.queryset.QuerySet.exec_js` - -The following example demonstrates the intended usage of -:meth:`~mongoengine.queryset.QuerySet.exec_js` by defining a function that sums -over a field on a document (this functionality is already available throught -:meth:`~mongoengine.queryset.QuerySet.sum` but is shown here for sake of -example):: - - def sum_field(document, field_name, include_negatives=True): - code = """ - function(sumField) { - var total = 0.0; - db[collection].find(query).forEach(function(doc) { - var val = doc[sumField]; - if (val >= 0.0 || options.includeNegatives) { - total += val; - } - }); - return total; - } - """ - options = {'includeNegatives': include_negatives} - return document.objects.exec_js(code, field_name, **options) - -As fields in MongoEngine may use different names in the database (set using the -:attr:`db_field` keyword argument to a :class:`Field` constructor), a mechanism -exists for replacing MongoEngine field names with the database field names in -Javascript code. When accessing a field on a collection object, use -square-bracket notation, and prefix the MongoEngine field name with a tilde. -The field name that follows the tilde will be translated to the name used in -the database. Note that when referring to fields on embedded documents, -the name of the :class:`~mongoengine.fields.EmbeddedDocumentField`, followed by a dot, -should be used before the name of the field on the embedded document. The -following example shows how the substitutions are made:: - - class Comment(EmbeddedDocument): - content = StringField(db_field='body') - - class BlogPost(Document): - title = StringField(db_field='doctitle') - comments = ListField(EmbeddedDocumentField(Comment), name='cs') - - # Returns a list of dictionaries. Each dictionary contains a value named - # "document", which corresponds to the "title" field on a BlogPost, and - # "comment", which corresponds to an individual comment. The substitutions - # made are shown in the comments. - BlogPost.objects.exec_js(""" - function() { - var comments = []; - db[collection].find(query).forEach(function(doc) { - // doc[~comments] -> doc["cs"] - var docComments = doc[~comments]; - - for (var i = 0; i < docComments.length; i++) { - // doc[~comments][i] -> doc["cs"][i] - var comment = doc[~comments][i]; - - comments.push({ - // doc[~title] -> doc["doctitle"] - 'document': doc[~title], - - // comment[~comments.content] -> comment["body"] - 'comment': comment[~comments.content] - }); - } - }); - return comments; - } - """) diff --git a/mongoengine/queryset/queryset.py b/mongoengine/queryset/queryset.py index f7653adda..841de2dfa 100644 --- a/mongoengine/queryset/queryset.py +++ b/mongoengine/queryset/queryset.py @@ -49,7 +49,6 @@ def __init__(self, document, collection): self._mongo_query = None self._query_obj = Q() self._initial_query = {} - self._where_clause = None self._loaded_fields = QueryFieldList() self._ordering = None self._timeout = True @@ -716,7 +715,7 @@ def clone(self): copy_props = ( '_mongo_query', '_initial_query', '_none', '_query_obj', - '_where_clause', '_loaded_fields', '_ordering', '_timeout', + '_loaded_fields', '_ordering', '_timeout', '_class_check', '_read_preference', '_iter', '_scalar', '_as_pymongo', '_as_pymongo_coerce', '_limit', '_skip', '_hint', '_batch_size', '_auto_dereference' @@ -1028,67 +1027,7 @@ def from_json(self, json_data): son_data = json_util.loads(json_data) return [self._document._from_son(data) for data in son_data] - # JS functionality - - def exec_js(self, code, *fields, **options): - """Execute a Javascript function on the server. A list of fields may be - provided, which will be translated to their correct names and supplied - as the arguments to the function. A few extra variables are added to - the function's scope: ``collection``, which is the name of the - collection in use; ``query``, which is an object representing the - current query; and ``options``, which is an object containing any - options specified as keyword arguments. - - As fields in MongoEngine may use different names in the database (set - using the :attr:`db_field` keyword argument to a :class:`Field` - constructor), a mechanism exists for replacing MongoEngine field names - with the database field names in Javascript code. When accessing a - field, use square-bracket notation, and prefix the MongoEngine field - name with a tilde (~). - - :param code: a string of Javascript code to execute - :param fields: fields that you will be using in your function, which - will be passed in to your function as arguments - :param options: options that you want available to the function - (accessed in Javascript through the ``options`` object) - """ - queryset = self.clone() - - code = queryset._sub_js_fields(code) - - fields = [queryset._document._translate_field_name(f) for f in fields] - collection = queryset._document._get_collection_name() - - scope = { - 'collection': collection, - 'options': options or {}, - } - - query = queryset._query - if queryset._where_clause: - query['$where'] = queryset._where_clause - - scope['query'] = query - code = Code(code, scope=scope) - - db = queryset._document._get_db() - return db.eval(code, *fields) - - def where(self, where_clause): - """Filter ``QuerySet`` results with a ``$where`` clause (a Javascript - expression). Performs automatic field name substitution like - :meth:`mongoengine.queryset.Queryset.exec_js`. - - .. note:: When using this mode of query, the database will call your - function, or evaluate your predicate clause, for each object - in the collection. - - .. versionadded:: 0.5 - """ - queryset = self.clone() - where_clause = queryset._sub_js_fields(where_clause) - queryset._where_clause = where_clause - return queryset + # Basic aggregations def sum(self, field): """Sum over the values of the specified field. @@ -1180,11 +1119,6 @@ def _cursor(self): self._cursor_obj = self._collection.find(self._query, **self._cursor_args) - # Apply "where" clauses to the cursor. - if self._where_clause: - where_clause = self._sub_js_fields(self._where_clause) - self._cursor_obj.where(where_clause) - if self._ordering: # Apply query ordering self._cursor_obj.sort(self._ordering) diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index ffc7e9567..8ec42ce6b 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -1049,101 +1049,6 @@ class BlogPost(Document): BlogPost.drop_collection() - def test_exec_js_query(self): - """Ensure that queries are properly formed for use in exec_js. - """ - class BlogPost(Document): - hits = IntField() - published = BooleanField() - - BlogPost.drop_collection() - - post1 = BlogPost(hits=1, published=False) - post1.save() - - post2 = BlogPost(hits=1, published=True) - post2.save() - - post3 = BlogPost(hits=1, published=True) - post3.save() - - js_func = """ - function(hitsField) { - var count = 0; - db[collection].find(query).forEach(function(doc) { - count += doc[hitsField]; - }); - return count; - } - """ - - # Ensure that normal queries work - c = BlogPost.objects(published=True).exec_js(js_func, 'hits') - self.assertEqual(c, 2) - - c = BlogPost.objects(published=False).exec_js(js_func, 'hits') - self.assertEqual(c, 1) - - BlogPost.drop_collection() - - def test_exec_js_field_sub(self): - """Ensure that field substitutions occur properly in exec_js functions. - """ - class Comment(EmbeddedDocument): - content = StringField(db_field='body') - - class BlogPost(Document): - name = StringField(db_field='doc-name') - comments = ListField(EmbeddedDocumentField(Comment), - db_field='cmnts') - - BlogPost.drop_collection() - - comments1 = [Comment(content='cool'), Comment(content='yay')] - post1 = BlogPost(name='post1', comments=comments1) - post1.save() - - comments2 = [Comment(content='nice stuff')] - post2 = BlogPost(name='post2', comments=comments2) - post2.save() - - code = """ - function getComments() { - var comments = []; - db[collection].find(query).forEach(function(doc) { - var docComments = doc[~comments]; - for (var i = 0; i < docComments.length; i++) { - comments.push({ - 'document': doc[~name], - 'comment': doc[~comments][i][~comments.content] - }); - } - }); - return comments; - } - """ - - sub_code = BlogPost.objects._sub_js_fields(code) - code_chunks = ['doc["cmnts"];', 'doc["doc-name"],', - 'doc["cmnts"][i]["body"]'] - for chunk in code_chunks: - self.assertTrue(chunk in sub_code) - - results = BlogPost.objects.exec_js(code) - expected_results = [ - {'comment': 'cool', 'document': 'post1'}, - {'comment': 'yay', 'document': 'post1'}, - {'comment': 'nice stuff', 'document': 'post2'}, - ] - self.assertEqual(results, expected_results) - - # Test template style - code = "{{~comments.content}}" - sub_code = BlogPost.objects._sub_js_fields(code) - self.assertEqual("cmnts.body", sub_code) - - BlogPost.drop_collection() - def test_delete(self): """Ensure that documents are properly deleted from the database. """ @@ -2314,47 +2219,6 @@ class Comment(Document): for key, value in info.items()] self.assertTrue(([('_cls', 1), ('message', 1)], False, False) in info) - def test_where(self): - """Ensure that where clauses work. - """ - - class IntPair(Document): - fielda = IntField() - fieldb = IntField() - - IntPair.objects._collection.delete_many({}) - - a = IntPair(fielda=1, fieldb=1) - b = IntPair(fielda=1, fieldb=2) - c = IntPair(fielda=2, fieldb=1) - a.save() - b.save() - c.save() - - query = IntPair.objects.where('this[~fielda] >= this[~fieldb]') - self.assertEqual('this["fielda"] >= this["fieldb"]', query._where_clause) - results = list(query) - self.assertEqual(2, len(results)) - self.assertTrue(a in results) - self.assertTrue(c in results) - - query = IntPair.objects.where('this[~fielda] == this[~fieldb]') - results = list(query) - self.assertEqual(1, len(results)) - self.assertTrue(a in results) - - query = IntPair.objects.where('function() { return this[~fielda] >= this[~fieldb] }') - self.assertEqual('function() { return this["fielda"] >= this["fieldb"] }', query._where_clause) - results = list(query) - self.assertEqual(2, len(results)) - self.assertTrue(a in results) - self.assertTrue(c in results) - - def invalid_where(): - list(IntPair.objects.where(fielda__gte=3)) - - self.assertRaises(TypeError, invalid_where) - def test_scalar(self): class Organization(Document): From 5234645f21ed600308abefebc4030ada3cf14693 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 14 Jan 2025 15:38:27 +0100 Subject: [PATCH 5/7] Remove the `push_all` / `$pushAll` operator. This operator has been deprecated since MongoDB v2.4 and has been removed from subsequent versions. --- docs/guide/querying.rst | 1 - mongoengine/queryset/transform.py | 5 ++--- tests/queryset/test_queryset.py | 8 ++------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/guide/querying.rst b/docs/guide/querying.rst index 5e7ecffd7..6feea6a8c 100644 --- a/docs/guide/querying.rst +++ b/docs/guide/querying.rst @@ -482,7 +482,6 @@ that you may use with these methods: * ``dec`` -- decrement a value by a given amount * ``pop`` -- remove the last item from a list * ``push`` -- append a value to a list -* ``push_all`` -- append several values to a list * ``pop`` -- remove the first or last element of a list * ``pull`` -- remove a value from a list * ``pull_all`` -- remove several values from a list diff --git a/mongoengine/queryset/transform.py b/mongoengine/queryset/transform.py index 6e8b2a12a..2157676f8 100644 --- a/mongoengine/queryset/transform.py +++ b/mongoengine/queryset/transform.py @@ -24,8 +24,7 @@ STRING_OPERATORS + CUSTOM_OPERATORS) UPDATE_OPERATORS = ('set', 'unset', 'inc', 'dec', 'pop', 'push', - 'push_all', 'pull', 'pull_all', 'add_to_set', - 'set_on_insert') + 'pull', 'pull_all', 'add_to_set', 'set_on_insert') def query(_doc_cls=None, _field_operation=False, **query): @@ -155,7 +154,7 @@ def update(_doc_cls=None, **update): if parts[0] in UPDATE_OPERATORS: op = parts.pop(0) # Convert Pythonic names to Mongo equivalents - if op in ('push_all', 'pull_all'): + if op == 'pull_all': op = op.replace('_all', 'All') elif op == 'dec': # Support decrement by flipping a positive value's sign diff --git a/tests/queryset/test_queryset.py b/tests/queryset/test_queryset.py index 8ec42ce6b..2259d3492 100644 --- a/tests/queryset/test_queryset.py +++ b/tests/queryset/test_queryset.py @@ -1305,10 +1305,6 @@ class BlogPost(Document): post.reload() self.assertTrue('mongo' in post.tags) - BlogPost.objects.update_one(push_all__tags=['db', 'nosql']) - post.reload() - self.assertTrue('db' in post.tags and 'nosql' in post.tags) - tags = post.tags[:-1] BlogPost.objects.update(pop__tags=1) post.reload() @@ -1342,9 +1338,9 @@ class BlogPost(Document): post.reload() self.assertEqual(post.tags, ["code"]) - BlogPost.objects.filter(id=post.id).update(push_all__tags=["mongodb", "code"]) + BlogPost.objects.filter(id=post.id).update(push__tags="mongodb") post.reload() - self.assertEqual(post.tags, ["code", "mongodb", "code"]) + self.assertEqual(post.tags, ["code", "mongodb"]) BlogPost.objects(slug="test").update(pull__tags="code") post.reload() From 77996dc62b494528a7036d7274d0d5d677a3fdb4 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 14 Jan 2025 15:51:48 +0100 Subject: [PATCH 6/7] Temporary debug statements. --- tests/document/test_instance.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index f47c01262..b0bdeadd8 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -382,6 +382,7 @@ class Animal(Document): query_op = q.db.system.profile.find_one({ 'ns': 'mongoenginetest.animal' }) + print(query_op) self.assertEqual( set(query_op['query']['filter'].keys()), set(['_id', 'superphylum']) @@ -407,6 +408,7 @@ class Animal(Document): doc.save() query_op = q.db.system.profile.find({ 'ns': 'mongoenginetest.animal' })[0] self.assertEqual(query_op['op'], 'update') + print(query_op) self.assertEqual(set(query_op['query'].keys()), set(['_id', 'is_mammal'])) Animal.drop_collection() From 701c6dbd8ac7a4f174a3b9b8154c0fd778cc4114 Mon Sep 17 00:00:00 2001 From: Stefan Wojcik Date: Tue, 14 Jan 2025 16:00:23 +0100 Subject: [PATCH 7/7] Fix tests using the profile collection. --- tests/document/test_instance.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/document/test_instance.py b/tests/document/test_instance.py index b0bdeadd8..642de4bac 100644 --- a/tests/document/test_instance.py +++ b/tests/document/test_instance.py @@ -382,9 +382,9 @@ class Animal(Document): query_op = q.db.system.profile.find_one({ 'ns': 'mongoenginetest.animal' }) - print(query_op) + query_command = query_op.get('query') or query_op['command'] self.assertEqual( - set(query_op['query']['filter'].keys()), + set(query_command['filter'].keys()), set(['_id', 'superphylum']) ) @@ -406,10 +406,11 @@ class Animal(Document): with query_counter() as q: doc.name = 'Cat' doc.save() + query_op = q.db.system.profile.find({ 'ns': 'mongoenginetest.animal' })[0] self.assertEqual(query_op['op'], 'update') - print(query_op) - self.assertEqual(set(query_op['query'].keys()), set(['_id', 'is_mammal'])) + query = query_op.get('query') or query_op['command']['q'] + self.assertEqual(set(query.keys()), set(['_id', 'is_mammal'])) Animal.drop_collection()