[WIP] [DONT MERGE] [EXPERIMENT] Use more efficient mongoengine fork

fixed-requirements.txt

@@ -18,8 +18,8 @@ requests[security]==2.22.0
 apscheduler==3.6.3
 gitpython==2.1.11
 jsonschema==2.6.0
+git+https://github.com/closeio/mongoengine.git@master#egg=mongoengine
 pymongo==3.10.0
-mongoengine==0.18.2
 passlib==1.7.1
 lockfile==0.12.2
 python-gnupg==0.4.5

st2actions/tests/unit/policies/test_retry_policy.py

@@ -13,7 +13,9 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
 import mock
+from bson import ObjectId
 
 import st2actions
 from st2common.constants.action import LIVEACTION_STATUS_REQUESTED
@@ -258,7 +260,7 @@ def test_no_retry_on_workflow_task(self):
         live_action_db = LiveActionDB(
             action='wolfpack.action-1',
             parameters={'actionstr': 'foo'},
-            context={'parent': {'execution_id': 'abcde'}}
+            context={'parent': {'execution_id': ObjectId('5609e91832ed356d04a93cc0')}}
         )
 
         live_action_db, execution_db = action_service.request(live_action_db)

st2actions/tests/unit/test_executions.py

@@ -100,7 +100,7 @@ def test_basic_execution(self):
         self.assertEqual(execution.result, liveaction.result)
         self.assertEqual(execution.status, liveaction.status)
         self.assertEqual(execution.context, liveaction.context)
-        self.assertEqual(execution.liveaction['callback'], liveaction.callback)
+        self.assertEqual(execution.liveaction.get('callback', {}), liveaction.callback)
         self.assertEqual(execution.liveaction['action'], liveaction.action)
 
     def test_basic_execution_history_create_failed(self):
@@ -127,7 +127,7 @@ def test_chained_executions(self):
         self.assertEqual(execution.result, liveaction.result)
         self.assertEqual(execution.status, liveaction.status)
         self.assertEqual(execution.context, liveaction.context)
-        self.assertEqual(execution.liveaction['callback'], liveaction.callback)
+        self.assertEqual(execution.liveaction.get('callback', {}), liveaction.callback)
         self.assertEqual(execution.liveaction['action'], liveaction.action)
         self.assertGreater(len(execution.children), 0)
 
@@ -184,7 +184,7 @@ def test_triggered_execution(self):
         self.assertEqual(execution.result, liveaction.result)
         self.assertEqual(execution.status, liveaction.status)
         self.assertEqual(execution.context, liveaction.context)
-        self.assertEqual(execution.liveaction['callback'], liveaction.callback)
+        self.assertEqual(execution.liveaction.get('callback', {}), liveaction.callback)
         self.assertEqual(execution.liveaction['action'], liveaction.action)
 
     def _get_action_execution(self, **kwargs):

st2api/st2api/controllers/v1/actionalias.py

@@ -166,6 +166,8 @@ def put(self, action_alias, ref_or_id, requester_user):
             old_action_alias_db = action_alias_db
             action_alias_db = ActionAliasAPI.to_model(action_alias)
             action_alias_db.id = ref_or_id
+            # mongo mallard way of saying that we are updating an existing document
+            action_alias_db._lazy = True
             action_alias_db = ActionAlias.add_or_update(action_alias_db)
         except (ValidationError, ValueError) as e:
             LOG.exception('Validation failed for action alias data=%s', action_alias)

st2api/st2api/controllers/v1/actions.py

@@ -171,6 +171,8 @@ def put(self, action, ref_or_id, requester_user):
             action_db = ActionAPI.to_model(action)
             LOG.debug('/actions/ PUT incoming action: %s', action_db)
             action_db.id = action_id
+            # mongo mallard way of saying that we are updating an existing document
+            action_db._lazy = True
             action_db = Action.add_or_update(action_db)
             LOG.debug('/actions/ PUT after add_or_update: %s', action_db)
         except (ValidationError, ValueError) as e:
@@ -288,9 +290,12 @@ def _update_pack_model(self, pack_ref, data_files, written_file_paths):
             file_paths.append(file_path)
 
         pack_db = Pack.get_by_ref(pack_ref)
-        pack_db.files = set(pack_db.files)
-        pack_db.files.update(set(file_paths))
-        pack_db.files = list(pack_db.files)
+
+        pack_files = set(pack_db.files)
+        pack_files.update(set(file_paths))
+        pack_files = list(pack_files)
+
+        pack_db.files = pack_files
         pack_db = Pack.add_or_update(pack_db)
 
         return pack_db

st2api/st2api/controllers/v1/auth.py

@@ -205,6 +205,8 @@ def put(self, api_key_api, api_key_id_or_key, requester_user):
             raise ValueError('Update of key_hash is not allowed.')
 
         api_key_db.id = old_api_key_db.id
+        # mongo mallard way of saying that we are updating an existing document
+        api_key_db._lazy = True
         api_key_db = ApiKey.add_or_update(api_key_db)
 
         extra = {'old_api_key_db': old_api_key_db, 'new_api_key_db': api_key_db}

st2api/st2api/controllers/v1/keyvalue.py

@@ -310,6 +310,8 @@ def put(self, kvp, name, requester_user, scope=FULL_SYSTEM_SCOPE):
 
                 if existing_kvp_api:
                     kvp_db.id = existing_kvp_api.id
+                    # mongo mallard way of saying that we are updating an existing document
+                    kvp_db._lazy = True
 
                 kvp_db = KeyValuePair.add_or_update(kvp_db)
             except (ValidationError, ValueError) as e:

st2api/st2api/controllers/v1/policies.py

@@ -198,6 +198,8 @@ def put(self, instance, ref_or_id, requester_user):
         try:
             db_model = self.model.to_model(instance)
             db_model.id = db_model_id
+            # mongo mallard way of saying that we are updating an existing document
+            db_model._lazy = True
             db_model = self.access.add_or_update(db_model)
         except (ValidationError, ValueError) as e:
             LOG.exception('%s unable to update object: %s', op, db_model)

st2api/st2api/controllers/v1/rules.py

@@ -196,6 +196,8 @@ def put(self, rule, rule_ref_or_id, requester_user):
                                                                           rule_api=rule)
 
             rule_db.id = rule_ref_or_id
+            # mongo mallard way of saying that we are updating an existing document
+            rule_db._lazy = True
             rule_db = Rule.add_or_update(rule_db)
             # After the rule has been added modify the ref_count. This way a failure to add
             # the rule due to violated constraints will have no impact on ref_count.

st2api/st2api/controllers/v1/sensors.py

@@ -92,6 +92,8 @@ def put(self, sensor_type, ref_or_id, requester_user):
         try:
             old_sensor_type_db = sensor_type_db
             sensor_type_db.id = sensor_type_id
+            # mongo mallard way of saying that we are updating an existing document
+            sensor_type_db._lazy = True
             sensor_type_db.enabled = getattr(sensor_type, 'enabled', False)
             sensor_type_db = SensorType.add_or_update(sensor_type_db)
         except (ValidationError, ValueError) as e:

st2api/st2api/controllers/v1/triggers.py

@@ -110,6 +110,8 @@ def put(self, triggertype, triggertype_ref_or_id):
                 LOG.warning('Discarding mismatched id=%s found in payload and using uri_id=%s.',
                             triggertype.id, triggertype_id)
             triggertype_db.id = triggertype_id
+            # mongo mallard way of saying that we are updating an existing document
+            triggertype_db._lazy = True
             old_triggertype_db = triggertype_db
             triggertype_db = TriggerType.add_or_update(triggertype_db)
         except (ValidationError, ValueError) as e:
@@ -254,6 +256,8 @@ def put(self, trigger, trigger_id):
                             trigger.id, trigger_id)
             trigger_db = TriggerAPI.to_model(trigger)
             trigger_db.id = trigger_id
+            # mongo mallard way of saying that we are updating an existing document
+            trigger_db._lazy = True
             trigger_db = Trigger.add_or_update(trigger_db)
         except (ValidationError, ValueError) as e:
             LOG.exception('Validation failed for trigger data=%s', trigger)

st2common/st2common/bootstrap/actionsregistrar.py

@@ -176,7 +176,10 @@ def _register_action(self, pack, action):
         else:
             LOG.debug('Action %s found. Will be updated from: %s to: %s',
                       action_ref, existing, model)
+            # NOTE: _lazy is needed because of mongo mallard changes to amke sure we update an
+            # existing object which doesn't have _db_data attribute populated
             model.id = existing.id
+            model._lazy = True
 
         try:
             model = Action.add_or_update(model)

st2common/st2common/bootstrap/aliasesregistrar.py

@@ -146,6 +146,7 @@ def _register_action_alias(self, pack, action_alias):
 
         try:
             action_alias_db.id = ActionAlias.get_by_name(action_alias_db.name).id
+            action_alias._lazy = True
         except StackStormDBObjectNotFoundError:
             LOG.debug('ActionAlias %s not found. Creating new one.', action_alias)
 

st2common/st2common/fields.py

@@ -16,7 +16,7 @@
 import datetime
 import calendar
 
-from mongoengine import LongField
+from mongoengine import IntField
 
 from st2common.util import date as date_utils
 
@@ -27,7 +27,7 @@
 SECOND_TO_MICROSECONDS = 1000000
 
 
-class ComplexDateTimeField(LongField):
+class ComplexDateTimeField(IntField):
     """
     Date time field which handles microseconds exactly and internally stores
     the timestamp as number of microseconds since the unix epoch.
@@ -92,6 +92,9 @@ def __get__(self, instance, owner):
         return self._convert_from_db(data)
 
     def __set__(self, instance, value):
+        #if isinstance(value, int):
+        #    value = self.to_python(value)
+
         value = self._convert_from_datetime(value) if value else value
         return super(ComplexDateTimeField, self).__set__(instance, value)
 
@@ -109,8 +112,12 @@ def to_python(self, value):
             return original_value
 
     def to_mongo(self, value):
+        if value is None:
+            return value
         value = self.to_python(value)
         return self._convert_from_datetime(value)
 
     def prepare_query_value(self, op, value):
-        return self._convert_from_datetime(value)
+        if isinstance(value, datetime.datetime):
+            return self._convert_from_datetime(value)
+        return value

st2common/st2common/models/db/__init__.py

@@ -236,9 +236,10 @@ def cleanup_extra_indexes(model_class):
     """
     Finds any extra indexes and removes those from mongodb.
     """
-    extra_indexes = model_class.compare_indexes().get('extra', None)
-    if not extra_indexes:
-        return 0
+    return 0
+    #extra_indexes = model_class.compare_indexes().get('extra', None)
+    #if not extra_indexes:
+    #    return 0
 
     # mongoengine does not have the necessary method so we need to drop to
     # pymongo interfaces via some private methods.

st2common/st2common/models/db/execution.py

@@ -103,7 +103,7 @@ def get_uid(self):
     def mask_secrets(self, value):
         result = copy.deepcopy(value)
 
-        liveaction = result['liveaction']
+        liveaction = result.get('liveaction', {})
         parameters = {}
         # pylint: disable=no-member
         parameters.update(value.get('action', {}).get('parameters', {}))

st2common/st2common/models/db/stormbase.py

@@ -123,8 +123,7 @@ class EscapedDictField(me.DictField):
 
     def to_mongo(self, value, use_db_field=True, fields=None):
         value = mongoescape.escape_chars(value)
-        return super(EscapedDictField, self).to_mongo(value=value, use_db_field=use_db_field,
-                                                      fields=fields)
+        return super(EscapedDictField, self).to_mongo(value)
 
     def to_python(self, value):
         value = super(EscapedDictField, self).to_python(value)
@@ -142,8 +141,7 @@ class EscapedDynamicField(me.DynamicField):
 
     def to_mongo(self, value, use_db_field=True, fields=None):
         value = mongoescape.escape_chars(value)
-        return super(EscapedDynamicField, self).to_mongo(value=value, use_db_field=use_db_field,
-                                                         fields=fields)
+        return super(EscapedDynamicField, self).to_mongo(value)
 
     def to_python(self, value):
         value = super(EscapedDynamicField, self).to_python(value)

st2common/tests/unit/test_db_fields.py

@@ -68,7 +68,7 @@ def test_round_trip_conversion(self):
             expected_value = datetime_values[index]
             self.assertEqual(actual_value, expected_value)
 
-    @mock.patch('st2common.fields.LongField.__get__')
+    @mock.patch('st2common.fields.IntField.__get__')
     def test_get_(self, mock_get):
         field = ComplexDateTimeField()