From e7aff8233e979c092b96925838b84a1baf19ed84 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 9 Feb 2017 17:02:00 +0100 Subject: [PATCH 1/5] Ignore "index with different options already exists" mongoengine errors. Right now we can't do anything about those errors, proper solution would involve, re-creating the index, but this could be very expensive and blocking. --- st2common/st2common/models/db/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index c0be7d6705..792b652588 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -118,6 +118,15 @@ def db_ensure_indexes(): try: model_class.ensure_indexes() + except OperationFailure as e: + # Note we ignore "index with different options already exists" error. + # Right now we can't do anything about it, one approach would be to re-create that + # index, but this could potentially be very expensive .and blocking + msg = str(e) + if 'already exists with different options' in msg: + LOG.debug('Ignoring index already exists with different options error: %s' % (msg)) + else: + raise e except Exception as e: tb_msg = traceback.format_exc() msg = 'Failed to ensure indexes for model "%s": %s' % (class_name, str(e)) From c130f8ef3d585c61af982ff8e9552127ef846096 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 9 Feb 2017 17:08:39 +0100 Subject: [PATCH 2/5] Improve comment. --- st2common/st2common/models/db/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index 792b652588..f05e98adf7 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -119,9 +119,10 @@ def db_ensure_indexes(): try: model_class.ensure_indexes() except OperationFailure as e: - # Note we ignore "index with different options already exists" error. - # Right now we can't do anything about it, one approach would be to re-create that - # index, but this could potentially be very expensive .and blocking + # Note: We ignore "index with different options already exists" error. + # Right now we can't do anything about it. Correct approach would be to re-create that + # index, but the operation is blocking and depending on the dataset and index size, + # this operation could be very expensive and take a long time. msg = str(e) if 'already exists with different options' in msg: LOG.debug('Ignoring index already exists with different options error: %s' % (msg)) From 90655da1f4b7bfba85a96527d8f70d6c218ca183 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 9 Feb 2017 18:13:42 +0100 Subject: [PATCH 3/5] Add some sleep to give time to MongoDB to start up. --- circle.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/circle.yml b/circle.yml index eeeb00653c..f7dfc2ef3e 100644 --- a/circle.yml +++ b/circle.yml @@ -20,6 +20,9 @@ dependencies: # We install and test with all the supported MongoDB versions - case $CIRCLE_NODE_INDEX in [1,2]) sudo service mongodb stop ; MONGODB_VERSION=3.2.12 sudo -E .circle/install-and-run-mongodb.sh > /tmp/mongodb-install.log 2>&1 ;; [3,4]) sudo service mongodb stop ; MONGODB_VERSION=3.4.2 sudo -E .circle/install-and-run-mongodb.sh > /tmp/mongodb-install.log 2>&1 ;; esac: background: true + # We sleep a bit to wait for the background process to start and script to + # finish + - case $CIRCLE_NODE_INDEX in [1,2]) sleep 10 ;; esac: # Tail the logs so it's easier to see what is going on. Sadly when using "background: true" # whole output is ignored. - case $CIRCLE_NODE_INDEX in [1,2]) tail -50 /tmp/mongodb-install.log ;; [3,4]) tail -50 /tmp/mongodb-install.log ;; esac From 58849fe898b17eb788f5105f792ce54b70fced05 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 9 Feb 2017 18:32:22 +0100 Subject: [PATCH 4/5] Fix typo. --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index f7dfc2ef3e..fbbc7ab8f3 100644 --- a/circle.yml +++ b/circle.yml @@ -22,7 +22,7 @@ dependencies: background: true # We sleep a bit to wait for the background process to start and script to # finish - - case $CIRCLE_NODE_INDEX in [1,2]) sleep 10 ;; esac: + - case $CIRCLE_NODE_INDEX in [1,2]) sleep 10 ;; esac # Tail the logs so it's easier to see what is going on. Sadly when using "background: true" # whole output is ignored. - case $CIRCLE_NODE_INDEX in [1,2]) tail -50 /tmp/mongodb-install.log ;; [3,4]) tail -50 /tmp/mongodb-install.log ;; esac From ebcf8cb20401ec6235143057e02530e4e9dc012e Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 10 Feb 2017 10:06:45 +0100 Subject: [PATCH 5/5] Update MongoDB set up code to drop offending and deprecated "types" indexes. Sadly there is no way around it but to drop the offending index and re-create it See http://docs.mongoengine.org/upgrade.html#inheritance --- st2common/st2common/models/db/__init__.py | 39 +++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/st2common/st2common/models/db/__init__.py b/st2common/st2common/models/db/__init__.py index f05e98adf7..e131d4ecdc 100644 --- a/st2common/st2common/models/db/__init__.py +++ b/st2common/st2common/models/db/__init__.py @@ -119,13 +119,14 @@ def db_ensure_indexes(): try: model_class.ensure_indexes() except OperationFailure as e: - # Note: We ignore "index with different options already exists" error. - # Right now we can't do anything about it. Correct approach would be to re-create that - # index, but the operation is blocking and depending on the dataset and index size, - # this operation could be very expensive and take a long time. + # Special case for "uid" index. MongoDB 3.4 has dropped "_types" index option so we + # need to re-create the index to make it work and avoid "index with different options + # already exists" error. + # Note: This condition would only be encountered when upgrading existing StackStorm + # installation from MongoDB 3.2 to 3.4. msg = str(e) - if 'already exists with different options' in msg: - LOG.debug('Ignoring index already exists with different options error: %s' % (msg)) + if 'already exists with different options' in msg and 'uid_1' in msg: + drop_obsolete_types_indexes(model_class=model_class) else: raise e except Exception as e: @@ -167,6 +168,32 @@ def cleanup_extra_indexes(model_class): return removed_count +def drop_obsolete_types_indexes(model_class): + """ + Special class for droping offending "types" indexes for which support has + been removed in mongoengine and MongoDB 3.4. + For more info, see: http://docs.mongoengine.org/upgrade.html#inheritance + """ + class_name = model_class.__name__ + + LOG.debug('Dropping obsolete types index for model "%s"' % (class_name)) + collection = model_class._get_collection() + collection.update({}, {'$unset': {'_types': 1}}, multi=True) + + info = collection.index_information() + indexes_to_drop = [key for key, value in info.iteritems() + if '_types' in dict(value['key']) or 'types' in value] + + LOG.debug('Will drop obsolete types indexes for model "%s": %s' % (class_name, + str(indexes_to_drop))) + + for index in indexes_to_drop: + collection.drop_index(index) + + LOG.debug('Recreating indexes for model "%s"' % (class_name)) + model_class.ensure_indexes() + + def db_teardown(): mongoengine.connection.disconnect()