From 41fe48a1de05d8347f19c281ded83783a75545cc Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Thu, 15 Aug 2019 21:10:24 +0200 Subject: [PATCH 1/6] Update various Python dependencies to latest stable versions. --- fixed-requirements.txt | 28 ++++++++++++++-------------- requirements.txt | 28 ++++++++++++++-------------- st2actions/requirements.txt | 8 ++++---- st2api/requirements.txt | 8 ++++---- st2auth/requirements.txt | 6 +++--- st2client/requirements.txt | 4 ++-- st2common/requirements.txt | 22 +++++++++++----------- st2debug/requirements.txt | 6 +++--- st2exporter/requirements.txt | 4 ++-- st2reactor/requirements.txt | 6 +++--- st2stream/requirements.txt | 8 ++++---- st2tests/requirements.txt | 2 +- test-requirements.txt | 4 ++-- 13 files changed, 67 insertions(+), 67 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 0695cb8e3e..12c3285301 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -3,35 +3,35 @@ greenlet==0.4.15 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely -eventlet==0.24.1 +eventlet==0.25.0 gunicorn==19.9.0 -kombu==4.5.0 +kombu==4.6.4 # Note: amqp is used by kombu -amqp==2.4.2 +amqp==2.5.0 # NOTE: Recent version substantially affect the performance and add big import time overhead # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details oslo.config>=1.12.1,<1.13 oslo.utils>=3.36.2,<=3.37.0 six==1.12.0 -pyyaml==5.1 +pyyaml==5.1.2 requests[security]>=2.22.0,<2.23.0 -apscheduler==3.6.0 +apscheduler==3.6.1 gitpython==2.1.11 jsonschema==2.6.0 -pymongo==3.7.2 -mongoengine==0.17.0 +pymongo==3.9.0 +mongoengine==0.18.2 passlib==1.7.1 lockfile==0.12.2 -python-gnupg==0.4.4 +python-gnupg==0.4.5 jsonpath-rw==1.4.0 pyinotify==0.9.6 semver==2.8.1 pytz==2019.1 stevedore==1.30.1 -paramiko==2.4.2 +paramiko==2.6.0 networkx==1.11 python-keyczar==0.716 -cryptography==2.6.1 +cryptography==2.7 retrying==1.3.3 # Note: We use latest version of virtualenv which uses pip 19 virtualenv==16.6.0 @@ -39,21 +39,21 @@ virtualenv==16.6.0 sseclient-py==1.7 python-editor==1.0.4 prompt-toolkit==1.0.15 -tooz==1.65.0 +tooz==1.66.1 zake==0.2.2 routes==2.4.1 -webob==1.8.4 +webob==1.8.5 flex==6.14.0 prance==0.9.0 pywinrm==0.3.0 # test requirements below nose-timer==0.7.5 nose-parallel==0.3.1 -psutil==5.6.1 +psutil==5.6.3 python-statsd==2.1.0 prometheus_client==0.1.1 mock==2.0.0 ujson==1.35 python-dateutil==2.8.0 -bcrypt==3.1.6 +bcrypt==3.1.7 jinja2==2.10.1 diff --git a/requirements.txt b/requirements.txt index fe7a5ba83c..8b2a68cc4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ # Don't edit this file. It's generated automatically! RandomWords -amqp==2.4.2 -apscheduler==3.6.0 +amqp==2.5.0 +apscheduler==3.6.1 argcomplete -bcrypt==3.1.6 -cryptography==2.6.1 -eventlet==0.24.1 +bcrypt==3.1.7 +cryptography==2.7 +eventlet==0.25.0 flex==6.14.0 git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper git+https://github.com/StackStorm/orquesta.git@e6ebbbeb2c661486067e659dc7552f0a986603a6#egg=orquesta @@ -18,32 +18,32 @@ ipaddr jinja2==2.10.1 jsonpath-rw==1.4.0 jsonschema==2.6.0 -kombu==4.5.0 +kombu==4.6.4 lockfile==0.12.2 mock==2.0.0 -mongoengine==0.17.0 +mongoengine==0.18.2 networkx==1.11 nose nose-parallel==0.3.1 nose-timer==0.7.5 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -paramiko==2.4.2 +paramiko==2.6.0 passlib==1.7.1 prettytable prompt-toolkit==1.0.15 -psutil==5.6.1 +psutil==5.6.3 pyinotify==0.9.6 -pymongo==3.7.2 +pymongo==3.9.0 pyrabbit python-dateutil==2.8.0 python-editor==1.0.4 -python-gnupg==0.4.4 +python-gnupg==0.4.5 python-json-logger python-statsd==2.1.0 pytz==2019.1 pywinrm==0.3.0 -pyyaml==5.1 +pyyaml==5.1.2 rednose requests[security]<2.23.0,>=2.22.0 retrying==1.3.3 @@ -52,9 +52,9 @@ semver==2.8.1 six==1.12.0 sseclient-py==1.7 stevedore==1.30.1 -tooz==1.65.0 +tooz==1.66.1 ujson==1.35 unittest2 -webob==1.8.4 +webob==1.8.5 webtest zake==0.2.2 diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index 23d109f47d..f0995e922a 100755 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -1,17 +1,17 @@ # Don't edit this file. It's generated automatically! -apscheduler==3.6.0 -eventlet==0.24.1 +apscheduler==3.6.1 +eventlet==0.25.0 git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient gitpython==2.1.11 jinja2==2.10.1 -kombu==4.5.0 +kombu==4.6.4 lockfile==0.12.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 pyinotify==0.9.6 python-dateutil==2.8.0 python-json-logger -pyyaml==5.1 +pyyaml==5.1.2 requests[security]<2.23.0,>=2.22.0 six==1.12.0 diff --git a/st2api/requirements.txt b/st2api/requirements.txt index 4952fb5a50..c63b226120 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -1,11 +1,11 @@ # Don't edit this file. It's generated automatically! -eventlet==0.24.1 +eventlet==0.25.0 git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient gunicorn==19.9.0 jsonschema==2.6.0 -kombu==4.5.0 -mongoengine==0.17.0 +kombu==4.6.4 +mongoengine==0.18.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -pymongo==3.7.2 +pymongo==3.9.0 six==1.12.0 diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index 0d6dd37f13..e82ecb42dd 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -1,10 +1,10 @@ # Don't edit this file. It's generated automatically! -bcrypt==3.1.6 -eventlet==0.24.1 +bcrypt==3.1.7 +eventlet==0.25.0 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file gunicorn==19.9.0 oslo.config<1.13,>=1.12.1 passlib==1.7.1 -pymongo==3.7.2 +pymongo==3.9.0 six==1.12.0 stevedore==1.30.1 diff --git a/st2client/requirements.txt b/st2client/requirements.txt index 9475249917..f7a2ba074b 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -1,6 +1,6 @@ # Don't edit this file. It's generated automatically! argcomplete -cryptography==2.6.1 +cryptography==2.7 jsonpath-rw==1.4.0 jsonschema==2.6.0 prettytable @@ -8,7 +8,7 @@ prompt-toolkit==1.0.15 python-dateutil==2.8.0 python-editor==1.0.4 pytz==2019.1 -pyyaml==5.1 +pyyaml==5.1.2 requests[security]<2.23.0,>=2.22.0 six==1.12.0 sseclient-py==1.7 diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 2a69c90af4..c343df8177 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -1,8 +1,8 @@ # Don't edit this file. It's generated automatically! -amqp==2.4.2 -apscheduler==3.6.0 -cryptography==2.6.1 -eventlet==0.24.1 +amqp==2.5.0 +apscheduler==3.6.1 +cryptography==2.7 +eventlet==0.25.0 flex==6.14.0 git+https://github.com/StackStorm/orquesta.git@e6ebbbeb2c661486067e659dc7552f0a986603a6#egg=orquesta gitpython==2.1.11 @@ -11,22 +11,22 @@ ipaddr jinja2==2.10.1 jsonpath-rw==1.4.0 jsonschema==2.6.0 -kombu==4.5.0 +kombu==4.6.4 lockfile==0.12.2 -mongoengine==0.17.0 +mongoengine==0.18.2 networkx==1.11 oslo.config<1.13,>=1.12.1 -paramiko==2.4.2 -pymongo==3.7.2 +paramiko==2.6.0 +pymongo==3.9.0 python-dateutil==2.8.0 python-statsd==2.1.0 -pyyaml==5.1 +pyyaml==5.1.2 requests[security]<2.23.0,>=2.22.0 retrying==1.3.3 routes==2.4.1 semver==2.8.1 six==1.12.0 -tooz==1.65.0 +tooz==1.66.1 ujson==1.35 -webob==1.8.4 +webob==1.8.5 zake==0.2.2 diff --git a/st2debug/requirements.txt b/st2debug/requirements.txt index 36fe7ede47..4e0f521ecf 100644 --- a/st2debug/requirements.txt +++ b/st2debug/requirements.txt @@ -1,6 +1,6 @@ # Don't edit this file. It's generated automatically! -eventlet==0.24.1 -python-gnupg==0.4.4 -pyyaml==5.1 +eventlet==0.25.0 +python-gnupg==0.4.5 +pyyaml==5.1.2 requests[security]<2.23.0,>=2.22.0 six==1.12.0 diff --git a/st2exporter/requirements.txt b/st2exporter/requirements.txt index de84ebad0b..b23198e559 100644 --- a/st2exporter/requirements.txt +++ b/st2exporter/requirements.txt @@ -1,5 +1,5 @@ # Don't edit this file. It's generated automatically! -eventlet==0.24.1 -kombu==4.5.0 +eventlet==0.25.0 +kombu==4.6.4 oslo.config<1.13,>=1.12.1 six==1.12.0 diff --git a/st2reactor/requirements.txt b/st2reactor/requirements.txt index ef707c1a67..9897717edb 100644 --- a/st2reactor/requirements.txt +++ b/st2reactor/requirements.txt @@ -1,9 +1,9 @@ # Don't edit this file. It's generated automatically! -apscheduler==3.6.0 -eventlet==0.24.1 +apscheduler==3.6.1 +eventlet==0.25.0 jsonpath-rw==1.4.0 jsonschema==2.6.0 -kombu==4.5.0 +kombu==4.6.4 oslo.config<1.13,>=1.12.1 python-dateutil==2.8.0 six==1.12.0 diff --git a/st2stream/requirements.txt b/st2stream/requirements.txt index 15e9de1eef..a03fb0f1fc 100644 --- a/st2stream/requirements.txt +++ b/st2stream/requirements.txt @@ -1,10 +1,10 @@ # Don't edit this file. It's generated automatically! -eventlet==0.24.1 +eventlet==0.25.0 gunicorn==19.9.0 jsonschema==2.6.0 -kombu==4.5.0 -mongoengine==0.17.0 +kombu==4.6.4 +mongoengine==0.18.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -pymongo==3.7.2 +pymongo==3.9.0 six==1.12.0 diff --git a/st2tests/requirements.txt b/st2tests/requirements.txt index a40f98860f..3d78725c35 100644 --- a/st2tests/requirements.txt +++ b/st2tests/requirements.txt @@ -4,7 +4,7 @@ mock==2.0.0 nose nose-parallel==0.3.1 nose-timer==0.7.5 -psutil==5.6.1 +psutil==5.6.3 pyrabbit rednose unittest2 diff --git a/test-requirements.txt b/test-requirements.txt index 8c5efa869b..24aa0ca6e2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,7 +7,7 @@ pylint==1.9.4 pylint-plugin-utils>=0.4 bandit==1.5.1 ipython<6.0.0 -mock==2.0.0 +mock==3.0.5 nose>=1.3.7 tabulate unittest2 @@ -25,7 +25,7 @@ gunicorn==19.9.0 psutil==5.6.1 webtest==2.0.25 rstcheck>=3.3.1,<3.4 -tox==3.8.6 +tox==3.13.2 pyrabbit # Since StackStorm v2.8.0 we now use cryptography instead of keyczar, but we still have some tests # which utilize keyczar and ensure new cryptography code is fully compatible with keyczar code From 70f3b999ec5d60e4b3388fcbcc010bc704598a52 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 16 Aug 2019 10:25:33 +0200 Subject: [PATCH 2/6] Add changelog entry. --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b6e5ec92e..f116589993 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -17,6 +17,8 @@ Changed * Install pack with the latest tag version if it exists when branch is not specialized. (improvement) #4743 * Implement "continue" engine command to orquesta workflow. (improvement) #4740 +* Update various internal dependencies to latest stable versions (apscheduler, eventlet, + kombu, amqp, pyyaml, pymongo, mongoengine, python-gnupg, paramiko, tooz, webob, bcrypt). #4767 Fixed ~~~~~ From a9f72ae54f41ead4dd7c20a0bf53be5bed0c6127 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 16 Aug 2019 14:50:04 +0200 Subject: [PATCH 3/6] Revert Pymongo upgrade due to regression in pymongo. See https://github.com/MongoEngine/mongoengine/pull/2147 for details. --- CHANGELOG.rst | 2 +- fixed-requirements.txt | 2 +- requirements.txt | 2 +- st2api/requirements.txt | 2 +- st2auth/requirements.txt | 2 +- st2common/requirements.txt | 2 +- st2stream/requirements.txt | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f116589993..d63345d2be 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,7 +18,7 @@ Changed (improvement) #4743 * Implement "continue" engine command to orquesta workflow. (improvement) #4740 * Update various internal dependencies to latest stable versions (apscheduler, eventlet, - kombu, amqp, pyyaml, pymongo, mongoengine, python-gnupg, paramiko, tooz, webob, bcrypt). #4767 + kombu, amqp, pyyaml, mongoengine, python-gnupg, paramiko, tooz, webob, bcrypt). #4767 Fixed ~~~~~ diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 12c3285301..a06c8d7f30 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -18,7 +18,7 @@ requests[security]>=2.22.0,<2.23.0 apscheduler==3.6.1 gitpython==2.1.11 jsonschema==2.6.0 -pymongo==3.9.0 +pymongo==3.7.2 mongoengine==0.18.2 passlib==1.7.1 lockfile==0.12.2 diff --git a/requirements.txt b/requirements.txt index 8b2a68cc4d..755c12b03e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ prettytable prompt-toolkit==1.0.15 psutil==5.6.3 pyinotify==0.9.6 -pymongo==3.9.0 +pymongo==3.7.2 pyrabbit python-dateutil==2.8.0 python-editor==1.0.4 diff --git a/st2api/requirements.txt b/st2api/requirements.txt index c63b226120..b791bcf92a 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -7,5 +7,5 @@ kombu==4.6.4 mongoengine==0.18.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -pymongo==3.9.0 +pymongo==3.7.2 six==1.12.0 diff --git a/st2auth/requirements.txt b/st2auth/requirements.txt index e82ecb42dd..b8a40a0e99 100644 --- a/st2auth/requirements.txt +++ b/st2auth/requirements.txt @@ -5,6 +5,6 @@ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2- gunicorn==19.9.0 oslo.config<1.13,>=1.12.1 passlib==1.7.1 -pymongo==3.9.0 +pymongo==3.7.2 six==1.12.0 stevedore==1.30.1 diff --git a/st2common/requirements.txt b/st2common/requirements.txt index c343df8177..ecadc3a41e 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -17,7 +17,7 @@ mongoengine==0.18.2 networkx==1.11 oslo.config<1.13,>=1.12.1 paramiko==2.6.0 -pymongo==3.9.0 +pymongo==3.7.2 python-dateutil==2.8.0 python-statsd==2.1.0 pyyaml==5.1.2 diff --git a/st2stream/requirements.txt b/st2stream/requirements.txt index a03fb0f1fc..7272fe4b0e 100644 --- a/st2stream/requirements.txt +++ b/st2stream/requirements.txt @@ -6,5 +6,5 @@ kombu==4.6.4 mongoengine==0.18.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -pymongo==3.9.0 +pymongo==3.7.2 six==1.12.0 From c75caea9956aa1d2984417ddb4c1ccb6dde2b805 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 16 Aug 2019 14:54:36 +0200 Subject: [PATCH 4/6] Add additional test cases for more robustness. --- st2api/tests/unit/controllers/v1/test_actions.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/st2api/tests/unit/controllers/v1/test_actions.py b/st2api/tests/unit/controllers/v1/test_actions.py index b905495220..d8c5aaf167 100644 --- a/st2api/tests/unit/controllers/v1/test_actions.py +++ b/st2api/tests/unit/controllers/v1/test_actions.py @@ -435,7 +435,8 @@ def test_post_name_unicode_action_already_exists(self): # Verify that exception messages containing unicode characters don't result in internal # server errors action = copy.deepcopy(ACTION_1) - action['name'] = 'žactionćšžž' + # NOTE: We explicitly don't prefix this string value with u"" + action['name'] = 'žactionćšžži💩' # 1. Initial creation post_resp = self.__do_post(action, expect_errors=True) @@ -446,6 +447,12 @@ def test_post_name_unicode_action_already_exists(self): self.assertEqual(post_resp.status_int, 409) self.assertTrue('Tried to save duplicate unique keys' in post_resp.json['faultstring']) + # 3. Action already exists (this time with unicode type) + action['name'] = u'žactionćšžži💩' + post_resp = self.__do_post(action, expect_errors=True) + self.assertEqual(post_resp.status_int, 409) + self.assertTrue('Tried to save duplicate unique keys' in post_resp.json['faultstring']) + @mock.patch.object(action_validator, 'validate_action', mock.MagicMock( return_value=True)) def test_post_parameter_type_is_array_and_invalid(self): From bec63d19831812e11affbb9e58dfcd6903bd0a21 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 16 Aug 2019 21:14:16 +0200 Subject: [PATCH 5/6] Update changelog. [ci skip] --- CHANGELOG.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d63345d2be..768def6b93 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -18,7 +18,10 @@ Changed (improvement) #4743 * Implement "continue" engine command to orquesta workflow. (improvement) #4740 * Update various internal dependencies to latest stable versions (apscheduler, eventlet, - kombu, amqp, pyyaml, mongoengine, python-gnupg, paramiko, tooz, webob, bcrypt). #4767 + kombu, amqp, pyyaml, mongoengine, python-gnupg, paramiko, tooz, webob, bcrypt). + + Latest version of mongoengine should show some performance improvements (5-20%) when + writing very large executions (executions with large results) to the database. #4767 Fixed ~~~~~ From 46b93bb2f5b823db0d0cdce58ff91230e16582c4 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Fri, 16 Aug 2019 21:31:19 +0200 Subject: [PATCH 6/6] Dummy commit to trigger build. --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 768def6b93..7080b72c6c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,7 @@ Changed Fixed ~~~~~ + * Fix rbac with execution view where the rbac is unable to verify the pack or uid of the execution because it was not returned from the action execution db. This would result in an internal server error when trying to view the results of a single execution.