From 013d2a18946bd9d3436dcadc25339d8a08522ba0 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 24 Mar 2020 04:34:18 -0700 Subject: [PATCH 01/17] Handle environment markers in requirements.txt files --- scripts/fixate-requirements.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/fixate-requirements.py b/scripts/fixate-requirements.py index 3e712bc9c7..1aa892b7fb 100755 --- a/scripts/fixate-requirements.py +++ b/scripts/fixate-requirements.py @@ -171,10 +171,13 @@ def write_requirements(sources=None, fixed_requirements=None, output_file=None, rline = '-e %s' % (rline) elif req.req: project = req.name - if project in fixedreq_hash: - rline = str(fixedreq_hash[project].req) - else: - rline = str(req.req) + req_obj = fixedreq_hash.get(project, req) + + rline = str(req_obj.req) + + # Also write out environment markers + if req_obj.markers: + rline += " ; {}".format(str(req_obj.markers)) lines_to_write.append(rline) From 80ec7e2d7d1da26489c09600018a9cd052a9bf71 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 24 Mar 2020 04:35:16 -0700 Subject: [PATCH 02/17] Loosen the pip and setuptools requirements to we don't constantly bounce between installed versions --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4d68fa4972..c1c954e62e 100644 --- a/Makefile +++ b/Makefile @@ -439,8 +439,8 @@ requirements: virtualenv .sdist-requirements install-runners @echo # Make sure we use latest version of pip which is 19 $(VIRTUALENV_DIR)/bin/pip --version - $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==19.3.1" - $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==41.0.1" # Required for packs.install in dev envs + $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=19.3.1" + $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools>=41.0.1" # Required for packs.install in dev envs $(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3" # workaround for pbr issue # Generate all requirements to support current CI pipeline. From 3fad961d615f072b69b661d51387ca31835760b4 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 24 Mar 2020 04:35:34 -0700 Subject: [PATCH 03/17] Format makefile output a bit better --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c1c954e62e..43b73fceac 100644 --- a/Makefile +++ b/Makefile @@ -453,10 +453,11 @@ requirements: virtualenv .sdist-requirements install-runners @for component in $(COMPONENTS_WITH_RUNNERS); do\ echo "==========================================================="; \ echo "Generating requirements.txt for" $$component; \ - echo "==========================================================="; \ $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s $$component/in-requirements.txt -f fixed-requirements.txt -o $$component/requirements.txt; \ done + @echo "===========================================================" + # Fix for Travis CI race $(VIRTUALENV_DIR)/bin/pip install "six==1.12.0" From 8ab4db18a061200ec0b248a3193ef47c6195787c Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 24 Mar 2020 04:48:55 -0700 Subject: [PATCH 04/17] Partially alphabetize fixed-requirements.txt --- fixed-requirements.txt | 70 +++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index af2c1baf05..d0dfa3965b 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -1,61 +1,61 @@ # Packages versions fixed for the whole st2 stack -# Note: greenlet is used by eventlet -greenlet==0.4.15 +# Note: amqp is used by kombu +amqp==2.5.2 +apscheduler==3.6.3 +cryptography==2.8 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely eventlet==0.25.1 +flex==6.14.0 +gitpython==2.1.15 +# Note: greenlet is used by eventlet +greenlet==0.4.15 gunicorn==19.9.0 +jsonpath-rw==1.4.0 +jsonschema==2.6.0 kombu==4.6.6 -# Note: amqp is used by kombu -amqp==2.5.2 +lockfile==0.12.2 +mongoengine==0.18.2 +networkx==1.11 # NOTE: Recent version substantially affect the performance and add big import time overhead # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details oslo.config>=1.12.1,<1.13 oslo.utils>=3.36.2,<=3.37.0 -six==1.13.0 -pyyaml==5.1.2 -requests[security]==2.23.0 -apscheduler==3.6.3 -gitpython==2.1.15 -jsonschema==2.6.0 -pymongo==3.10.0 -mongoengine==0.18.2 +paramiko==2.6.0 passlib==1.7.1 -lockfile==0.12.2 -python-gnupg==0.4.5 -jsonpath-rw==1.4.0 +prance==0.9.0 +prompt-toolkit==1.0.15 pyinotify==0.9.6 -semver==2.9.0 -pytz==2019.1 -stevedore==1.30.1 -paramiko==2.6.0 -networkx==1.11 +pymongo==3.10.0 +python-editor==1.0.4 +python-gnupg==0.4.5 python-keyczar==0.716 -cryptography==2.8 +pytz==2019.1 +pywinrm==0.3.0 +pyyaml==5.1.2 +requests[security]==2.23.0 retrying==1.3.3 -# Note: We use latest version of virtualenv which uses pip 19 -virtualenv==16.6.0 +routes==2.4.1 +semver==2.9.0 +six==1.13.0 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc. sseclient-py==1.7 -python-editor==1.0.4 -prompt-toolkit==1.0.15 +stevedore==1.30.1 tooz==1.66.1 -zake==0.2.2 -routes==2.4.1 +# Note: We use latest version of virtualenv which uses pip 19 +virtualenv==16.6.0 webob==1.8.5 -flex==6.14.0 -prance==0.9.0 -pywinrm==0.3.0 +zake==0.2.2 # test requirements below +bcrypt==3.1.7 +jinja2==2.10.3 +mock==2.0.0 +more-itertools==5.0.0 nose-timer==0.7.5 nose-parallel==0.3.1 psutil==5.6.3 +python-dateutil==2.8.0 python-statsd==2.1.0 prometheus_client==0.1.1 -mock==2.0.0 ujson==1.35 -python-dateutil==2.8.0 -bcrypt==3.1.7 -jinja2==2.10.3 -more-itertools==5.0.0 zipp>=0.5,<=1.0.0 From 2dcdcf38031943f71e847dd7a768f85fb75280f6 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 24 Mar 2020 12:08:29 -0700 Subject: [PATCH 05/17] Pin pip and setuptools back to specific versions so we don't run into issues --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43b73fceac..3b85891769 100644 --- a/Makefile +++ b/Makefile @@ -439,8 +439,8 @@ requirements: virtualenv .sdist-requirements install-runners @echo # Make sure we use latest version of pip which is 19 $(VIRTUALENV_DIR)/bin/pip --version - $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=19.3.1" - $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools>=41.0.1" # Required for packs.install in dev envs + $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==19.3.1" + $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==41.0.1" # Required for packs.install in dev envs $(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3" # workaround for pbr issue # Generate all requirements to support current CI pipeline. From 821985dc95a2e7189c2714bd5ffd34fb6b155862 Mon Sep 17 00:00:00 2001 From: blag Date: Thu, 26 Mar 2020 00:30:08 -0700 Subject: [PATCH 06/17] Add improvement to CHANGELOG.rst --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aebe1b2371..e882a3afa3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -56,6 +56,8 @@ Changed * Remove `.scrutinizer.yml` config file. No longer used. * Convert escaped dict and dynamic fields in workflow db models to normal dict and dynamic fields. (performnce improvement) +* Add support for `PEP 508 `_ environment markers in generated ``requirements.txt`` files. + (improvement) #4895 Fixed ~~~~~ From 3aebb17a1d68cf5227c43a1773df5a7eb8d2ea31 Mon Sep 17 00:00:00 2001 From: blag Date: Thu, 26 Mar 2020 00:35:46 -0700 Subject: [PATCH 07/17] Fix backticked text for ReStructured Text (eg: not Markdown) --- CHANGELOG.rst | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e882a3afa3..d415d7337a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,7 +12,7 @@ Added #4757 * Add ``user`` parameter to ``re_run`` method of st2client. #4785 * Install pack dependencies automatically. #4769 -* Add support for `immutable_parameters` on Action Aliases. This feature allows default +* Add support for ``immutable_parameters`` on Action Aliases. This feature allows default parameters to be supplied to the action on every execution of the alias. #4786 * Add ``get_entrypoint()`` method to ``ActionResourceManager`` attribute of st2client. #4791 @@ -53,11 +53,11 @@ Changed connection related errors, our code would first wait for this timeout to be reached (30 seconds) before returning error to the end user. #4834 * Upgrade ``pymongo`` to the latest stable version (``3.10.0.``). #4835 (improvement) -* Remove `.scrutinizer.yml` config file. No longer used. +* Remove ``.scrutinizer.yml`` config file. No longer used. * Convert escaped dict and dynamic fields in workflow db models to normal dict and dynamic fields. (performnce improvement) -* Add support for `PEP 508 `_ environment markers in generated ``requirements.txt`` files. - (improvement) #4895 +* Add support for `PEP 508 `_ + environment markers in generated ``requirements.txt`` files. (improvement) #4895 Fixed ~~~~~ @@ -74,7 +74,7 @@ Fixed Contributed by Nick Maludy (@nmaludy Encore Technologies) * Fix the workflow execution cancelation to proceed even if the workflow execution is not found or completed. (bug fix) #4735 -* Added better error handling to `contrib/linux/actions/dig.py` to inform if dig is not installed. +* Added better error handling to ``contrib/linux/actions/dig.py`` to inform if dig is not installed. Contributed by JP Bourget (@punkrokk Syncurity) #4732 * Update ``dist_utils`` module which is bundled with ``st2client`` and other Python packages so it doesn't depend on internal pip API and so it works with latest pip version. (bug fix) #4750 @@ -1109,7 +1109,7 @@ Added after it has been scheduled. (new feature) #3867 * Added flag ``--auto-dict`` to ``st2 run`` and ``st2 execution re-run`` commands. This flag must now be specified in order to automatically convert list items to dicts based on presence of colon - (`:`) in all of the list items (new feature) #3909 + (``:``) in all of the list items (new feature) #3909 * Allow user to set default log level used by all the Python runner actions by setting ``actionrunner.pythonrunner```` option in ``st2.conf`` (new feature) #3929 * Update ``st2client`` package which is also utilized by the CLI so it also works under Python 3. @@ -1282,8 +1282,8 @@ Added client commands for interacting with Inquiries Contributed by mierdin. #3653 -* Added two new rule operators, `inside` and `ninside` which allow for the reverse intent of - the `contains` and `ncontains` operators. #3781 +* Added two new rule operators, ``inside`` and ``ninside`` which allow for the reverse intent of + the ``contains`` and ``ncontains`` operators. #3781 Contributed by @lampwins. * Allow user to use more expressive regular expressions inside action alias format string by @@ -1476,7 +1476,7 @@ Fixed This also fixes an issue with Redis kombu backend not working. (bug fix) #3635 #3639 #3648 * Fix logrotate configuration to delete stale compressed st2actionrunner logs #3647 -* Fix trace list API endpoint sorting by `start_timestamp`, using ``?sort_desc=True|False`` query +* Fix trace list API endpoint sorting by ``start_timestamp``, using ``?sort_desc=True|False`` query parameters and by passing ``--sort=asc|desc`` parameter to the ``st2 trace list`` CLI command. Descending order by default.(bug fix) #3237 #3665 * Fix pack index health endpoint. It now points to the right controller. #3672 @@ -1780,7 +1780,7 @@ Added Fixed ~~~~~ -* Fix ``st2ctl reload`` command so it preserves exit code from `st2-register-content` script and +* Fix ``st2ctl reload`` command so it preserves exit code from ``st2-register-content`` script and correctly fails on failure by default. * Fix base action alias test class (``BaseActionAliasTestCase``) so it also works if the local pack directory name doesn't match the pack name (this might be the case with new pack management @@ -1849,8 +1849,8 @@ Added (it's disabled by default) and if trigger object defines ``payload_schema`` attribute. Contribution by Hiroyasu OHYAMA. #3094 -* Add support for `st2 login` and `st2 whoami` commands. These add some additional functionality - beyond the existing `st2 auth` command and actually works with the local configuration so that +* Add support for ``st2 login`` and ``st2 whoami`` commands. These add some additional functionality + beyond the existing ``st2 auth`` command and actually works with the local configuration so that users do not have to. * Add support for complex rendering inside of array and object types. This allows the user to nest Jinja variables in array and object types. @@ -1988,7 +1988,7 @@ Added chain or Mistral workflows where waiting / sleeping is desired before proceeding with a next task. Contribution by Paul Mulvihill. (new feature) #2933. * Allow user to supply multiple resource ids using ``?id`` query parameter when filtering - "get all" API endpoint result set (e.g. `?id=1,2,3,4`). This allows for a better client and + "get all" API endpoint result set (e.g. ``?id=1,2,3,4``). This allows for a better client and servers performance when user is polling and interested in multiple resources such as polling on multiple action executions. (improvement) * Add support for ssh config file for ParamikoSSHrunner. Now ``ssh_config_file_path`` can be set @@ -2298,13 +2298,13 @@ Fixed parameter. Previously only raw key material was supported. (improvement) * Allow ``register-setup-virtualenvs`` flag to be used in combination with ``register-all`` in the ``st2-register-content`` script. -* Add missing `pytz` dependency to ``st2client`` requirements file. (bug-fix) +* Add missing ``pytz`` dependency to ``st2client`` requirements file. (bug-fix) * Fix datastore access on Python runner actions (set ``ST2_AUTH_TOKEN`` and ``ST2_API_URL`` env variables in Python runner actions to match sensors). (bug-fix) * Alias names are now correctly scoped to a pack. This means the same name for alias can be used across different packs. (bug-fix) * Fix a regression in filtering rules by pack with CLI. (bug-fix) -* Make sure `st2-submit-debug-info` cleans up after itself and deletes a temporary directory it +* Make sure ``st2-submit-debug-info`` cleans up after itself and deletes a temporary directory it creates. (improvement) #2714 [Kale Blankenship] * Fix string parameter casting - leave actual ``None`` value as-is and don't try to cast it to a @@ -2712,7 +2712,7 @@ Deprecated Fixed ~~~~~ -* Fix ``timestamp_lt`` and ``timestamp_gt`` filtering in the `/executions` API endpoint. Now we +* Fix ``timestamp_lt`` and ``timestamp_gt`` filtering in the ``/executions`` API endpoint. Now we return a correct result which is expected from a user-perspective. (bug-fix) * Make sure that alias execution endpoint returns a correct status code and error message if the referenced action doesn't exist. From dd53fddd53beae89bb27e4b0798603164b68fc8c Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 30 Mar 2020 15:16:53 -0700 Subject: [PATCH 08/17] Update setuptools to >=42 so it respects python_requires in configparser --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3b85891769..f8616af6f5 100644 --- a/Makefile +++ b/Makefile @@ -328,9 +328,9 @@ flake8: requirements .flake8 touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate - $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==19.3.1" + $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip>=19.3.1" # NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions - $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==41.0.1" + $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools>=42" $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd .. $(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version $(VIRTUALENV_ST2CLIENT_DIR)/bin/python -c "import st2client" From 749d7a80c53db2d05f9f218cd2b041b7d7d1d70c Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 30 Mar 2020 18:22:46 -0700 Subject: [PATCH 09/17] [skip ci] Add comments about version specifiers --- .travis.yml | 2 ++ Makefile | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 79dec62b3f..4756107dd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,6 +76,8 @@ cache: #- .tox/ before_install: + # If you update these versions, make sure you update the versions in the .st2client-install-check + # and the requirements targets in the Makefile to match - pip install --upgrade "pip>=19.0,<20.0" - sudo pip install --upgrade "virtualenv==16.6.0" diff --git a/Makefile b/Makefile index f8616af6f5..a5c556d059 100644 --- a/Makefile +++ b/Makefile @@ -328,8 +328,13 @@ flake8: requirements .flake8 touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate + # If you update these versions, make sure you also update the versions in the + # requirements target and .travis.yml to match + # Make sure we use the latest version of pip $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip>=19.3.1" # NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions + # Setuptools 42 added support for python_requires, which is used by the configparser package, + # which is required by the importlib-metadata package $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools>=42" $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd .. $(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version @@ -437,9 +442,13 @@ requirements: virtualenv .sdist-requirements install-runners @echo @echo "==================== requirements ====================" @echo - # Make sure we use latest version of pip which is 19 + # If you update these versions, make sure you also update the versions in the + # .st2client-install-check target and .travis.yml to match + # Make sure we use latest version of pip $(VIRTUALENV_DIR)/bin/pip --version $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==19.3.1" + # setuptools >= 41.0.1 is required for packs.install in dev envs + # setuptools >= 42 is required so setup.py install respects dependencies' python_requires $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==41.0.1" # Required for packs.install in dev envs $(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3" # workaround for pbr issue From a6c4b89373e01f1c38396ce65d9dda3e3be0e842 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 30 Mar 2020 18:23:25 -0700 Subject: [PATCH 10/17] Update pip and setuptools version specifiers everywhere --- .travis.yml | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4756107dd3..a8d868a83d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -78,7 +78,7 @@ cache: before_install: # If you update these versions, make sure you update the versions in the .st2client-install-check # and the requirements targets in the Makefile to match - - pip install --upgrade "pip>=19.0,<20.0" + - pip install --upgrade "pip>=19.3.1" - sudo pip install --upgrade "virtualenv==16.6.0" install: diff --git a/Makefile b/Makefile index a5c556d059..5675982ae3 100644 --- a/Makefile +++ b/Makefile @@ -446,10 +446,10 @@ requirements: virtualenv .sdist-requirements install-runners # .st2client-install-check target and .travis.yml to match # Make sure we use latest version of pip $(VIRTUALENV_DIR)/bin/pip --version - $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==19.3.1" + $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=19.3.1" # setuptools >= 41.0.1 is required for packs.install in dev envs # setuptools >= 42 is required so setup.py install respects dependencies' python_requires - $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==41.0.1" # Required for packs.install in dev envs + $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools>=42" $(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3" # workaround for pbr issue # Generate all requirements to support current CI pipeline. From 364955a8b59edfaac78bd990f10d5a136ec871a1 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 30 Mar 2020 18:25:04 -0700 Subject: [PATCH 11/17] [skip ci] Betterize Travis job descriptions --- .travis.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index a8d868a83d..eac901c32e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,21 +36,25 @@ jobs: # If you rename or reorder make targets in TASK, you may need to adjust: # scripts/travis/install-requirements.sh # scripts/travis/run-nightly-make-task-if-exists.sh - - env: TASK=ci-unit CACHE_NAME=py2 COMMAND_THRESHOLD=700 + - name: "Unit Tests (Python 2.7 MongoDB 3.4)" python: 2.7 - name: "Unit Tests (Python 2.7 MongoDB 3.4)" - - env: TASK=ci-integration CACHE_NAME=py2 COMMAND_THRESHOLD=700 + env: TASK=ci-unit CACHE_NAME=py2 COMMAND_THRESHOLD=700 + + - name: "Integration Tests (Python 2.7)" python: 2.7 - name: "Integration Tests (Python 2.7)" - - env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py2 COMMAND_THRESHOLD=430 + env: TASK=ci-integration CACHE_NAME=py2 COMMAND_THRESHOLD=700 + + - name: "Lint Checks, Packs Tests (Python 2.7)" python: 2.7 - name: "Lint Checks, Packs Tests (Python 2.7)" - - env: TASK="compilepy3 ci-py3-unit ci-py3-packs-tests" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=750 + env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py2 COMMAND_THRESHOLD=430 + + - name: "Unit Tests, Pack Tests (Python 3.6)" python: 3.6 - name: "Unit Tests, Pack Tests (Python 3.6)" - - env: TASK="ci-py3-integration" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=770 + env: TASK="compilepy3 ci-py3-unit ci-py3-packs-tests" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=750 + + - name: "Integration Tests (Python 3.6)" python: 3.6 - name: "Integration Tests (Python 3.6)" + env: TASK="ci-py3-integration" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=770 addons: apt: From 18ef428dac97958afb41d6bed939cc89469f27ed Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 30 Mar 2020 18:26:05 -0700 Subject: [PATCH 12/17] Only try to install pyinotify on Linux --- fixed-requirements.txt | 2 +- requirements.txt | 2 +- st2actions/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fixed-requirements.txt b/fixed-requirements.txt index d0dfa3965b..62770d76e2 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -25,7 +25,7 @@ paramiko==2.6.0 passlib==1.7.1 prance==0.9.0 prompt-toolkit==1.0.15 -pyinotify==0.9.6 +pyinotify==0.9.6; platform_system=="Linux" pymongo==3.10.0 python-editor==1.0.4 python-gnupg==0.4.5 diff --git a/requirements.txt b/requirements.txt index 3898401f16..40e8a6ea0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,7 @@ passlib==1.7.1 prettytable prompt-toolkit==1.0.15 psutil==5.6.3 -pyinotify==0.9.6 +pyinotify==0.9.6 ; platform_system == "Linux" pymongo==3.10.0 pyrabbit python-dateutil==2.8.0 diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index f102a5c49f..958215c561 100755 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -15,7 +15,7 @@ kombu==4.6.6 lockfile==0.12.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -pyinotify==0.9.6 +pyinotify==0.9.6 ; platform_system == "Linux" python-dateutil==2.8.0 python-json-logger pyyaml==5.1.2 From 5802a101ed934449ad7866c23a774e4a831227b5 Mon Sep 17 00:00:00 2001 From: blag Date: Mon, 30 Mar 2020 20:34:21 -0700 Subject: [PATCH 13/17] Make sure the fetch_requirements function also properly handles environment markers --- scripts/dist_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index aa3e4afdbf..45fa34a20c 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -115,6 +115,9 @@ def _get_link(line): else: req_name = line + if ';' in req_name: + req_name = req_name.split(';')[0].strip() + reqs.append(req_name) return (reqs, links) From 5817a24cd5a6db0441549a2353290d97c91c3fb1 Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 27 Mar 2020 00:02:46 -0700 Subject: [PATCH 14/17] Specify .git to remain consistent with st2actions/in-requirements.txt --- st2api/in-requirements.txt | 2 +- st2api/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/st2api/in-requirements.txt b/st2api/in-requirements.txt index 1ad7b3002e..76ea3129ee 100644 --- a/st2api/in-requirements.txt +++ b/st2api/in-requirements.txt @@ -8,5 +8,5 @@ oslo.config oslo.utils pymongo six -git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient +git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient gunicorn diff --git a/st2api/requirements.txt b/st2api/requirements.txt index 735c125d26..854a05e273 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt eventlet==0.25.1 -git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient +git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient gunicorn==19.9.0 jsonschema==2.6.0 kombu==4.6.6 From 498fbb20d116c237d9c37fb9218ea1627817a390 Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 27 Mar 2020 00:01:25 -0700 Subject: [PATCH 15/17] Use pip-compile from pip-tools to check for dependency version conflicts --- Makefile | 4 +++- test-requirements.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5675982ae3..5227584f91 100644 --- a/Makefile +++ b/Makefile @@ -505,7 +505,9 @@ requirements: virtualenv .sdist-requirements install-runners git submodule update --recursive --remote # Verify there are no conflicting dependencies - $(VIRTUALENV_DIR)/bin/pipconflictchecker + cat st2*/requirements.txt contrib/runners/*/requirements.txt | sort -u > req.txt && \ + $(VIRTUALENV_DIR)/bin/pip-compile req.txt; \ + if [[ -e req.txt ]]; then rm req.txt; fi .PHONY: virtualenv # Note: We always want to update virtualenv/bin/activate file to make sure diff --git a/test-requirements.txt b/test-requirements.txt index db927f9997..c4970ef10e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -32,4 +32,4 @@ pyrabbit # (those tests only run under Python 2.7 since keyczar doesn't support Python 3.x). # See https://github.com/StackStorm/st2/pull/4165 python-keyczar -git+https://github.com/StackStorm/pip-conflict-checker.git@fix_pip_issue#egg=pip-conflict-checker +pip-tools # For pip-compile, to check for version conflicts From 42caba6109428a183a015d1b1ca1d2be96834a73 Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 27 Mar 2020 13:59:09 -0700 Subject: [PATCH 16/17] Update the CHANGELOG --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d415d7337a..6e451217ca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -58,6 +58,7 @@ Changed (performnce improvement) * Add support for `PEP 508 `_ environment markers in generated ``requirements.txt`` files. (improvement) #4895 +* Use ``pip-compile`` from ``pip-tools`` instead of ``pip-conflict-checker`` (improvement) #4896 Fixed ~~~~~ From 4baf4b73d3ec913cb3cdd3333178f343bdae2e3b Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 31 Mar 2020 16:38:54 -0700 Subject: [PATCH 17/17] [skip ci] Add comment to look into using the packaging package on PyPI --- scripts/dist_utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index 45fa34a20c..0aa90ec445 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -22,6 +22,15 @@ from distutils.version import StrictVersion # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here +# +# TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import +# from pip? +# +# TODO: Dear future developer, if you are back here fixing a bug with how we parse +# requirements files, please look into using the packaging package on PyPI: +# https://packaging.pypa.io/en/latest/requirements/ +# At the very least we can vendorize some of their code instead of reimplementing +# each piece of their code every time our parsing breaks. PY3 = sys.version_info[0] == 3 if PY3: