Various requirements fixups

.travis.yml

@@ -36,21 +36,25 @@ jobs:
     # If you rename or reorder make targets in TASK, you may need to adjust:
     # scripts/travis/install-requirements.sh
     # scripts/travis/run-nightly-make-task-if-exists.sh
-    - env: TASK=ci-unit CACHE_NAME=py2 COMMAND_THRESHOLD=700
+    - name: "Unit Tests (Python 2.7 MongoDB 3.4)"
       python: 2.7
-      name: "Unit Tests (Python 2.7 MongoDB 3.4)"
-    - env: TASK=ci-integration CACHE_NAME=py2 COMMAND_THRESHOLD=700
+      env: TASK=ci-unit CACHE_NAME=py2 COMMAND_THRESHOLD=700
+
+    - name: "Integration Tests (Python 2.7)"
       python: 2.7
-      name: "Integration Tests (Python 2.7)"
-    - env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py2 COMMAND_THRESHOLD=430
+      env: TASK=ci-integration CACHE_NAME=py2 COMMAND_THRESHOLD=700
+
+    - name: "Lint Checks, Packs Tests (Python 2.7)"
       python: 2.7
-      name: "Lint Checks, Packs Tests (Python 2.7)"
-    - env: TASK="compilepy3 ci-py3-unit ci-py3-packs-tests" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=750
+      env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py2 COMMAND_THRESHOLD=430
+
+    - name: "Unit Tests, Pack Tests (Python 3.6)"
       python: 3.6
-      name: "Unit Tests, Pack Tests (Python 3.6)"
-    - env: TASK="ci-py3-integration" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=770
+      env: TASK="compilepy3 ci-py3-unit ci-py3-packs-tests" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=750
+
+    - name: "Integration Tests (Python 3.6)"
       python: 3.6
-      name: "Integration Tests (Python 3.6)"
+      env: TASK="ci-py3-integration" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=770
 
 addons:
   apt:
@@ -76,7 +80,9 @@ cache:
     #- .tox/
 
 before_install:
-  - pip install --upgrade "pip>=19.0,<20.0"
+  # If you update these versions, make sure you update the versions in the .st2client-install-check
+  # and the requirements targets in the Makefile to match
+  - pip install --upgrade "pip>=19.3.1"
   - sudo pip install --upgrade "virtualenv==16.6.0"
 
 install:

CHANGELOG.rst

@@ -12,7 +12,7 @@ Added
   #4757
 * Add ``user`` parameter to ``re_run`` method of st2client. #4785
 * Install pack dependencies automatically. #4769
-* Add support for `immutable_parameters` on Action Aliases. This feature allows default
+* Add support for ``immutable_parameters`` on Action Aliases. This feature allows default
   parameters to be supplied to the action on every execution of the alias. #4786
 * Add ``get_entrypoint()`` method to ``ActionResourceManager`` attribute of st2client.
   #4791
@@ -53,9 +53,12 @@ Changed
   connection related errors, our code would first wait for this timeout to be reached (30 seconds)
   before returning error to the end user. #4834
 * Upgrade ``pymongo`` to the latest stable version (``3.10.0.``). #4835 (improvement)
-* Remove `.scrutinizer.yml` config file. No longer used.
+* Remove ``.scrutinizer.yml`` config file. No longer used.
 * Convert escaped dict and dynamic fields in workflow db models to normal dict and dynamic fields.
   (performnce improvement)
+* Add support for `PEP 508 <https://www.python.org/dev/peps/pep-0508/#environment-markers>`_
+  environment markers in generated ``requirements.txt`` files. (improvement) #4895
+* Use ``pip-compile`` from ``pip-tools`` instead of ``pip-conflict-checker`` (improvement) #4896
 
 Fixed
 ~~~~~
@@ -72,7 +75,7 @@ Fixed
   Contributed by Nick Maludy (@nmaludy Encore Technologies)
 * Fix the workflow execution cancelation to proceed even if the workflow execution is not found or
   completed. (bug fix) #4735
-* Added better error handling to `contrib/linux/actions/dig.py` to inform if dig is not installed.
+* Added better error handling to ``contrib/linux/actions/dig.py`` to inform if dig is not installed.
   Contributed by JP Bourget (@punkrokk Syncurity) #4732
 * Update ``dist_utils`` module which is bundled with ``st2client`` and other Python packages so it
   doesn't depend on internal pip API and so it works with latest pip version. (bug fix) #4750
@@ -1107,7 +1110,7 @@ Added
   after it has been scheduled. (new feature) #3867
 * Added flag ``--auto-dict`` to ``st2 run`` and ``st2 execution re-run`` commands. This flag must now
   be specified in order to automatically convert list items to dicts based on presence of colon
-  (`:`) in all of the list items (new feature) #3909
+  (``:``) in all of the list items (new feature) #3909
 * Allow user to set default log level used by all the Python runner actions by setting
   ``actionrunner.pythonrunner```` option in ``st2.conf`` (new feature) #3929
 * Update ``st2client`` package which is also utilized by the CLI so it also works under Python 3.
@@ -1280,8 +1283,8 @@ Added
   client commands for interacting with Inquiries
 
   Contributed by mierdin. #3653
-* Added two new rule operators, `inside` and `ninside` which allow for the reverse intent of
-  the `contains` and `ncontains` operators. #3781
+* Added two new rule operators, ``inside`` and ``ninside`` which allow for the reverse intent of
+  the ``contains`` and ``ncontains`` operators. #3781
 
   Contributed by @lampwins.
 * Allow user to use more expressive regular expressions inside action alias format string by
@@ -1474,7 +1477,7 @@ Fixed
 
   This also fixes an issue with Redis kombu backend not working. (bug fix) #3635 #3639 #3648
 * Fix logrotate configuration to delete stale compressed st2actionrunner logs #3647
-* Fix trace list API endpoint sorting by `start_timestamp`, using ``?sort_desc=True|False`` query
+* Fix trace list API endpoint sorting by ``start_timestamp``, using ``?sort_desc=True|False`` query
   parameters and by passing ``--sort=asc|desc`` parameter to the ``st2 trace list`` CLI command.
   Descending order by default.(bug fix) #3237 #3665
 * Fix pack index health endpoint. It now points to the right controller. #3672
@@ -1778,7 +1781,7 @@ Added
 Fixed
 ~~~~~
 
-* Fix ``st2ctl reload`` command so it preserves exit code from `st2-register-content` script and
+* Fix ``st2ctl reload`` command so it preserves exit code from ``st2-register-content`` script and
   correctly fails on failure by default.
 * Fix base action alias test class (``BaseActionAliasTestCase``) so it also works if the local pack
   directory name doesn't match the pack name (this might be the case with new pack management
@@ -1847,8 +1850,8 @@ Added
   (it's disabled by default) and if trigger object defines ``payload_schema`` attribute.
 
   Contribution by Hiroyasu OHYAMA. #3094
-* Add support for `st2 login` and `st2 whoami` commands. These add some additional functionality
-  beyond the existing `st2 auth` command and actually works with the local configuration so that
+* Add support for ``st2 login`` and ``st2 whoami`` commands. These add some additional functionality
+  beyond the existing ``st2 auth`` command and actually works with the local configuration so that
   users do not have to.
 * Add support for complex rendering inside of array and object types. This allows the user to
   nest Jinja variables in array and object types.
@@ -1986,7 +1989,7 @@ Added
   chain or Mistral workflows where waiting / sleeping is desired before proceeding with a next
   task. Contribution by Paul Mulvihill. (new feature) #2933.
 * Allow user to supply multiple resource ids using ``?id`` query parameter when filtering
-  "get all" API endpoint result set (e.g. `?id=1,2,3,4`). This allows for a better client and
+  "get all" API endpoint result set (e.g. ``?id=1,2,3,4``). This allows for a better client and
   servers performance when user is polling and interested in multiple resources such as polling on
   multiple action executions. (improvement)
 * Add support for ssh config file for ParamikoSSHrunner. Now ``ssh_config_file_path`` can be set
@@ -2296,13 +2299,13 @@ Fixed
   parameter. Previously only raw key material was supported. (improvement)
 * Allow ``register-setup-virtualenvs`` flag to be used in combination with ``register-all`` in the
   ``st2-register-content`` script.
-* Add missing `pytz` dependency to ``st2client`` requirements file. (bug-fix)
+* Add missing ``pytz`` dependency to ``st2client`` requirements file. (bug-fix)
 * Fix datastore access on Python runner actions (set ``ST2_AUTH_TOKEN`` and ``ST2_API_URL`` env
   variables in Python runner actions to match sensors). (bug-fix)
 * Alias names are now correctly scoped to a pack. This means the same name for alias can be used
   across different packs. (bug-fix)
 * Fix a regression in filtering rules by pack with CLI. (bug-fix)
-* Make sure `st2-submit-debug-info` cleans up after itself and deletes a temporary directory it
+* Make sure ``st2-submit-debug-info`` cleans up after itself and deletes a temporary directory it
   creates. (improvement) #2714
   [Kale Blankenship]
 * Fix string parameter casting - leave actual ``None`` value as-is and don't try to cast it to a
@@ -2710,7 +2713,7 @@ Deprecated
 Fixed
 ~~~~~
 
-* Fix ``timestamp_lt`` and ``timestamp_gt`` filtering in the `/executions` API endpoint. Now we
+* Fix ``timestamp_lt`` and ``timestamp_gt`` filtering in the ``/executions`` API endpoint. Now we
   return a correct result which is expected from a user-perspective. (bug-fix)
 * Make sure that alias execution endpoint returns a correct status code and error message if the
   referenced action doesn't exist.

Makefile

@@ -328,9 +328,14 @@ flake8: requirements .flake8
 	touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 	chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 
-	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==19.3.1"
+	# If you update these versions, make sure you also update the versions in the
+	# requirements target and .travis.yml to match
+	# Make sure we use the latest version of pip
+	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip>=19.3.1"
 	# NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions
-	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==41.0.1"
+	# Setuptools 42 added support for python_requires, which is used by the configparser package,
+	# which is required by the importlib-metadata package
+	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools>=42"
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd ..
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/python -c "import st2client"
@@ -437,10 +442,14 @@ requirements: virtualenv .sdist-requirements install-runners
 	@echo
 	@echo "==================== requirements ===================="
 	@echo
-	# Make sure we use latest version of pip which is 19
+	# If you update these versions, make sure you also update the versions in the
+	# .st2client-install-check target and .travis.yml to match
+	# Make sure we use latest version of pip
 	$(VIRTUALENV_DIR)/bin/pip --version
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==19.3.1"
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==41.0.1"  # Required for packs.install in dev envs
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=19.3.1"
+	# setuptools >= 41.0.1 is required for packs.install in dev envs
+	# setuptools >= 42     is required so setup.py install respects dependencies' python_requires
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools>=42"
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
 
 	# Generate all requirements to support current CI pipeline.
@@ -453,10 +462,11 @@ requirements: virtualenv .sdist-requirements install-runners
 	@for component in $(COMPONENTS_WITH_RUNNERS); do\
 		echo "==========================================================="; \
 		echo "Generating requirements.txt for" $$component; \
-		echo "==========================================================="; \
 		$(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s $$component/in-requirements.txt -f fixed-requirements.txt -o $$component/requirements.txt; \
 	done
 
+	@echo "==========================================================="
+
 	# Fix for Travis CI race
 	$(VIRTUALENV_DIR)/bin/pip install "six==1.12.0"
 
@@ -495,7 +505,9 @@ requirements: virtualenv .sdist-requirements install-runners
 	git submodule update --recursive --remote
 
 	# Verify there are no conflicting dependencies
-	$(VIRTUALENV_DIR)/bin/pipconflictchecker
+	cat st2*/requirements.txt contrib/runners/*/requirements.txt | sort -u > req.txt && \
+	$(VIRTUALENV_DIR)/bin/pip-compile req.txt; \
+	if [[ -e req.txt ]]; then rm req.txt; fi
 
 .PHONY: virtualenv
 	# Note: We always want to update virtualenv/bin/activate file to make sure

fixed-requirements.txt

@@ -1,61 +1,61 @@
 # Packages versions fixed for the whole st2 stack
-# Note: greenlet is used by eventlet
-greenlet==0.4.15
+# Note: amqp is used by kombu
+amqp==2.5.2
+apscheduler==3.6.3
+cryptography==2.8
 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we
 # depend on rely
 eventlet==0.25.1
+flex==6.14.0
+gitpython==2.1.15
+# Note: greenlet is used by eventlet
+greenlet==0.4.15
 gunicorn==19.9.0
+jsonpath-rw==1.4.0
+jsonschema==2.6.0
 kombu==4.6.6
-# Note: amqp is used by kombu
-amqp==2.5.2
+lockfile==0.12.2
+mongoengine==0.18.2
+networkx==1.11
 # NOTE: Recent version substantially affect the performance and add big import time overhead
 # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details
 oslo.config>=1.12.1,<1.13
 oslo.utils>=3.36.2,<=3.37.0
-six==1.13.0
-pyyaml==5.1.2
-requests[security]==2.23.0
-apscheduler==3.6.3
-gitpython==2.1.15
-jsonschema==2.6.0
-pymongo==3.10.0
-mongoengine==0.18.2
+paramiko==2.6.0
 passlib==1.7.1
-lockfile==0.12.2
+prance==0.9.0
+prompt-toolkit==1.0.15
+pyinotify==0.9.6; platform_system=="Linux"
+pymongo==3.10.0
+python-editor==1.0.4
 python-gnupg==0.4.5
-jsonpath-rw==1.4.0
-pyinotify==0.9.6
-semver==2.9.0
-pytz==2019.1
-stevedore==1.30.1
-paramiko==2.6.0
-networkx==1.11
 python-keyczar==0.716
-cryptography==2.8
+pytz==2019.1
+pywinrm==0.3.0
+pyyaml==5.1.2
+requests[security]==2.23.0
 retrying==1.3.3
-# Note: We use latest version of virtualenv which uses pip 19
-virtualenv==16.6.0
+routes==2.4.1
+semver==2.9.0
+six==1.13.0
 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc.
 sseclient-py==1.7
-python-editor==1.0.4
-prompt-toolkit==1.0.15
+stevedore==1.30.1
 tooz==1.66.1
-zake==0.2.2
-routes==2.4.1
+# Note: We use latest version of virtualenv which uses pip 19
+virtualenv==16.6.0
 webob==1.8.5
-flex==6.14.0
-prance==0.9.0
-pywinrm==0.3.0
+zake==0.2.2
 # test requirements below
+bcrypt==3.1.7
+jinja2==2.10.3
+mock==2.0.0
+more-itertools==5.0.0
 nose-timer==0.7.5
 nose-parallel==0.3.1
 psutil==5.6.3
+python-dateutil==2.8.0
 python-statsd==2.1.0
 prometheus_client==0.1.1
-mock==2.0.0
 ujson==1.35
-python-dateutil==2.8.0
-bcrypt==3.1.7
-jinja2==2.10.3
-more-itertools==5.0.0
 zipp>=0.5,<=1.0.0

requirements.txt

@@ -40,7 +40,7 @@ passlib==1.7.1
 prettytable
 prompt-toolkit==1.0.15
 psutil==5.6.3
-pyinotify==0.9.6
+pyinotify==0.9.6 ; platform_system == "Linux"
 pymongo==3.10.0
 pyrabbit
 python-dateutil==2.8.0

scripts/dist_utils.py

@@ -22,6 +22,15 @@
 from distutils.version import StrictVersion
 
 # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here
+#
+# TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import
+#       from pip?
+#
+# TODO: Dear future developer, if you are back here fixing a bug with how we parse
+#       requirements files, please look into using the packaging package on PyPI:
+#       https://packaging.pypa.io/en/latest/requirements/
+#       At the very least we can vendorize some of their code instead of reimplementing
+#       each piece of their code every time our parsing breaks.
 PY3 = sys.version_info[0] == 3
 
 if PY3:
@@ -115,6 +124,9 @@ def _get_link(line):
             else:
                 req_name = line
 
+                if ';' in req_name:
+                    req_name = req_name.split(';')[0].strip()
+
             reqs.append(req_name)
 
     return (reqs, links)

scripts/fixate-requirements.py

@@ -171,10 +171,13 @@ def write_requirements(sources=None, fixed_requirements=None, output_file=None,
                 rline = '-e %s' % (rline)
         elif req.req:
             project = req.name
-            if project in fixedreq_hash:
-                rline = str(fixedreq_hash[project].req)
-            else:
-                rline = str(req.req)
+            req_obj = fixedreq_hash.get(project, req)
+
+            rline = str(req_obj.req)
+
+            # Also write out environment markers
+            if req_obj.markers:
+                rline += " ; {}".format(str(req_obj.markers))
 
         lines_to_write.append(rline)
 

st2actions/requirements.txt

@@ -15,7 +15,7 @@ kombu==4.6.6
 lockfile==0.12.2
 oslo.config<1.13,>=1.12.1
 oslo.utils<=3.37.0,>=3.36.2
-pyinotify==0.9.6
+pyinotify==0.9.6 ; platform_system == "Linux"
 python-dateutil==2.8.0
 python-json-logger
 pyyaml==5.1.2

st2api/in-requirements.txt

@@ -8,5 +8,5 @@ oslo.config
 oslo.utils
 pymongo
 six
-git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient
+git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient
 gunicorn

st2api/requirements.txt

@@ -6,7 +6,7 @@
 # in-requirements.txt for that component and then run 'make requirements' to
 # update the component requirements.txt
 eventlet==0.25.1
-git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient
+git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient
 gunicorn==19.9.0
 jsonschema==2.6.0
 kombu==4.6.6