diff --git a/BUILD b/BUILD index 389d4a2126..67c925d4cf 100644 --- a/BUILD +++ b/BUILD @@ -1,5 +1,37 @@ python_requirements( - name="root", + name="reqs", + source="requirements-pants.txt", + # module_mapping can be removed once pants is released with + # https://github.com/pantsbuild/pants/pull/17390 + module_mapping={ + "python-editor": ["editor"], + "python-json-logger": ["pythonjsonlogger"], + "python-statsd": ["statsd"], + "sseclient-py": ["sseclient"], + "oslo.config": ["oslo_config"], + "RandomWords": ["random_words"], + }, + overrides={ + # flex and stevedore uses pkg_resources w/o declaring the dep + ("flex", "stevedore"): { + "dependencies": [ + "//:reqs#setuptools", + ] + }, + # do not use the prance[flex] extra as that pulls in an old version of flex + "prance": { + "dependencies": [ + "//:reqs#flex", + ] + }, + # tooz needs one or more backends (tooz is used by the st2 coordination backend) + "tooz": { + "dependencies": [ + "//:reqs#redis", + "//:reqs#zake", + ] + }, + }, ) python_test_utils( diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d2ac9493f4..e7b64c505d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,7 +10,7 @@ Added * Continue introducing `pants `_ to improve DX (Developer Experience) working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. - #5778 + #5778 #5789 Contributed by @cognifloyd diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 24a2f3fe28..a71b6ad328 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -3,4 +3,7 @@ python_sources() python_requirements( name="reqs", source="requirements-tests.txt", + # module_mapping can be removed once pants is released with + # https://github.com/pantsbuild/pants/pull/17390 + module_mapping={"mail-parser": ["mailparser"]}, ) diff --git a/contrib/runners/winrm_runner/BUILD b/contrib/runners/winrm_runner/BUILD new file mode 100644 index 0000000000..a6be53dbd1 --- /dev/null +++ b/contrib/runners/winrm_runner/BUILD @@ -0,0 +1,7 @@ +python_requirement( + name="winrm", + requirements=["pywinrm"], + # modules can be removed once pants is released with + # https://github.com/pantsbuild/pants/pull/17390 + modules=["winrm"], +) diff --git a/lockfiles/st2-constraints.txt b/lockfiles/st2-constraints.txt new file mode 100644 index 0000000000..1383a2af96 --- /dev/null +++ b/lockfiles/st2-constraints.txt @@ -0,0 +1,113 @@ +# Add/remove version constraints for transitive dependencies in this file +# (transitive dependencies are dependencies of our direct dependencies). +# +# Direct dependencies should be recorded in `requirements-pants.txt`, not here. + +# please document each version constraint as follows: +# +# REQUIRED BY: , , ... +# REASON: +# NOTE: +# DROPS RESOLVED VERSION: +# + +# ############################################ # +# pinned transitive deps from requirements.txt # +# ############################################ # + +# REQUIRED BY: jinja2 +# REASON: Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode >=0.23 was from jinja2 +# NOTE: try to remove constraint later. +# DROPS RESOLVED VERSION: unknown +MarkupSafe<2.1.0,>=0.23 + +# REQUIRED BY: kombu +# REASON: unknown -- this looks like a lockfile-style pin +# NOTE: try to remove constraint later. +# DROPS RESOLVED VERSION: 5.1.1 +amqp==5.0.6 + +# REQUIRED BY: cryptography, paramiko, passlib +# REASON: unknown -- this looks like a lockfile-style pin +# NOTE: try to remove constraint later. +# DROPS RESOLVED VERSION: 4.0.1 +bcrypt==3.2.0 + +# REQUIRED BY: bcrypt, cryptography, pynacl, zstandard +# REASON: unknown +# NOTE: try to remove constraint later. +# DROPS RESOLVED VERSION: 1.15.1 +cffi<1.15.0 + +# REQUIRED BY: orquesta, prance, requests +# REASON: requests 2.23 requires chardet < 3.1.0 +# NOTE: orquesta already constrains this, so this is just documentation. +# DROPS RESOLVED VERSION: 3.0.4 +#chardet<3.1.0 + +# REQUIRED BY: jsonpath-rw, networkx +# REASON: +# networkx requires decorator>=4.3,<5 which should resolve to version 4.4.2 +# but the wheel on pypi does not say it supports python3.8, so pip gets +# confused. For now, pin decorator to work around pip's confusion. +# NOTE: Since pants/pex use a newer version of pip, this is not an issue. +# DROPS RESOLVED VERSION: 4.4.2 +#decorator==4.4.2 + +# REQUIRED BY: eventlet, pymongo +# REASON: 2.0 version breaks pymongo work with hosts +# NOTE: try to remove this later +# DROPS RESOLVED VERSION: 1.16 +dnspython>=1.16.0,<2.0.0 + +# REQUIRED BY: eventlet +# REASON: unknown -- this looks like a lockfile-style pin +# NOTE: We are having a hard time upgrading eventlet, so this pin is commented +# out to see if that will help. If any tests fail, uncomment this. +# DROPS RESOLVED VERSION: 1.1.3.post0 +#greenlet==1.0.0 + +# REQUIRED BY: argcomplete, click, debtcollector, kombu, pluggy, prettytable, +# pytest, virtualenv +# REASON: unknown +# NOTE: This pinned version (3.10.1) actually conflicts with other requirements. +# So, it is commented out. If there are issues with newer versions, +# update this with a range of valid versions. +# DROPS RESOLVED VERSION: 4.8.3 +#importlib-metadata==3.10.1 + +# REQUIRED BY: tooz +# REASON: unknown +# NOTE: try to remove constraint later. +# DROPS RESOLVED VERSION: 4.13 +oslo.utils<5.0,>=4.0.0 + +# REQUIRED BY: tooz +# REASON: unknown +# NOTE: try to remove constraint later. +# DROPS RESOLVED VERSION: 8.1 +tenacity>=3.2.1,<7.0.0 + +# REQUIRED BY: st2-auth-backend-flat-file +# REASON: unknown -- this looks like a lockfile-style pin +# NOTE: st2-auth-backend-flat-file has a version range >=1.7.1,<1.8.0 +# If we need to narrow that range, we should do so in: +# https://github.com/StackStorm/st2-auth-backend-flat-file/blob/master/requirements.txt +# DROPS RESOLVED VERSION: 1.7.4 +#passlib==1.7.4 + +# pyOpenSSL required by: pymongo[ocsp], redis[ocsp], urllib3[secure] +# but we don't use any of those, so skip copying from fixed-requirements.txt + +# REQUIRED BY: httplib2, oslo.utils, packaging +# REASON: unknown -- It looks like <3 was only needed for python2 compatibility. +# NOTE: this is still here, commented, until we can validate that all test are +# passing without it. +# DROPS RESOLVED VERSION: 3.0.7 +#pyparsing<3 + +# REQUIRED BY: async-timeout, gitpython, importlib-metadata, redis +# REASON: importlib-metadata requires typing-extensions but v4.2.0 requires py3.7+ +# NOTE: try to remove constraint later. +# DROPS RESOLVED VERSION: 4.1.1 +typing-extensions<4.2 diff --git a/pants.toml b/pants.toml index 1e128d6442..3ddbf7b88f 100644 --- a/pants.toml +++ b/pants.toml @@ -35,9 +35,9 @@ pants_ignore.add = [ "st2common/tests/fixtures/requirements-used-for-tests.txt", "/fixed-requirements.txt", "/test-requirements.txt", - # keep requirements.txt for now. We might ignore it if we need an alternate interrim - # file that is decoupled from our legacy requirements files generation. - # "/requirements.txt", + # ignore requirements.txt for now, preferring interim files that are decoupled from + # legacy requirements files generation: requirements-pants.txt & lockfiles/st2-constraints.txt + "/requirements.txt", ] [source] diff --git a/requirements-pants.txt b/requirements-pants.txt new file mode 100644 index 0000000000..f712a48a10 --- /dev/null +++ b/requirements-pants.txt @@ -0,0 +1,101 @@ +# Add/remove direct 3rd party dependencies here, with version constraints if necessary. +# +# Please do not add transitive dependencies in this file (ie dependencies of our dependencies). +# Use `lockfiles/st2-constraints.txt` to constrain the version of these transitive dependencies. +# +# Please keep this list alphabetical, with tooz backends in a separate list. + +apscheduler +argcomplete +ciso8601 +cryptography +# eventlet 0.31+ and gunicorn 20.1.0 are not compatible +eventlet<0.31 +# flex parses the openapi 2 spec in our router +flex +# gitpython & gitdb are used for pack management +gitdb +gitpython +# st2common/tests/integration/test_util_green.py requires greenlet (as does eventlet) +greenlet +gunicorn +jinja2 +jsonpath-rw +jsonschema +kombu +lockfile +mock +mongoengine +# Note: networkx v2.6 dropped support for Python3.6 +# networkx version is constrained in orquesta. +networkx +orjson +orquesta @ git+https://github.com/StackStorm/orquesta.git@v1.5.0 +# NOTE: Recent version substantially affect the performance and add big import time overhead +# See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details +oslo.config>=1.12.1,<1.13 +paramiko +# prance is used by st2-validate-api-spec to validate the openapi spec +# prance needs flex, but do not use the extra as that gets an old version. +prance +prettytable +# For st2client: prompt-toolkit v2+ does not have prompt_toolkit.token.Token +prompt-toolkit<2 +psutil +pymongo +# pyrabbit used in an integration test +pyrabbit +pytest +python-dateutil +python-editor +# pythonjsonlogger referenced in st2actions/conf/logging.conf +python-json-logger +python-statsd +pytz +PyYAML +# RandomWords used in some tests +RandomWords +requests[security] +retrying +routes +semver +# setuptools provides pkg_resources +setuptools +simplejson +six +# NOTE: we use sseclient-py instead of sseclient because sseclient +# has various issues which sometimes hang the connection for a long time, etc. +sseclient-py +# bandit doesn't work w/ stevedore 3+ +stevedore<3 +# For backward compatibility reasons, flat file backend is installed by default +st2-auth-backend-flat-file @ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master +st2-auth-ldap @ git+https://github.com/StackStorm/st2-auth-ldap.git@master +st2-rbac-backend @ git+https://github.com/StackStorm/st2-rbac-backend.git@master +# tabulate used by tools/log_watcher.py +tabulate +tooz +udatetime +ujson +unittest2 +virtualenv +webob +webtest +# zstandard is used for micro benchmarks +zstandard + +# tooz backends +redis +zake + +# was in fixed-requirements.txt, but not in requirements-pants.txt +# keyczar is used by a python2-only test. +#python-keyczar + +########### + +# not needed with switch to pytest +#nose +#nose-timer +#nose-parallel +#rednose