diff --git a/.travis.yml b/.travis.yml index 79dec62b3f..eac901c32e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,21 +36,25 @@ jobs: # If you rename or reorder make targets in TASK, you may need to adjust: # scripts/travis/install-requirements.sh # scripts/travis/run-nightly-make-task-if-exists.sh - - env: TASK=ci-unit CACHE_NAME=py2 COMMAND_THRESHOLD=700 + - name: "Unit Tests (Python 2.7 MongoDB 3.4)" python: 2.7 - name: "Unit Tests (Python 2.7 MongoDB 3.4)" - - env: TASK=ci-integration CACHE_NAME=py2 COMMAND_THRESHOLD=700 + env: TASK=ci-unit CACHE_NAME=py2 COMMAND_THRESHOLD=700 + + - name: "Integration Tests (Python 2.7)" python: 2.7 - name: "Integration Tests (Python 2.7)" - - env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py2 COMMAND_THRESHOLD=430 + env: TASK=ci-integration CACHE_NAME=py2 COMMAND_THRESHOLD=700 + + - name: "Lint Checks, Packs Tests (Python 2.7)" python: 2.7 - name: "Lint Checks, Packs Tests (Python 2.7)" - - env: TASK="compilepy3 ci-py3-unit ci-py3-packs-tests" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=750 + env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py2 COMMAND_THRESHOLD=430 + + - name: "Unit Tests, Pack Tests (Python 3.6)" python: 3.6 - name: "Unit Tests, Pack Tests (Python 3.6)" - - env: TASK="ci-py3-integration" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=770 + env: TASK="compilepy3 ci-py3-unit ci-py3-packs-tests" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=750 + + - name: "Integration Tests (Python 3.6)" python: 3.6 - name: "Integration Tests (Python 3.6)" + env: TASK="ci-py3-integration" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=770 addons: apt: @@ -76,7 +80,9 @@ cache: #- .tox/ before_install: - - pip install --upgrade "pip>=19.0,<20.0" + # If you update these versions, make sure you update the versions in the .st2client-install-check + # and the requirements targets in the Makefile to match + - pip install --upgrade "pip>=19.3.1" - sudo pip install --upgrade "virtualenv==16.6.0" install: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aebe1b2371..6e451217ca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,7 +12,7 @@ Added #4757 * Add ``user`` parameter to ``re_run`` method of st2client. #4785 * Install pack dependencies automatically. #4769 -* Add support for `immutable_parameters` on Action Aliases. This feature allows default +* Add support for ``immutable_parameters`` on Action Aliases. This feature allows default parameters to be supplied to the action on every execution of the alias. #4786 * Add ``get_entrypoint()`` method to ``ActionResourceManager`` attribute of st2client. #4791 @@ -53,9 +53,12 @@ Changed connection related errors, our code would first wait for this timeout to be reached (30 seconds) before returning error to the end user. #4834 * Upgrade ``pymongo`` to the latest stable version (``3.10.0.``). #4835 (improvement) -* Remove `.scrutinizer.yml` config file. No longer used. +* Remove ``.scrutinizer.yml`` config file. No longer used. * Convert escaped dict and dynamic fields in workflow db models to normal dict and dynamic fields. (performnce improvement) +* Add support for `PEP 508 `_ + environment markers in generated ``requirements.txt`` files. (improvement) #4895 +* Use ``pip-compile`` from ``pip-tools`` instead of ``pip-conflict-checker`` (improvement) #4896 Fixed ~~~~~ @@ -72,7 +75,7 @@ Fixed Contributed by Nick Maludy (@nmaludy Encore Technologies) * Fix the workflow execution cancelation to proceed even if the workflow execution is not found or completed. (bug fix) #4735 -* Added better error handling to `contrib/linux/actions/dig.py` to inform if dig is not installed. +* Added better error handling to ``contrib/linux/actions/dig.py`` to inform if dig is not installed. Contributed by JP Bourget (@punkrokk Syncurity) #4732 * Update ``dist_utils`` module which is bundled with ``st2client`` and other Python packages so it doesn't depend on internal pip API and so it works with latest pip version. (bug fix) #4750 @@ -1107,7 +1110,7 @@ Added after it has been scheduled. (new feature) #3867 * Added flag ``--auto-dict`` to ``st2 run`` and ``st2 execution re-run`` commands. This flag must now be specified in order to automatically convert list items to dicts based on presence of colon - (`:`) in all of the list items (new feature) #3909 + (``:``) in all of the list items (new feature) #3909 * Allow user to set default log level used by all the Python runner actions by setting ``actionrunner.pythonrunner```` option in ``st2.conf`` (new feature) #3929 * Update ``st2client`` package which is also utilized by the CLI so it also works under Python 3. @@ -1280,8 +1283,8 @@ Added client commands for interacting with Inquiries Contributed by mierdin. #3653 -* Added two new rule operators, `inside` and `ninside` which allow for the reverse intent of - the `contains` and `ncontains` operators. #3781 +* Added two new rule operators, ``inside`` and ``ninside`` which allow for the reverse intent of + the ``contains`` and ``ncontains`` operators. #3781 Contributed by @lampwins. * Allow user to use more expressive regular expressions inside action alias format string by @@ -1474,7 +1477,7 @@ Fixed This also fixes an issue with Redis kombu backend not working. (bug fix) #3635 #3639 #3648 * Fix logrotate configuration to delete stale compressed st2actionrunner logs #3647 -* Fix trace list API endpoint sorting by `start_timestamp`, using ``?sort_desc=True|False`` query +* Fix trace list API endpoint sorting by ``start_timestamp``, using ``?sort_desc=True|False`` query parameters and by passing ``--sort=asc|desc`` parameter to the ``st2 trace list`` CLI command. Descending order by default.(bug fix) #3237 #3665 * Fix pack index health endpoint. It now points to the right controller. #3672 @@ -1778,7 +1781,7 @@ Added Fixed ~~~~~ -* Fix ``st2ctl reload`` command so it preserves exit code from `st2-register-content` script and +* Fix ``st2ctl reload`` command so it preserves exit code from ``st2-register-content`` script and correctly fails on failure by default. * Fix base action alias test class (``BaseActionAliasTestCase``) so it also works if the local pack directory name doesn't match the pack name (this might be the case with new pack management @@ -1847,8 +1850,8 @@ Added (it's disabled by default) and if trigger object defines ``payload_schema`` attribute. Contribution by Hiroyasu OHYAMA. #3094 -* Add support for `st2 login` and `st2 whoami` commands. These add some additional functionality - beyond the existing `st2 auth` command and actually works with the local configuration so that +* Add support for ``st2 login`` and ``st2 whoami`` commands. These add some additional functionality + beyond the existing ``st2 auth`` command and actually works with the local configuration so that users do not have to. * Add support for complex rendering inside of array and object types. This allows the user to nest Jinja variables in array and object types. @@ -1986,7 +1989,7 @@ Added chain or Mistral workflows where waiting / sleeping is desired before proceeding with a next task. Contribution by Paul Mulvihill. (new feature) #2933. * Allow user to supply multiple resource ids using ``?id`` query parameter when filtering - "get all" API endpoint result set (e.g. `?id=1,2,3,4`). This allows for a better client and + "get all" API endpoint result set (e.g. ``?id=1,2,3,4``). This allows for a better client and servers performance when user is polling and interested in multiple resources such as polling on multiple action executions. (improvement) * Add support for ssh config file for ParamikoSSHrunner. Now ``ssh_config_file_path`` can be set @@ -2296,13 +2299,13 @@ Fixed parameter. Previously only raw key material was supported. (improvement) * Allow ``register-setup-virtualenvs`` flag to be used in combination with ``register-all`` in the ``st2-register-content`` script. -* Add missing `pytz` dependency to ``st2client`` requirements file. (bug-fix) +* Add missing ``pytz`` dependency to ``st2client`` requirements file. (bug-fix) * Fix datastore access on Python runner actions (set ``ST2_AUTH_TOKEN`` and ``ST2_API_URL`` env variables in Python runner actions to match sensors). (bug-fix) * Alias names are now correctly scoped to a pack. This means the same name for alias can be used across different packs. (bug-fix) * Fix a regression in filtering rules by pack with CLI. (bug-fix) -* Make sure `st2-submit-debug-info` cleans up after itself and deletes a temporary directory it +* Make sure ``st2-submit-debug-info`` cleans up after itself and deletes a temporary directory it creates. (improvement) #2714 [Kale Blankenship] * Fix string parameter casting - leave actual ``None`` value as-is and don't try to cast it to a @@ -2710,7 +2713,7 @@ Deprecated Fixed ~~~~~ -* Fix ``timestamp_lt`` and ``timestamp_gt`` filtering in the `/executions` API endpoint. Now we +* Fix ``timestamp_lt`` and ``timestamp_gt`` filtering in the ``/executions`` API endpoint. Now we return a correct result which is expected from a user-perspective. (bug-fix) * Make sure that alias execution endpoint returns a correct status code and error message if the referenced action doesn't exist. diff --git a/Makefile b/Makefile index 4d68fa4972..5227584f91 100644 --- a/Makefile +++ b/Makefile @@ -328,9 +328,14 @@ flake8: requirements .flake8 touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate - $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==19.3.1" + # If you update these versions, make sure you also update the versions in the + # requirements target and .travis.yml to match + # Make sure we use the latest version of pip + $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip>=19.3.1" # NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions - $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==41.0.1" + # Setuptools 42 added support for python_requires, which is used by the configparser package, + # which is required by the importlib-metadata package + $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools>=42" $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd .. $(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version $(VIRTUALENV_ST2CLIENT_DIR)/bin/python -c "import st2client" @@ -437,10 +442,14 @@ requirements: virtualenv .sdist-requirements install-runners @echo @echo "==================== requirements ====================" @echo - # Make sure we use latest version of pip which is 19 + # If you update these versions, make sure you also update the versions in the + # .st2client-install-check target and .travis.yml to match + # Make sure we use latest version of pip $(VIRTUALENV_DIR)/bin/pip --version - $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==19.3.1" - $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==41.0.1" # Required for packs.install in dev envs + $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip>=19.3.1" + # setuptools >= 41.0.1 is required for packs.install in dev envs + # setuptools >= 42 is required so setup.py install respects dependencies' python_requires + $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools>=42" $(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3" # workaround for pbr issue # Generate all requirements to support current CI pipeline. @@ -453,10 +462,11 @@ requirements: virtualenv .sdist-requirements install-runners @for component in $(COMPONENTS_WITH_RUNNERS); do\ echo "==========================================================="; \ echo "Generating requirements.txt for" $$component; \ - echo "==========================================================="; \ $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s $$component/in-requirements.txt -f fixed-requirements.txt -o $$component/requirements.txt; \ done + @echo "===========================================================" + # Fix for Travis CI race $(VIRTUALENV_DIR)/bin/pip install "six==1.12.0" @@ -495,7 +505,9 @@ requirements: virtualenv .sdist-requirements install-runners git submodule update --recursive --remote # Verify there are no conflicting dependencies - $(VIRTUALENV_DIR)/bin/pipconflictchecker + cat st2*/requirements.txt contrib/runners/*/requirements.txt | sort -u > req.txt && \ + $(VIRTUALENV_DIR)/bin/pip-compile req.txt; \ + if [[ -e req.txt ]]; then rm req.txt; fi .PHONY: virtualenv # Note: We always want to update virtualenv/bin/activate file to make sure diff --git a/fixed-requirements.txt b/fixed-requirements.txt index af2c1baf05..62770d76e2 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -1,61 +1,61 @@ # Packages versions fixed for the whole st2 stack -# Note: greenlet is used by eventlet -greenlet==0.4.15 +# Note: amqp is used by kombu +amqp==2.5.2 +apscheduler==3.6.3 +cryptography==2.8 # Note: 0.20.0 removed select.poll() on which some of our code and libraries we # depend on rely eventlet==0.25.1 +flex==6.14.0 +gitpython==2.1.15 +# Note: greenlet is used by eventlet +greenlet==0.4.15 gunicorn==19.9.0 +jsonpath-rw==1.4.0 +jsonschema==2.6.0 kombu==4.6.6 -# Note: amqp is used by kombu -amqp==2.5.2 +lockfile==0.12.2 +mongoengine==0.18.2 +networkx==1.11 # NOTE: Recent version substantially affect the performance and add big import time overhead # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details oslo.config>=1.12.1,<1.13 oslo.utils>=3.36.2,<=3.37.0 -six==1.13.0 -pyyaml==5.1.2 -requests[security]==2.23.0 -apscheduler==3.6.3 -gitpython==2.1.15 -jsonschema==2.6.0 -pymongo==3.10.0 -mongoengine==0.18.2 +paramiko==2.6.0 passlib==1.7.1 -lockfile==0.12.2 +prance==0.9.0 +prompt-toolkit==1.0.15 +pyinotify==0.9.6; platform_system=="Linux" +pymongo==3.10.0 +python-editor==1.0.4 python-gnupg==0.4.5 -jsonpath-rw==1.4.0 -pyinotify==0.9.6 -semver==2.9.0 -pytz==2019.1 -stevedore==1.30.1 -paramiko==2.6.0 -networkx==1.11 python-keyczar==0.716 -cryptography==2.8 +pytz==2019.1 +pywinrm==0.3.0 +pyyaml==5.1.2 +requests[security]==2.23.0 retrying==1.3.3 -# Note: We use latest version of virtualenv which uses pip 19 -virtualenv==16.6.0 +routes==2.4.1 +semver==2.9.0 +six==1.13.0 # NOTE: sseclient has various issues which sometimes hang the connection for a long time, etc. sseclient-py==1.7 -python-editor==1.0.4 -prompt-toolkit==1.0.15 +stevedore==1.30.1 tooz==1.66.1 -zake==0.2.2 -routes==2.4.1 +# Note: We use latest version of virtualenv which uses pip 19 +virtualenv==16.6.0 webob==1.8.5 -flex==6.14.0 -prance==0.9.0 -pywinrm==0.3.0 +zake==0.2.2 # test requirements below +bcrypt==3.1.7 +jinja2==2.10.3 +mock==2.0.0 +more-itertools==5.0.0 nose-timer==0.7.5 nose-parallel==0.3.1 psutil==5.6.3 +python-dateutil==2.8.0 python-statsd==2.1.0 prometheus_client==0.1.1 -mock==2.0.0 ujson==1.35 -python-dateutil==2.8.0 -bcrypt==3.1.7 -jinja2==2.10.3 -more-itertools==5.0.0 zipp>=0.5,<=1.0.0 diff --git a/requirements.txt b/requirements.txt index 3898401f16..40e8a6ea0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,7 @@ passlib==1.7.1 prettytable prompt-toolkit==1.0.15 psutil==5.6.3 -pyinotify==0.9.6 +pyinotify==0.9.6 ; platform_system == "Linux" pymongo==3.10.0 pyrabbit python-dateutil==2.8.0 diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index aa3e4afdbf..0aa90ec445 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -22,6 +22,15 @@ from distutils.version import StrictVersion # NOTE: This script can't rely on any 3rd party dependency so we need to use this code here +# +# TODO: Why can't this script rely on 3rd party dependencies? Is it because it has to import +# from pip? +# +# TODO: Dear future developer, if you are back here fixing a bug with how we parse +# requirements files, please look into using the packaging package on PyPI: +# https://packaging.pypa.io/en/latest/requirements/ +# At the very least we can vendorize some of their code instead of reimplementing +# each piece of their code every time our parsing breaks. PY3 = sys.version_info[0] == 3 if PY3: @@ -115,6 +124,9 @@ def _get_link(line): else: req_name = line + if ';' in req_name: + req_name = req_name.split(';')[0].strip() + reqs.append(req_name) return (reqs, links) diff --git a/scripts/fixate-requirements.py b/scripts/fixate-requirements.py index 3e712bc9c7..1aa892b7fb 100755 --- a/scripts/fixate-requirements.py +++ b/scripts/fixate-requirements.py @@ -171,10 +171,13 @@ def write_requirements(sources=None, fixed_requirements=None, output_file=None, rline = '-e %s' % (rline) elif req.req: project = req.name - if project in fixedreq_hash: - rline = str(fixedreq_hash[project].req) - else: - rline = str(req.req) + req_obj = fixedreq_hash.get(project, req) + + rline = str(req_obj.req) + + # Also write out environment markers + if req_obj.markers: + rline += " ; {}".format(str(req_obj.markers)) lines_to_write.append(rline) diff --git a/st2actions/requirements.txt b/st2actions/requirements.txt index f102a5c49f..958215c561 100755 --- a/st2actions/requirements.txt +++ b/st2actions/requirements.txt @@ -15,7 +15,7 @@ kombu==4.6.6 lockfile==0.12.2 oslo.config<1.13,>=1.12.1 oslo.utils<=3.37.0,>=3.36.2 -pyinotify==0.9.6 +pyinotify==0.9.6 ; platform_system == "Linux" python-dateutil==2.8.0 python-json-logger pyyaml==5.1.2 diff --git a/st2api/in-requirements.txt b/st2api/in-requirements.txt index 1ad7b3002e..76ea3129ee 100644 --- a/st2api/in-requirements.txt +++ b/st2api/in-requirements.txt @@ -8,5 +8,5 @@ oslo.config oslo.utils pymongo six -git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient +git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient gunicorn diff --git a/st2api/requirements.txt b/st2api/requirements.txt index 735c125d26..854a05e273 100644 --- a/st2api/requirements.txt +++ b/st2api/requirements.txt @@ -6,7 +6,7 @@ # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt eventlet==0.25.1 -git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient +git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient gunicorn==19.9.0 jsonschema==2.6.0 kombu==4.6.6 diff --git a/test-requirements.txt b/test-requirements.txt index db927f9997..c4970ef10e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -32,4 +32,4 @@ pyrabbit # (those tests only run under Python 2.7 since keyczar doesn't support Python 3.x). # See https://github.com/StackStorm/st2/pull/4165 python-keyczar -git+https://github.com/StackStorm/pip-conflict-checker.git@fix_pip_issue#egg=pip-conflict-checker +pip-tools # For pip-compile, to check for version conflicts