diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e4f6c0b07..ec135060be 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -92,7 +92,7 @@ jobs: # TODO: maybe make the virtualenv a partial cache to exclude st2*? # !virtualenv/lib/python*/site-packages/st2* # !virtualenv/bin/st2* - key: ${{ runner.os }}-v2-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + key: ${{ runner.os }}-v3-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} restore-keys: | ${{ runner.os }}-v2-python-${{ matrix.python }}- - name: Cache APT Dependencies @@ -233,7 +233,7 @@ jobs: # TODO: maybe make the virtualenv a partial cache to exclude st2*? # !virtualenv/lib/python*/site-packages/st2* # !virtualenv/bin/st2* - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + key: ${{ runner.os }}-v3-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} restore-keys: | ${{ runner.os }}-python-${{ matrix.python }}- - name: Cache APT Dependencies @@ -448,7 +448,7 @@ jobs: # TODO: maybe make the virtualenv a partial cache to exclude st2*? # !virtualenv/lib/python*/site-packages/st2* # !virtualenv/bin/st2* - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + key: ${{ runner.os }}-v3-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} restore-keys: | ${{ runner.os }}-python-${{ matrix.python }}- - name: Cache APT Dependencies diff --git a/.github/workflows/orquesta-integration-tests.yaml b/.github/workflows/orquesta-integration-tests.yaml index 98159afe8f..a7733b6512 100644 --- a/.github/workflows/orquesta-integration-tests.yaml +++ b/.github/workflows/orquesta-integration-tests.yaml @@ -139,7 +139,7 @@ jobs: # TODO: maybe make the virtualenv a partial cache to exclude st2*? # !virtualenv/lib/python*/site-packages/st2* # !virtualenv/bin/st2* - key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} + key: ${{ runner.os }}-v3-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }} restore-keys: | ${{ runner.os }}-python-${{ matrix.python }}- - name: Cache APT Dependencies diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 676df63b7d..8e07295e6d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,11 @@ Changed Contributed by Amanda McGuinness (@amanda11 Ammeon Solutions) +* Pinned python module `networkx` to versions between 2.5.1(included) and 2.6(excluded) because Python v3.6 support was dropped in v2.6. + Also pinned `decorator==4.4.2` (dependency of `networkx<2.6`) to work around missing python 3.8 classifiers on `decorator`'s wheel. #5376 + + Contributed by @nzlosh + Fixed ~~~~~ diff --git a/Makefile b/Makefile index 018641bffd..e15bb11277 100644 --- a/Makefile +++ b/Makefile @@ -252,7 +252,7 @@ check-python-packages: @echo "" @echo "================== CHECK PYTHON PACKAGES ====================" @echo "" - test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --system-site-packages + test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || $(PYTHON_VERSION) -m venv $(VIRTUALENV_COMPONENTS_DIR) --system-site-packages @for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \ echo "==========================================================="; \ echo "Checking component:" $$component; \ @@ -268,7 +268,7 @@ check-python-packages-nightly: @echo "================== CHECK PYTHON PACKAGES ====================" @echo "" - test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download + test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || $(PYTHON_VERSION) -m venv $(VIRTUALENV_COMPONENTS_DIR) @for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \ echo "==========================================================="; \ echo "Checking component:" $$component; \ @@ -392,7 +392,7 @@ black: requirements .black-check . $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml pylint_plugins/*.py || exit 1; # Black task which reformats the code using black -.PHONY: black-format +.PHONY: black black: requirements .black-format .PHONY: .black-format @@ -406,7 +406,7 @@ black: requirements .black-format echo "Running black on" $$component; \ echo "==========================================================="; \ . $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$component/ || exit 1; \ - . $(VIRTUALENV_DIR)/bin/activate ; black $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \ + . $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \ done # runner modules and packages @for component in $(COMPONENTS_RUNNERS); do\ @@ -414,7 +414,7 @@ black: requirements .black-format echo "Running black on" $$component; \ echo "==========================================================="; \ . $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$component/ || exit 1; \ - . $(VIRTUALENV_DIR)/bin/activate ; black $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \ + . $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \ done . $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml contrib/ || exit 1; . $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml scripts/*.py || exit 1; @@ -486,7 +486,7 @@ flake8: requirements .flake8 @echo @echo "==================== st2client pypi check ====================" @echo - test -f $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/activate || virtualenv --python=python3 $(VIRTUALENV_ST2CLIENT_PYPI_DIR) --no-download + test -f $(VIRTUALENV_ST2CLIENT_PYPI_DIR)/bin/activate || $(PYTHON_VERSION) -m venv $(VIRTUALENV_ST2CLIENT_PYPI_DIR) # Setup PYTHONPATH in bash activate script... # Delete existing entries (if any) @@ -514,7 +514,7 @@ flake8: requirements .flake8 @echo @echo "==================== st2client install check ====================" @echo - test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_ST2CLIENT_DIR) --no-download + test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || $(PYTHON_VERSION) -m venv $(VIRTUALENV_ST2CLIENT_DIR) # Setup PYTHONPATH in bash activate script... # Delete existing entries (if any) @@ -740,7 +740,7 @@ virtualenv: @echo @echo "==================== virtualenv ====================" @echo - test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --no-download + test -f $(VIRTUALENV_DIR)/bin/activate || $(PYTHON_VERSION) -m venv $(VIRTUALENV_DIR) # Setup PYTHONPATH in bash activate script... # Delete existing entries (if any) diff --git a/OWNERS.md b/OWNERS.md index 6ccccc305c..52431bf0fc 100644 --- a/OWNERS.md +++ b/OWNERS.md @@ -19,8 +19,6 @@ Senior [@StackStorm/maintainers](https://github.com/orgs/StackStorm/teams/mainta Have deep platform knowledge & experience and demonstrate technical leadership as well as driving the project forward. * Amanda McGuinness ([@amanda11](https://github.com/amanda11)), _Ammeon Solutions_ <> - Ansible, Core, deb/rpm packages, CI/CD, Deployments, Release Engineering, Infrastructure, Documentation. -* blag ([@blag](https://github.com/blag)) <> - - ChatOps, StackStorm Exchange, Community, Documentation, CI/CD. * Eugen Cusmaunsa ([@armab](https://github.com/armab)) <> - Systems, Deployments, Docker, K8s, HA, Ansible, Chef, Vagrant, deb/rpm, CI/CD, Infrastructure, Release Engineering, Community. * Nick Maludy ([@nmaludy](https://github.com/nmaludy)) <> @@ -67,6 +65,8 @@ Thank you, Friends! * Anthony Shaw ([@tonybaloney](https://github.com/tonybaloney)) - Contribution via Ideas, Feedback, ChatOps improvements, core Architecture, Community and even [Marketing](https://news.ycombinator.com/item?id=14368748). [Case Study](https://stackstorm.com/case-study-dimension-data/). * Andy Moore ([@AndyMoore](https://github.com/AndyMoore)) - Community, StackStorm Exchange packs, Kubernetes. * Anirudh Rekhi ([@humblearner](https://github.com/humblearner)) - ex Stormer. Systems, CI/CD, DevOps, Infrastructure. +* blag ([@blag](https://github.com/blag)) + - ChatOps, StackStorm Exchange, Community, Documentation, CI/CD. * Denis Barishev ([@dennybaa](https://github.com/dennybaa)) - ex Stormer. Re-architeced StackStorm installer via deb/rpm packages, made it stable, repeatable, fixed first platform pain points. * Ed Medvedev ([@emedvedev](https://github.com/emedvedev)) - ex Stormer. Made WebUI slick and ChatOps awesome, discovered and implemented today's [StackStorm Exchange](https://exchange.stackstorm.org/). * Evan Powell ([@epowell101](https://github.com/epowell101)) - StackStorm co-founder, first CEO, Stormer forever. diff --git a/contrib/runners/orquesta_runner/in-requirements.txt b/contrib/runners/orquesta_runner/in-requirements.txt index 77570be456..770d7eaa07 100644 --- a/contrib/runners/orquesta_runner/in-requirements.txt +++ b/contrib/runners/orquesta_runner/in-requirements.txt @@ -1 +1 @@ -git+https://github.com/StackStorm/orquesta.git@219f00db5192321af9d29b4c51ec748846ab90c6#egg=orquesta +git+https://github.com/StackStorm/orquesta.git@c971b94d5c3dc065ee18386bdf3609a3d2de4a2c#egg=orquesta diff --git a/contrib/runners/orquesta_runner/requirements.txt b/contrib/runners/orquesta_runner/requirements.txt index 03b03964bd..22ce5138bc 100644 --- a/contrib/runners/orquesta_runner/requirements.txt +++ b/contrib/runners/orquesta_runner/requirements.txt @@ -5,4 +5,4 @@ # If you want to update depdencies for a single component, modify the # in-requirements.txt for that component and then run 'make requirements' to # update the component requirements.txt -git+https://github.com/StackStorm/orquesta.git@219f00db5192321af9d29b4c51ec748846ab90c6#egg=orquesta +git+https://github.com/StackStorm/orquesta.git@c971b94d5c3dc065ee18386bdf3609a3d2de4a2c#egg=orquesta diff --git a/fixed-requirements.txt b/fixed-requirements.txt index 2155f566fd..11648b1c10 100644 --- a/fixed-requirements.txt +++ b/fixed-requirements.txt @@ -22,8 +22,12 @@ jsonschema==2.6.0 kombu==5.0.2 lockfile==0.12.2 mongoengine==0.23.0 -#Update networkx to match orquesta -networkx>=2.5.1,<3.0 +# networkx v2.6 does not support Python3.6. Update networkx to match orquesta +networkx>=2.5.1,<2.6 +# networkx requires decorator>=4.3,<5 which should resolve to version 4.4.2 +# but the wheel on pypi does not say it supports python3.8, so pip gets +# confused. For now, pin decorator to work around pip's confusion. +decorator==4.4.2 # NOTE: Recent version substantially affect the performance and add big import time overhead # See https://github.com/StackStorm/st2/issues/4160#issuecomment-394386433 for details oslo.config>=1.12.1,<1.13 diff --git a/requirements.txt b/requirements.txt index 0c2438e1ca..c0693bb05f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,11 +12,12 @@ argcomplete==1.12.2 bcrypt==3.2.0 chardet<3.1.0 cryptography==3.4.7 +decorator==4.4.2 dnspython>=1.16.0,<2.0.0 eventlet==0.30.2 flex==6.14.1 git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper -git+https://github.com/StackStorm/orquesta.git@219f00db5192321af9d29b4c51ec748846ab90c6#egg=orquesta +git+https://github.com/StackStorm/orquesta.git@c971b94d5c3dc065ee18386bdf3609a3d2de4a2c#egg=orquesta git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend @@ -32,7 +33,7 @@ kombu==5.0.2 lockfile==0.12.2 mock==4.0.3 mongoengine==0.23.0 -networkx>=2.5.1,<3.0 +networkx>=2.5.1,<2.6 nose nose-parallel==0.4.0 nose-timer==1.0.1 diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index 9584efb4a7..4e2ff9a295 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -16,6 +16,7 @@ COMPONENTS="st2actionrunner st2api st2stream st2auth st2garbagecollector st2notifier st2rulesengine st2sensorcontainer st2chatops st2timersengine st2workflowengine st2scheduler" ST2_CONF="/etc/st2/st2.conf" +SYSTEMD_RELOADED="" # Ensure global environment is sourced if exists # Does not happen consistently with all OSes we support. @@ -108,6 +109,11 @@ function service_manager() { local svcname=$1 action=$2 if [ -d /run/systemd/system ]; then # systemd is running + if [ -z $SYSTEMD_RELOADED ]; then + #Reload systemd to regenerate socket files from st2.conf + systemctl daemon-reload + SYSTEMD_RELOADED="yes" + fi systemctl $action $svcname elif [ $(cat /proc/1/comm) = init ] && (/sbin/initctl version 2>/dev/null | grep -q upstart) && [ -f /etc/init/${svcname}.conf ]; then diff --git a/st2common/in-requirements.txt b/st2common/in-requirements.txt index e32d20a07b..4e3579d1ed 100644 --- a/st2common/in-requirements.txt +++ b/st2common/in-requirements.txt @@ -10,7 +10,9 @@ jsonschema kombu mongoengine networkx -git+https://github.com/StackStorm/orquesta.git@219f00db5192321af9d29b4c51ec748846ab90c6#egg=orquesta +# used by networkx +decorator +git+https://github.com/StackStorm/orquesta.git@c971b94d5c3dc065ee18386bdf3609a3d2de4a2c#egg=orquesta git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend oslo.config paramiko diff --git a/st2common/requirements.txt b/st2common/requirements.txt index 33ea04318a..b5d2745364 100644 --- a/st2common/requirements.txt +++ b/st2common/requirements.txt @@ -9,10 +9,11 @@ amqp==5.0.6 apscheduler==3.7.0 chardet<3.1.0 cryptography==3.4.7 +decorator==4.4.2 dnspython>=1.16.0,<2.0.0 eventlet==0.30.2 flex==6.14.1 -git+https://github.com/StackStorm/orquesta.git@219f00db5192321af9d29b4c51ec748846ab90c6#egg=orquesta +git+https://github.com/StackStorm/orquesta.git@c971b94d5c3dc065ee18386bdf3609a3d2de4a2c#egg=orquesta git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend gitdb==4.0.2 gitpython==3.1.15 @@ -23,7 +24,7 @@ jsonschema==2.6.0 kombu==5.0.2 lockfile==0.12.2 mongoengine==0.23.0 -networkx>=2.5.1,<3.0 +networkx>=2.5.1,<2.6 orjson==3.5.2 oslo.config>=1.12.1,<1.13 paramiko==2.7.2