Include requirements.txt files for all the components Python packages

.gitignore

@@ -2,8 +2,6 @@
 *.sqlite
 *.swp
 *.log
-*/requirements.txt
-!st2client/requirements.txt
 .stamp*
 
 # C extensions
@@ -28,6 +26,8 @@ virtualenv-py3
 virtualenv-osx
 virtualenv-st2client
 virtualenv-st2client-osx
+virtualenv-components
+virtualenv-components-osx
 .venv-st2devbox
 
 # Installer logs

.travis.yml

@@ -12,16 +12,18 @@ branches:
 
 env:
   global:
+    - TRAVIS_EVENT_TYPE=cron
+    - IS_NIGHTLY_BUILD=$([ "${TRAVIS_EVENT_TYPE}" = "cron" ] && echo "yes" || echo "no")
     # NOTE: We only enable coverage for master builds and not pull requests
-    # since it has huge performance overhead (etests are 50% or so slower)
-    - ENABLE_COVERAGE=$([ "${TRAVIS_PULL_REQUEST}" = "false" ] && echo "yes" || echo "no")
+    # since it has huge performance overhead (tests are 50% or so slower)
+    - ENABLE_COVERAGE=$([ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ] && echo "yes" || echo "no")
     # We need to explicitly specify terminal width otherwise some CLI tests fail on container
     # environments where small terminal size is used.
     - COLUMNS=120
     - PYLINT_CONCURRENCY=2
     # We only run tests with "--with-timer" flag on master and not for PRs since it adds 1-2
     # # minutes of overhead to each build.
-    - NOSE_TIME=$([ "${TRAVIS_PULL_REQUEST}" = "false" ] && echo "yes" || echo "no")
+    - NOSE_TIME=$([ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ] && echo "yes" || echo "no")
 matrix:
   include:
     # NOTE: We combine builds because Travis offers a maximum of 5 concurrent
@@ -132,7 +134,9 @@ script:
   # Clean up egg-info directories which get created when installing runners
   # NOTE: We enable code coverage and per test timing information on master so build can take twice
   # as long as PR builds
-  - if [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD}
+  - if [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then COMMAND_THRESHOLD=$(expr ${COMMAND_THRESHOLD} \* 2); fi; ./scripts/travis/time-command.sh "make ${TASK}" ${COMMAND_THRESHOLD}
+  # Run any additional nightly checks only as part of a nightly (cron) build
+  - if [ "${IS_NIGHTLY_BUILD}" = "yes" ] && [ "${TASK}" = "ci-checks ci-packs-tests" ]; then make ci-checks-nightly; fi
   # NOTE: We only generate and submit coverage report for master and version branches
   # NOTE: We put this here and not after_success so build is marked as failed if this step fails
   # See https://docs.travis-ci.com/user/customizing-the-build/#breaking-the-build
@@ -143,4 +147,4 @@ script:
 # See: https://docs.travis-ci.com/user/caching/#Pull-request-builds-and-caches
 # Alternative: use strict pip pinning, including git-based pip packages
 before_cache:
-  - if [ ${TRAVIS_PULL_REQUEST} = 'false' ]; then rm -rf virtualenv/; fi
+  - if [ ${TRAVIS_PULL_REQUEST} = 'false' ] && [ "${IS_NIGHTLY_BUILD}" = "no" ]; then rm -rf virtualenv/; fi

Makefile

@@ -8,9 +8,11 @@ OS := $(shell uname)
 ifeq ($(OS),Darwin)
 	VIRTUALENV_DIR ?= virtualenv-osx
 	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client-osx
+	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components-osx
 else
 	VIRTUALENV_DIR ?= virtualenv
 	VIRTUALENV_ST2CLIENT_DIR ?= virtualenv-st2client
+	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components
 endif
 
 PYTHON_VERSION ?= python2.7
@@ -20,6 +22,7 @@ BINARIES := bin
 # All components are prefixed by st2 and not .egg-info.
 COMPONENTS := $(shell ls -a | grep ^st2 | grep -v .egg-info)
 COMPONENTS_RUNNERS := $(wildcard contrib/runners/*)
+COMPONENTS_WITHOUT_ST2TESTS := $(shell ls -a | grep ^st2 | grep -v .egg-info | grep -v st2tests | grep -v st2exporter)
 
 COMPONENTS_WITH_RUNNERS := $(COMPONENTS) $(COMPONENTS_RUNNERS)
 
@@ -120,6 +123,8 @@ play:
 	@echo
 	@echo TRAVIS_PULL_REQUEST=$(TRAVIS_PULL_REQUEST)
 	@echo
+	@echo TRAVIS_EVENT_TYPE=$(TRAVIS_EVENT_TYPE)
+	@echo
 	@echo NOSE_OPTS=$(NOSE_OPTS)
 	@echo
 	@echo ENABLE_COVERAGE=$(ENABLE_COVERAGE)
@@ -159,6 +164,44 @@ check-requirements: requirements
 	git status -- *requirements.txt */*requirements.txt | grep -q "nothing to commit"
 	@echo "All requirements files up-to-date!"
 
+.PHONY: check-python-packages
+check-python-packages:
+	# Make target which verifies all the components Python packages are valid
+	@echo ""
+	@echo "================== CHECK PYTHON PACKAGES ===================="
+	@echo ""
+
+	test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) --no-site-packages $(VIRTUALENV_COMPONENTS_DIR) --no-download
+	@for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \
+		echo "==========================================================="; \
+		echo "Checking component:" $$component; \
+		echo "==========================================================="; \
+		(set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python setup.py --version) || exit 1; \
+	done
+
+.PHONY: check-python-packages-nightly
+check-python-packages-nightly:
+	# NOTE: This is subset of check-python-packages target.
+	# We run more extensive and slower tests as part of the nightly build to speed up PR builds
+	@echo ""
+	@echo "================== CHECK PYTHON PACKAGES ===================="
+	@echo ""
+
+	test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) --no-site-packages $(VIRTUALENV_COMPONENTS_DIR) --no-download
+	@for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \
+		echo "==========================================================="; \
+		echo "Checking component:" $$component; \
+		echo "==========================================================="; \
+		(set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python setup.py --version) || exit 1; \
+		(set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python setup.py sdist bdist_wheel) || exit 1; \
+		(set -e; cd $$component; ../$(VIRTUALENV_COMPONENTS_DIR)/bin/python setup.py develop --no-deps) || exit 1; \
+		($(VIRTUALENV_COMPONENTS_DIR)/bin/python -c "import $$component") || exit 1; \
+		(set -e; cd $$component; rm -rf dist/; rm -rf $$component.egg-info) || exit 1; \
+	done
+
+.PHONY: ci-checks-nightly
+ci-checks-nightly: check-python-packages-nightly
+
 .PHONY: checklogs
 checklogs:
 	@echo
@@ -758,6 +801,8 @@ packs-tests: requirements .packs-tests
 	@echo
 	@echo "==================== packs-tests ===================="
 	@echo
+	# Install st2common to register metrics drivers
+	(cd ${ROOT_DIR}/st2common; ${ROOT_DIR}/$(VIRTUALENV_DIR)/bin/python setup.py develop --no-deps)
 	. $(VIRTUALENV_DIR)/bin/activate; find ${ROOT_DIR}/contrib/* -maxdepth 0 -type d -print0 | xargs -0 -I FILENAME ./st2common/bin/st2-run-pack-tests -c -t -x -p FILENAME
 
 
@@ -862,7 +907,7 @@ debs:
 ci: ci-checks ci-unit ci-integration ci-mistral ci-packs-tests
 
 .PHONY: ci-checks
-ci-checks: compile .generated-files-check .pylint .flake8 check-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check
+ci-checks: compile .generated-files-check .pylint .flake8 check-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
 
 .PHONY: ci-py3-unit
 ci-py3-unit:

scripts/dist_utils.py

@@ -31,16 +31,8 @@
 
 GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python'
 
-try:
-    import pip
-except ImportError as e:
-    print('Failed to import pip: %s' % (text_type(e)))
-    print('')
-    print('Download pip:\n%s' % (GET_PIP))
-    sys.exit(1)
-
-
 __all__ = [
+    'check_pip_is_installed',
     'check_pip_version',
     'fetch_requirements',
     'apply_vagrant_workaround',
@@ -49,17 +41,38 @@
 ]
 
 
+def check_pip_is_installed():
+    """
+    Ensure that pip is installed.
+    """
+    try:
+        import pip  # NOQA
+    except ImportError as e:
+        print('Failed to import pip: %s' % (text_type(e)))
+        print('')
+        print('Download pip:\n%s' % (GET_PIP))
+        sys.exit(1)
+
+    return True
+
+
 def check_pip_version(min_version='6.0.0'):
     """
     Ensure that a minimum supported version of pip is installed.
     """
+    check_pip_is_installed()
+
+    import pip
+
     if StrictVersion(pip.__version__) < StrictVersion(min_version):
         print("Upgrade pip, your version '{0}' "
               "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__,
                                                                             min_version,
                                                                             GET_PIP))
         sys.exit(1)
 
+    return True
+
 
 def fetch_requirements(requirements_file_path):
     """

st2actions/requirements.txt

@@ -0,0 +1,17 @@
+# Don't edit this file. It's generated automatically!
+apscheduler==3.6.0
+eventlet==0.24.1
+git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper
+git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient
+gitpython==2.1.11
+jinja2==2.10.1
+kombu==4.5.0
+lockfile==0.12.2
+oslo.config<1.13,>=1.12.1
+oslo.utils<=3.37.0,>=3.36.2
+pyinotify==0.9.6
+python-dateutil==2.8.0
+python-json-logger
+pyyaml==5.1
+requests[security]<2.23.0,>=2.22.0
+six==1.12.0

st2actions/st2actions/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = '3.2dev'

st2api/requirements.txt

@@ -0,0 +1,11 @@
+# Don't edit this file. It's generated automatically!
+eventlet==0.24.1
+git+https://github.com/StackStorm/python-mistralclient#egg=python-mistralclient
+gunicorn==19.9.0
+jsonschema==2.6.0
+kombu==4.5.0
+mongoengine==0.17.0
+oslo.config<1.13,>=1.12.1
+oslo.utils<=3.37.0,>=3.36.2
+pymongo==3.7.2
+six==1.12.0

st2api/st2api/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = '3.2dev'

st2auth/requirements.txt

@@ -0,0 +1,10 @@
+# Don't edit this file. It's generated automatically!
+bcrypt==3.1.6
+eventlet==0.24.1
+git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
+gunicorn==19.9.0
+oslo.config<1.13,>=1.12.1
+passlib==1.7.1
+pymongo==3.7.2
+six==1.12.0
+stevedore==1.30.1

st2auth/st2auth/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = '3.2dev'

st2common/st2common/util/green/shell.py

@@ -127,15 +127,20 @@ def on_timeout_expired(timeout):
             # Command has timed out, kill the process and propagate the error.
             # Note: We explicitly set the returncode to indicate the timeout.
             LOG.debug('Command execution timeout reached.')
+
+            # NOTE: It's important we set returncode twice - here and below to avoid race in this
+            # function because "kill_func()" is async and "process.kill()" is not.
+            process.returncode = TIMEOUT_EXIT_CODE
+
             if kill_func:
                 LOG.debug('Calling kill_func.')
                 kill_func(process=process)
             else:
                 LOG.debug('Killing process.')
                 process.kill()
 
-            # NOTE: It's imporant to set returncode here, since call to kill()
-            # sets it and overwrites it if we set it earlier
+            # NOTE: It's imporant to set returncode here as well, since call to process.kill() sets
+            # it and overwrites it if we set it earlier.
             process.returncode = TIMEOUT_EXIT_CODE
 
             if read_stdout_func and read_stderr_func:

st2common/tests/fixtures/version_file.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Copyright 2019 Extreme Networks, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = '1.2.3'