diff --git a/CHANGELOG.rst b/CHANGELOG.rst index af716c63df..1efce25f3a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,9 +1,41 @@ Changelog ========= -in development --------------- +2.8.1 - TBD +----------- +Added +----- + +* Update ``st2`` CLI to inspect ``COLUMNS`` environment variable first when determining the + terminal size. Previously this environment variable was checked second last (after trying to + retrieve terminal size using various OS specific methods and before falling back to the default + value). + + This approach is more performant and allows user to easily overwrite the default value or value + returned by the operating system checks - e.g. by running ``COLUMNS=200 st2 action list``. + (improvement) #4242 + +Changed +------- + +* Update ``st2client/setup.py`` file to dynamically load requirements from + ``st2client/requirements.txt`` file. The code works with pip >= 6.0.0, although using pip 9.0.0 + or higher is strongly recommended. (improvement) #4209 +* Update ``st2`` CLI to use a more sensible default terminal size for table formatting purposes if + we are unable to retrieve terminal size using various system-specific approaches. + + Previously we would fall back to a very unfriendly default of 20 columns for a total terminal + width. This would cause every table column to wrap and make output impossible / hard to read. + (improvement) #4242 + +Fixed +~~~~~ + +* Fixed a bug where ``secret: true`` was not applying to full object and array trees. (bugfix) #4234 + Reported by @jjm + + Contributed by Nick Maludy (Encore Technologies). 2.8.0 - July 10, 2018 --------------------- diff --git a/Makefile b/Makefile index 6226f4e3f2..0227bab172 100644 --- a/Makefile +++ b/Makefile @@ -316,10 +316,10 @@ requirements: virtualenv .sdist-requirements $(VIRTUALENV_DIR)/bin/pip install --upgrade "virtualenv==15.1.0" # Required for packs.install in dev envs. # Generate all requirements to support current CI pipeline. - $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv -s st2*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt + $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt # Generate finall requirements.txt file for each component - @for component in $(COMPONENTS); do\ + @for component in $(COMPONENTS_WITH_RUNNERS); do\ echo "==========================================================="; \ echo "Generating requirements.txt for" $$component; \ echo "==========================================================="; \ diff --git a/contrib/examples/triggers/sample-trigger.yaml b/contrib/examples/triggers/sample-trigger.yaml index ae140cad67..105e427cd3 100644 --- a/contrib/examples/triggers/sample-trigger.yaml +++ b/contrib/examples/triggers/sample-trigger.yaml @@ -1,3 +1,10 @@ --- name: sample_trigger description: Sample trigger +payload_schema: + type: "object" + properties: + executed_at: + type: "string" + format: "date-time" + default: "2014-07-30 05:04:24.578325" diff --git a/contrib/packs/actions/pack_mgmt/get_installed.py b/contrib/packs/actions/pack_mgmt/get_installed.py index f768172cba..07a57c006c 100644 --- a/contrib/packs/actions/pack_mgmt/get_installed.py +++ b/contrib/packs/actions/pack_mgmt/get_installed.py @@ -87,5 +87,5 @@ def run(self, pack): def _parse_yaml_file(self, file_path): with open(file_path) as data_file: - details = yaml.load(data_file) + details = yaml.safe_load(data_file) return details diff --git a/contrib/runners/action_chain_runner/dist_utils.py b/contrib/runners/action_chain_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/action_chain_runner/dist_utils.py +++ b/contrib/runners/action_chain_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/action_chain_runner/in-requirements.txt b/contrib/runners/action_chain_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/action_chain_runner/requirements.txt b/contrib/runners/action_chain_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/action_chain_runner/requirements.txt +++ b/contrib/runners/action_chain_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/announcement_runner/dist_utils.py b/contrib/runners/announcement_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/announcement_runner/dist_utils.py +++ b/contrib/runners/announcement_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/announcement_runner/in-requirements.txt b/contrib/runners/announcement_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/announcement_runner/requirements.txt b/contrib/runners/announcement_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/announcement_runner/requirements.txt +++ b/contrib/runners/announcement_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/cloudslang_runner/dist_utils.py b/contrib/runners/cloudslang_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/cloudslang_runner/dist_utils.py +++ b/contrib/runners/cloudslang_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/cloudslang_runner/in-requirements.txt b/contrib/runners/cloudslang_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/cloudslang_runner/requirements.txt b/contrib/runners/cloudslang_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/cloudslang_runner/requirements.txt +++ b/contrib/runners/cloudslang_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/http_runner/dist_utils.py b/contrib/runners/http_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/http_runner/dist_utils.py +++ b/contrib/runners/http_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/http_runner/in-requirements.txt b/contrib/runners/http_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/http_runner/requirements.txt b/contrib/runners/http_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/http_runner/requirements.txt +++ b/contrib/runners/http_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/inquirer_runner/dist_utils.py b/contrib/runners/inquirer_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/inquirer_runner/dist_utils.py +++ b/contrib/runners/inquirer_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/inquirer_runner/in-requirements.txt b/contrib/runners/inquirer_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/inquirer_runner/requirements.txt b/contrib/runners/inquirer_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/inquirer_runner/requirements.txt +++ b/contrib/runners/inquirer_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/local_runner/dist_utils.py b/contrib/runners/local_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/local_runner/dist_utils.py +++ b/contrib/runners/local_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/local_runner/in-requirements.txt b/contrib/runners/local_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/local_runner/requirements.txt b/contrib/runners/local_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/local_runner/requirements.txt +++ b/contrib/runners/local_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/mistral_v2/dist_utils.py b/contrib/runners/mistral_v2/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/mistral_v2/dist_utils.py +++ b/contrib/runners/mistral_v2/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/mistral_v2/in-requirements.txt b/contrib/runners/mistral_v2/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/mistral_v2/requirements.txt b/contrib/runners/mistral_v2/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/mistral_v2/requirements.txt +++ b/contrib/runners/mistral_v2/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/noop_runner/dist_utils.py b/contrib/runners/noop_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/noop_runner/dist_utils.py +++ b/contrib/runners/noop_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/noop_runner/in-requirements.txt b/contrib/runners/noop_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/noop_runner/requirements.txt b/contrib/runners/noop_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/noop_runner/requirements.txt +++ b/contrib/runners/noop_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/orchestra_runner/dist_utils.py b/contrib/runners/orchestra_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/orchestra_runner/dist_utils.py +++ b/contrib/runners/orchestra_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/orchestra_runner/in-requirements.txt b/contrib/runners/orchestra_runner/in-requirements.txt new file mode 100644 index 0000000000..236a1e3040 --- /dev/null +++ b/contrib/runners/orchestra_runner/in-requirements.txt @@ -0,0 +1 @@ +git+https://github.com/StackStorm/orchestra.git#egg=orchestra diff --git a/contrib/runners/orchestra_runner/requirements.txt b/contrib/runners/orchestra_runner/requirements.txt index 236a1e3040..5aa11564d7 100644 --- a/contrib/runners/orchestra_runner/requirements.txt +++ b/contrib/runners/orchestra_runner/requirements.txt @@ -1 +1,2 @@ +# Don't edit this file. It's generated automatically! git+https://github.com/StackStorm/orchestra.git#egg=orchestra diff --git a/contrib/runners/python_runner/dist_utils.py b/contrib/runners/python_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/python_runner/dist_utils.py +++ b/contrib/runners/python_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/python_runner/in-requirements.txt b/contrib/runners/python_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/python_runner/requirements.txt b/contrib/runners/python_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/python_runner/requirements.txt +++ b/contrib/runners/python_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/remote_runner/dist_utils.py b/contrib/runners/remote_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/remote_runner/dist_utils.py +++ b/contrib/runners/remote_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/remote_runner/in-requirements.txt b/contrib/runners/remote_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/remote_runner/requirements.txt b/contrib/runners/remote_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/remote_runner/requirements.txt +++ b/contrib/runners/remote_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/contrib/runners/windows_runner/dist_utils.py b/contrib/runners/windows_runner/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/contrib/runners/windows_runner/dist_utils.py +++ b/contrib/runners/windows_runner/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/contrib/runners/windows_runner/in-requirements.txt b/contrib/runners/windows_runner/in-requirements.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contrib/runners/windows_runner/requirements.txt b/contrib/runners/windows_runner/requirements.txt index e69de29bb2..b4be240ac2 100644 --- a/contrib/runners/windows_runner/requirements.txt +++ b/contrib/runners/windows_runner/requirements.txt @@ -0,0 +1,2 @@ +# Don't edit this file. It's generated automatically! + diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2actions/dist_utils.py b/st2actions/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2actions/dist_utils.py +++ b/st2actions/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2api/dist_utils.py b/st2api/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2api/dist_utils.py +++ b/st2api/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2auth/dist_utils.py b/st2auth/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2auth/dist_utils.py +++ b/st2auth/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2client/dist_utils.py b/st2client/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2client/dist_utils.py +++ b/st2client/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2client/in-requirements.txt b/st2client/in-requirements.txt index 3214109105..1bcf2d0b7a 100644 --- a/st2client/in-requirements.txt +++ b/st2client/in-requirements.txt @@ -4,6 +4,7 @@ prettytable pytz python-dateutil pyyaml +jsonschema jsonpath-rw requests six diff --git a/st2client/requirements.txt b/st2client/requirements.txt index f514d2b738..604d661fff 100644 --- a/st2client/requirements.txt +++ b/st2client/requirements.txt @@ -1,6 +1,7 @@ # Don't edit this file. It's generated automatically! argcomplete jsonpath-rw==1.4.0 +jsonschema==2.6.0 prettytable prompt-toolkit==1.0.15 python-dateutil diff --git a/st2client/setup.py b/st2client/setup.py index f673d207d3..587f2c0faf 100644 --- a/st2client/setup.py +++ b/st2client/setup.py @@ -18,19 +18,20 @@ from setuptools import setup, find_packages -# Note: We should re-enable usage of dist_utils once we ensure -# that we install new version of virtualenv which ships with -# pip >= 6.1 in all the environments -# from dist_utils import fetch_requirements -# from dist_utils import apply_vagrant_workaround +from dist_utils import check_pip_version +from dist_utils import fetch_requirements +from dist_utils import apply_vagrant_workaround + from st2client import __version__ +check_pip_version() + ST2_COMPONENT = 'st2client' BASE_DIR = os.path.dirname(os.path.abspath(__file__)) REQUIREMENTS_FILE = os.path.join(BASE_DIR, 'requirements.txt') -# install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE) -# apply_vagrant_workaround() +install_reqs, dep_links = fetch_requirements(REQUIREMENTS_FILE) +apply_vagrant_workaround() setup( name=ST2_COMPONENT, @@ -52,15 +53,8 @@ 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7' ], - install_requires=[ - 'jsonpath-rw>=1.3.0', - 'prettytable', - 'python-dateutil', - 'pyyaml<4.0,>=3.11', - 'requests[security]<2.15,>=2.14.1', - 'six==1.11.0' - ], - dependency_links=[], + install_requires=install_reqs, + dependency_links=dep_links, test_suite=ST2_COMPONENT, zip_safe=False, include_package_data=True, diff --git a/st2client/st2client/formatters/table.py b/st2client/st2client/formatters/table.py index ff783349fc..c29c583426 100644 --- a/st2client/st2client/formatters/table.py +++ b/st2client/st2client/formatters/table.py @@ -27,7 +27,7 @@ from st2client import formatters from st2client.utils import strutil -from st2client.utils.terminal import get_terminal_size +from st2client.utils.terminal import get_terminal_size_columns LOG = logging.getLogger(__name__) @@ -65,7 +65,7 @@ def format(cls, entries, *args, **kwargs): if not widths and attributes: # Dynamically calculate column size based on the terminal size - lines, cols = get_terminal_size() + cols = get_terminal_size_columns() if attributes[0] == 'id': # consume iterator and save as entries so collection is accessible later. diff --git a/st2client/st2client/utils/terminal.py b/st2client/st2client/utils/terminal.py index d53a342fc5..c65bc378e2 100644 --- a/st2client/st2client/utils/terminal.py +++ b/st2client/st2client/utils/terminal.py @@ -14,6 +14,7 @@ # limitations under the License. from __future__ import absolute_import + import os import struct import subprocess @@ -21,35 +22,62 @@ from st2client.utils.color import format_status +DEFAULT_TERMINAL_SIZE_COLUMNS = 150 + __all__ = [ - 'get_terminal_size' + 'DEFAULT_TERMINAL_SIZE_COLUMNS', + + 'get_terminal_size_columns' ] -def get_terminal_size(default=(80, 20)): +def get_terminal_size_columns(default=DEFAULT_TERMINAL_SIZE_COLUMNS): """ - :return: (lines, cols) + Try to retrieve COLUMNS value of terminal size using various system specific approaches. + + If terminal size can't be retrieved, default value is returned. + + NOTE 1: COLUMNS environment variable is checked first, if the value is not set / available, + other methods are tried. + + :rtype: ``int`` + :return: columns """ + # 1. Try COLUMNS environment variable first like in upstream Python 3 method - + # https://github.com/python/cpython/blob/master/Lib/shutil.py#L1203 + # This way it's consistent with upstream implementation. In the past, our implementation + # checked those variables at the end as a fall back. + try: + columns = os.environ['COLUMNS'] + + return int(columns) + except (KeyError, ValueError): + pass + def ioctl_GWINSZ(fd): import fcntl import termios + # Return a tuple (lines, columns) return struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) - # try stdin, stdout, stderr + + # 2. try stdin, stdout, stderr for fd in (0, 1, 2): try: - return ioctl_GWINSZ(fd) - except: + return ioctl_GWINSZ(fd)[1] + except Exception: pass - # try os.ctermid() + + # 3. try os.ctermid() try: fd = os.open(os.ctermid(), os.O_RDONLY) try: - return ioctl_GWINSZ(fd) + return ioctl_GWINSZ(fd)[1] finally: os.close(fd) - except: + except Exception: pass - # try `stty size` + + # 4. try `stty size` try: process = subprocess.Popen(['stty', 'size'], shell=False, @@ -57,15 +85,17 @@ def ioctl_GWINSZ(fd): stderr=open(os.devnull, 'w')) result = process.communicate() if process.returncode == 0: - return tuple(int(x) for x in result[0].split()) - except: + return tuple(int(x) for x in result[0].split())[1] + except Exception: pass + # try environment variables try: return tuple(int(os.getenv(var)) for var in ('LINES', 'COLUMNS')) except: pass - # return default. + + # 5. return default fallback value return default diff --git a/st2client/tests/unit/test_util_terminal.py b/st2client/tests/unit/test_util_terminal.py new file mode 100644 index 0000000000..b7635f2494 --- /dev/null +++ b/st2client/tests/unit/test_util_terminal.py @@ -0,0 +1,63 @@ +# Licensed to the StackStorm, Inc ('StackStorm') under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import + +import os + +import unittest2 +import mock + +from st2client.utils.terminal import DEFAULT_TERMINAL_SIZE_COLUMNS +from st2client.utils.terminal import get_terminal_size_columns + +__all__ = [ + 'TerminalUtilsTestCase' +] + + +class TerminalUtilsTestCase(unittest2.TestCase): + @mock.patch.dict(os.environ, {'LINES': '111', 'COLUMNS': '222'}) + def test_get_terminal_size_columns_columns_environment_variable_has_precedence(self): + # Verify that COLUMNS environment variables has precedence over other approaches + columns = get_terminal_size_columns() + + self.assertEqual(columns, 222) + + @mock.patch('struct.unpack', mock.Mock(return_value=(333, 444))) + def test_get_terminal_size_columns_stdout_is_used(self): + columns = get_terminal_size_columns() + self.assertEqual(columns, 444) + + @mock.patch('struct.unpack', mock.Mock(side_effect=Exception('a'))) + @mock.patch('subprocess.Popen') + def test_get_terminal_size_subprocess_popen_is_used(self, mock_popen): + mock_communicate = mock.Mock(return_value=['555 666']) + + mock_process = mock.Mock() + mock_process.returncode = 0 + mock_process.communicate = mock_communicate + + mock_popen.return_value = mock_process + + columns = get_terminal_size_columns() + self.assertEqual(columns, 666) + + @mock.patch('struct.unpack', mock.Mock(side_effect=Exception('a'))) + @mock.patch('subprocess.Popen', mock.Mock(side_effect=Exception('b'))) + def test_get_terminal_size_default_values_are_used(self): + columns = get_terminal_size_columns() + + self.assertEqual(columns, DEFAULT_TERMINAL_SIZE_COLUMNS) diff --git a/st2common/dist_utils.py b/st2common/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2common/dist_utils.py +++ b/st2common/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2common/st2common/util/secrets.py b/st2common/st2common/util/secrets.py index af12aa92b6..57d8a0b08f 100644 --- a/st2common/st2common/util/secrets.py +++ b/st2common/st2common/util/secrets.py @@ -66,6 +66,17 @@ def get_secret_parameters(parameters): secret_parameters = {} parameters_type = parameters.get('type') + # If the parameter itself is secret, then skip all processing below it + # and return the type of this parameter. + # + # **This causes the _full_ object / array tree to be secret (no children will be shown).** + # + # **Important** that we do this check first, so in case this parameter + # is an `object` or `array`, and the user wants the full thing + # to be secret, that it is marked as secret. + if parameters.get('secret', False): + return parameters_type + iterator = None if parameters_type == 'object': # if this is an object, then iterate over the properties within @@ -95,7 +106,24 @@ def get_secret_parameters(parameters): continue parameter_type = options.get('type') - if parameter_type in ['object', 'array']: + if options.get('secret', False): + # If this parameter is secret, then add it our secret parameters + # + # **This causes the _full_ object / array tree to be secret + # (no children will be shown)** + # + # **Important** that we do this check first, so in case this parameter + # is an `object` or `array`, and the user wants the full thing + # to be secret, that it is marked as secret. + if isinstance(secret_parameters, list): + secret_parameters.append(parameter_type) + elif isinstance(secret_parameters, dict): + secret_parameters[parameter] = parameter_type + else: + return parameter_type + elif parameter_type in ['object', 'array']: + # otherwise recursively dive into the `object`/`array` and + # find individual parameters marked as secret sub_params = get_secret_parameters(options) if sub_params: if isinstance(secret_parameters, list): @@ -104,14 +132,6 @@ def get_secret_parameters(parameters): secret_parameters[parameter] = sub_params else: return sub_params - elif options.get('secret', False): - # if this parameter is secret, then add it our secret parameters - if isinstance(secret_parameters, list): - secret_parameters.append(parameter_type) - elif isinstance(secret_parameters, dict): - secret_parameters[parameter] = parameter_type - else: - return parameter_type return secret_parameters diff --git a/st2common/st2common/util/spec_loader.py b/st2common/st2common/util/spec_loader.py index c2ce023017..743d897ebb 100644 --- a/st2common/st2common/util/spec_loader.py +++ b/st2common/st2common/util/spec_loader.py @@ -39,7 +39,7 @@ def load_spec(module_name, spec_file): spec_string = generate_spec(module_name, spec_file) - spec = yaml.load(spec_string) + spec = yaml.safe_load(spec_string) return spec diff --git a/st2common/tests/unit/test_jinja_render_data_filters.py b/st2common/tests/unit/test_jinja_render_data_filters.py index 864afe33b8..594de540da 100644 --- a/st2common/tests/unit/test_jinja_render_data_filters.py +++ b/st2common/tests/unit/test_jinja_render_data_filters.py @@ -96,5 +96,5 @@ def test_filter_to_yaml_string(self): template = '{{k1 | to_yaml_string}}' obj_yaml_str = env.from_string(template).render({'k1': obj}) - actual_obj = yaml.load(obj_yaml_str) + actual_obj = yaml.safe_load(obj_yaml_str) self.assertDictEqual(obj, actual_obj) diff --git a/st2common/tests/unit/test_util_secrets.py b/st2common/tests/unit/test_util_secrets.py index 848b372574..b110492180 100644 --- a/st2common/tests/unit/test_util_secrets.py +++ b/st2common/tests/unit/test_util_secrets.py @@ -145,6 +145,32 @@ ################################################################################ +TEST_ROOT_OBJECT_SCHEMA = { + 'description': 'root', + 'type': 'object', + 'properties': { + 'arg_level_one': { + 'description': 'down', + 'type': 'object', + 'properties': { + 'secret_field_in_object': { + 'type': 'string', + 'secret': True + } + } + } + } +} + +TEST_ROOT_OBJECT_SECRET_PARAMS = { + 'arg_level_one': + { + 'secret_field_in_object': 'string' + } +} + +################################################################################ + TEST_NESTED_ARRAYS_SCHEMA = { 'arg_optional_array': { 'description': 'Mirror', @@ -319,6 +345,208 @@ ] } +################################################################################ + +TEST_SECRET_ARRAY_SCHEMA = { + 'arg_secret_array': { + 'description': 'Mirror', + 'type': 'array', + 'secret': True, + } +} + +TEST_SECRET_ARRAY_SECRET_PARAMS = { + 'arg_secret_array': 'array' +} + +################################################################################ + +TEST_SECRET_OBJECT_SCHEMA = { + 'arg_secret_object': { + 'type': 'object', + 'secret': True, + } +} + +TEST_SECRET_OBJECT_SECRET_PARAMS = { + 'arg_secret_object': 'object' +} + + +################################################################################ + +TEST_SECRET_ROOT_ARRAY_SCHEMA = { + 'description': 'secret array', + 'type': 'array', + 'secret': True, + 'items': { + 'description': 'down', + 'type': 'object', + 'properties': { + 'secret_field_in_object': { + 'type': 'string', + 'secret': True + } + } + } +} + +TEST_SECRET_ROOT_ARRAY_SECRET_PARAMS = 'array' + +################################################################################ + +TEST_SECRET_ROOT_OBJECT_SCHEMA = { + 'description': 'secret object', + 'type': 'object', + 'secret': True, + 'proeprteis': { + 'arg_level_one': { + 'description': 'down', + 'type': 'object', + 'properties': { + 'secret_field_in_object': { + 'type': 'string', + 'secret': True + } + } + } + } +} + +TEST_SECRET_ROOT_OBJECT_SECRET_PARAMS = 'object' + +################################################################################ + +TEST_SECRET_NESTED_OBJECTS_SCHEMA = { + 'arg_object': { + 'description': 'Mirror', + 'type': 'object', + 'properties': { + 'arg_nested_object': { + 'description': 'Mirror mirror', + 'type': 'object', + 'secret': True, + 'properties': { + 'arg_double_nested_secret': { + 'description': 'Deep, deep down', + 'type': 'string', + 'secret': True + } + } + }, + 'arg_nested_secret': { + 'description': 'Deep down', + 'type': 'string', + 'secret': True + } + } + }, + 'arg_secret_object': { + 'description': 'Mirror', + 'type': 'object', + 'secret': True, + 'properties': { + 'arg_nested_object': { + 'description': 'Mirror mirror', + 'type': 'object', + 'secret': True, + 'properties': { + 'arg_double_nested_secret': { + 'description': 'Deep, deep down', + 'type': 'string', + 'secret': True + } + } + }, + 'arg_nested_secret': { + 'description': 'Deep down', + 'type': 'string', + 'secret': True + } + } + } +} + +TEST_SECRET_NESTED_OBJECTS_SECRET_PARAMS = { + 'arg_object': { + 'arg_nested_secret': 'string', + 'arg_nested_object': 'object' + }, + 'arg_secret_object': 'object' +} + + +################################################################################ + +TEST_SECRET_NESTED_ARRAYS_SCHEMA = { + 'arg_optional_array': { + 'description': 'Mirror', + 'type': 'array', + 'secret': True, + 'items': { + 'description': 'Deep down', + 'type': 'string' + } + }, + 'arg_optional_double_array': { + 'description': 'Mirror', + 'type': 'array', + 'secret': True, + 'items': { + 'type': 'array', + 'items': { + 'description': 'Deep down', + 'type': 'string', + } + } + }, + 'arg_optional_tripple_array': { + 'description': 'Mirror', + 'type': 'array', + 'items': { + 'type': 'array', + 'secret': True, + 'items': { + 'type': 'array', + 'items': { + 'description': 'Deep down', + 'type': 'string', + } + } + } + }, + 'arg_optional_quad_array': { + 'description': 'Mirror', + 'type': 'array', + 'items': { + 'type': 'array', + 'items': { + 'type': 'array', + 'secret': True, + 'items': { + 'type': 'array', + 'items': { + 'description': 'Deep down', + 'type': 'string', + } + } + } + } + } +} + +TEST_SECRET_NESTED_ARRAYS_SECRET_PARAMS = { + 'arg_optional_array': 'array', + 'arg_optional_double_array': 'array', + 'arg_optional_tripple_array': [ + 'array' + ], + 'arg_optional_quad_array': [ + [ + 'array' + ] + ] +} ################################################################################ @@ -345,6 +573,10 @@ def test_get_secret_parameters_root_array(self): result = secrets.get_secret_parameters(TEST_ROOT_ARRAY_SCHEMA) self.assertEqual(TEST_ROOT_ARRAY_SECRET_PARAMS, result) + def test_get_secret_parameters_root_object(self): + result = secrets.get_secret_parameters(TEST_ROOT_OBJECT_SCHEMA) + self.assertEqual(TEST_ROOT_OBJECT_SECRET_PARAMS, result) + def test_get_secret_parameters_nested_arrays(self): result = secrets.get_secret_parameters(TEST_NESTED_ARRAYS_SCHEMA) self.assertEqual(TEST_NESTED_ARRAYS_SECRET_PARAMS, result) @@ -361,8 +593,31 @@ def test_get_secret_parameters_nested_array_with_object(self): result = secrets.get_secret_parameters(TEST_NESTED_ARRAY_WITH_OBJECT_SCHEMA) self.assertEqual(TEST_NESTED_ARRAY_WITH_OBJECT_SECRET_PARAMS, result) + def test_get_secret_parameters_secret_array(self): + result = secrets.get_secret_parameters(TEST_SECRET_ARRAY_SCHEMA) + self.assertEqual(TEST_SECRET_ARRAY_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_object(self): + result = secrets.get_secret_parameters(TEST_SECRET_OBJECT_SCHEMA) + self.assertEqual(TEST_SECRET_OBJECT_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_root_array(self): + result = secrets.get_secret_parameters(TEST_SECRET_ROOT_ARRAY_SCHEMA) + self.assertEqual(TEST_SECRET_ROOT_ARRAY_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_root_object(self): + result = secrets.get_secret_parameters(TEST_SECRET_ROOT_OBJECT_SCHEMA) + self.assertEqual(TEST_SECRET_ROOT_OBJECT_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_nested_arrays(self): + result = secrets.get_secret_parameters(TEST_SECRET_NESTED_ARRAYS_SCHEMA) + self.assertEqual(TEST_SECRET_NESTED_ARRAYS_SECRET_PARAMS, result) + + def test_get_secret_parameters_secret_nested_objects(self): + result = secrets.get_secret_parameters(TEST_SECRET_NESTED_OBJECTS_SCHEMA) + self.assertEqual(TEST_SECRET_NESTED_OBJECTS_SECRET_PARAMS, result) + ############################################################################ - # TODO unit tests for mask_secret_parameters def test_mask_secret_parameters_flat(self): parameters = { @@ -459,6 +714,23 @@ def test_mask_secret_parameters_root_array(self): ] self.assertEqual(expected, result) + def test_mask_secret_parameters_root_object(self): + parameters = { + 'arg_level_one': + { + 'secret_field_in_object': 'Secret $tr!ng' + } + } + + result = secrets.mask_secret_parameters(parameters, TEST_ROOT_OBJECT_SECRET_PARAMS) + expected = { + 'arg_level_one': + { + 'secret_field_in_object': MASKED_ATTRIBUTE_VALUE + } + } + self.assertEqual(expected, result) + def test_mask_secret_parameters_nested_arrays(self): parameters = { 'arg_optional_array': [ @@ -669,3 +941,159 @@ def test_mask_secret_parameters_nested_array_with_object(self): ] } self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_array(self): + parameters = { + 'arg_secret_array': [ + "abc", + 123, + True + ] + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_ARRAY_SECRET_PARAMS) + expected = { + 'arg_secret_array': MASKED_ATTRIBUTE_VALUE + } + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_object(self): + parameters = { + 'arg_secret_object': + { + "abc": 123, + "key": "value", + "bool": True, + "array": ["x", "y", "z"], + "obj": + { + "x": "deep" + } + } + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_OBJECT_SECRET_PARAMS) + expected = { + 'arg_secret_object': MASKED_ATTRIBUTE_VALUE + } + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_root_array(self): + parameters = [ + "abc", + 123, + True + ] + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_ROOT_ARRAY_SECRET_PARAMS) + expected = MASKED_ATTRIBUTE_VALUE + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_root_object(self): + parameters = { + 'arg_level_one': + { + 'secret_field_in_object': 'Secret $tr!ng' + } + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_ROOT_OBJECT_SECRET_PARAMS) + expected = MASKED_ATTRIBUTE_VALUE + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_nested_arrays(self): + parameters = { + 'arg_optional_array': [ + 'secret 1', + 'secret 2', + 'secret 3', + ], + 'arg_optional_double_array': [ + [ + 'secret 4', + 'secret 5', + 'secret 6', + ], + [ + 'secret 7', + 'secret 8', + 'secret 9', + ] + ], + 'arg_optional_tripple_array': [ + [ + [ + 'secret 10', + 'secret 11' + ], + [ + 'secret 12', + 'secret 13', + 'secret 14' + ] + ], + [ + [ + 'secret 15', + 'secret 16' + ] + ] + ], + 'arg_optional_quad_array': [ + [ + [ + [ + 'secret 17', + 'secret 18' + ], + [ + 'secret 19' + ] + ] + ] + ] + } + + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_NESTED_ARRAYS_SECRET_PARAMS) + + expected = { + 'arg_optional_array': MASKED_ATTRIBUTE_VALUE, + 'arg_optional_double_array': MASKED_ATTRIBUTE_VALUE, + 'arg_optional_tripple_array': [ + MASKED_ATTRIBUTE_VALUE, + MASKED_ATTRIBUTE_VALUE, + ], + 'arg_optional_quad_array': [ + [ + MASKED_ATTRIBUTE_VALUE, + ] + ] + } + self.assertEqual(expected, result) + + def test_mask_secret_parameters_secret_nested_objects(self): + parameters = { + 'arg_object': { + 'arg_nested_secret': 'nested Secret', + 'arg_nested_object': { + 'arg_double_nested_secret': 'double nested $ecret', + } + }, + 'arg_secret_object': { + 'arg_nested_secret': 'secret data', + 'arg_nested_object': { + 'arg_double_nested_secret': 'double nested $ecret', + } + } + } + result = secrets.mask_secret_parameters(parameters, + TEST_SECRET_NESTED_OBJECTS_SECRET_PARAMS) + expected = { + 'arg_object': { + 'arg_nested_secret': MASKED_ATTRIBUTE_VALUE, + 'arg_nested_object': MASKED_ATTRIBUTE_VALUE, + }, + 'arg_secret_object': MASKED_ATTRIBUTE_VALUE, + } + self.assertEqual(expected, result) diff --git a/st2debug/dist_utils.py b/st2debug/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2debug/dist_utils.py +++ b/st2debug/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2exporter/dist_utils.py b/st2exporter/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2exporter/dist_utils.py +++ b/st2exporter/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2reactor/dist_utils.py b/st2reactor/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2reactor/dist_utils.py +++ b/st2reactor/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2stream/dist_utils.py b/st2stream/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2stream/dist_utils.py +++ b/st2stream/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2tests/dist_utils.py b/st2tests/dist_utils.py index ede9e4a358..965e6387da 100644 --- a/st2tests/dist_utils.py +++ b/st2tests/dist_utils.py @@ -54,13 +54,15 @@ ] -def check_pip_version(): +def check_pip_version(min_version='6.0.0'): """ Ensure that a minimum supported version of pip is installed. """ - if StrictVersion(pip.__version__) < StrictVersion('6.0.0'): - print("Upgrade pip, your version `{0}' " - "is outdated:\n{1}".format(pip.__version__, GET_PIP)) + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) sys.exit(1) @@ -71,8 +73,10 @@ def fetch_requirements(requirements_file_path): links = [] reqs = [] for req in parse_requirements(requirements_file_path, session=False): - if req.link: - links.append(str(req.link)) + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) reqs.append(str(req.req)) return (reqs, links) diff --git a/st2tests/integration/mistral/test_filters.py b/st2tests/integration/mistral/test_filters.py index ed26075c9f..916f52a42f 100644 --- a/st2tests/integration/mistral/test_filters.py +++ b/st2tests/integration/mistral/test_filters.py @@ -243,8 +243,8 @@ def test_to_yaml_string(self): ex = self._execute_workflow('examples.mistral-test-func-to-yaml-string', params) ex = self._wait_for_completion(ex) self.assertEqual(ex.status, action_constants.LIVEACTION_STATUS_SUCCEEDED) - jinja_dict = yaml.load(ex.result['result_jinja']) - yaql_dict = yaml.load(ex.result['result_yaql']) + jinja_dict = yaml.safe_load(ex.result['result_jinja']) + yaql_dict = yaml.safe_load(ex.result['result_yaql']) self.assertTrue(isinstance(jinja_dict, dict)) self.assertEqual(jinja_dict['a'], 'b') self.assertTrue(isinstance(yaql_dict, dict))