diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1046b4adb1..adb08ce0b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,8 @@ Fixed completed. (bug fix) #4735 * Added better error handling to `contrib/linux/actions/dig.py` to inform if dig is not installed. Contributed by JP Bourget (@punkrokk Syncurity) #4732 +* Update ``dist_utils`` module which is bundled with ``st2client`` and other Python packages so it + doesn't depend on internal pip API and so it works with latest pip version. (bug fix) #4750 3.1.0 - June 27, 2019 --------------------- diff --git a/Makefile b/Makefile index 3b6f5b5fba..f6880e1b79 100644 --- a/Makefile +++ b/Makefile @@ -847,6 +847,7 @@ debs: # Copy over shared dist utils module which is needed by setup.py @for component in $(COMPONENTS_WITH_RUNNERS); do\ cp -f ./scripts/dist_utils.py $$component/dist_utils.py;\ + sed -i -e '1s;^;# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY\n;' $$component/dist_utils.py;\ done # Copy over CHANGELOG.RST, CONTRIBUTING.RST and LICENSE file to each component directory diff --git a/contrib/runners/action_chain_runner/dist_utils.py b/contrib/runners/action_chain_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/action_chain_runner/dist_utils.py +++ b/contrib/runners/action_chain_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/announcement_runner/dist_utils.py b/contrib/runners/announcement_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/announcement_runner/dist_utils.py +++ b/contrib/runners/announcement_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/http_runner/dist_utils.py b/contrib/runners/http_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/http_runner/dist_utils.py +++ b/contrib/runners/http_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/inquirer_runner/dist_utils.py b/contrib/runners/inquirer_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/inquirer_runner/dist_utils.py +++ b/contrib/runners/inquirer_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/local_runner/dist_utils.py b/contrib/runners/local_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/local_runner/dist_utils.py +++ b/contrib/runners/local_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/mistral_v2/dist_utils.py b/contrib/runners/mistral_v2/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/mistral_v2/dist_utils.py +++ b/contrib/runners/mistral_v2/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/noop_runner/dist_utils.py b/contrib/runners/noop_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/noop_runner/dist_utils.py +++ b/contrib/runners/noop_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/orquesta_runner/dist_utils.py b/contrib/runners/orquesta_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/orquesta_runner/dist_utils.py +++ b/contrib/runners/orquesta_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/python_runner/dist_utils.py b/contrib/runners/python_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/python_runner/dist_utils.py +++ b/contrib/runners/python_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/remote_runner/dist_utils.py b/contrib/runners/remote_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/remote_runner/dist_utils.py +++ b/contrib/runners/remote_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/contrib/runners/winrm_runner/dist_utils.py b/contrib/runners/winrm_runner/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/contrib/runners/winrm_runner/dist_utils.py +++ b/contrib/runners/winrm_runner/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/scripts/dist_utils.py b/scripts/dist_utils.py index 63b868b145..6e9fbfcb52 100644 --- a/scripts/dist_utils.py +++ b/scripts/dist_utils.py @@ -27,31 +27,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +67,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/scripts/dist_utils_old.py b/scripts/dist_utils_old.py new file mode 100644 index 0000000000..175a181f62 --- /dev/null +++ b/scripts/dist_utils_old.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +NOTE: This file is only used for backward compatibility tests with new dist_utils.py in +st2common/tests/unit/test_dist_utils.py. + +DO NOT USE THIS FILE ANYWHERE ELSE! +""" + +from __future__ import absolute_import + +import os +import re +import sys + +from distutils.version import StrictVersion + +# NOTE: This script can't rely on any 3rd party dependency so we need to use this code here +PY3 = sys.version_info[0] == 3 + +if PY3: + text_type = str +else: + text_type = unicode + +GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' + +try: + import pip + from pip import __version__ as pip_version +except ImportError as e: + print('Failed to import pip: %s' % (text_type(e))) + print('') + print('Download pip:\n%s' % (GET_PIP)) + sys.exit(1) + +try: + # pip < 10.0 + from pip.req import parse_requirements +except ImportError: + # pip >= 10.0 + + try: + from pip._internal.req.req_file import parse_requirements + except ImportError as e: + print('Failed to import parse_requirements from pip: %s' % (text_type(e))) + print('Using pip: %s' % (str(pip_version))) + sys.exit(1) + +__all__ = [ + 'check_pip_version', + 'fetch_requirements', + 'apply_vagrant_workaround', + 'get_version_string', + 'parse_version_string' +] + + +def check_pip_version(min_version='6.0.0'): + """ + Ensure that a minimum supported version of pip is installed. + """ + if StrictVersion(pip.__version__) < StrictVersion(min_version): + print("Upgrade pip, your version '{0}' " + "is outdated. Minimum required version is '{1}':\n{2}".format(pip.__version__, + min_version, + GET_PIP)) + sys.exit(1) + + +def fetch_requirements(requirements_file_path): + """ + Return a list of requirements and links by parsing the provided requirements file. + """ + links = [] + reqs = [] + for req in parse_requirements(requirements_file_path, session=False): + # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions + link = getattr(req, 'link', getattr(req, 'url', None)) + if link: + links.append(str(link)) + reqs.append(str(req.req)) + return (reqs, links) + + +def apply_vagrant_workaround(): + """ + Function which detects if the script is being executed inside vagrant and if it is, it deletes + "os.link" attribute. + Note: Without this workaround, setup.py sdist will fail when running inside a shared directory + (nfs / virtualbox shared folders). + """ + if os.environ.get('USER', None) == 'vagrant': + del os.link + + +def get_version_string(init_file): + """ + Read __version__ string for an init file. + """ + + with open(init_file, 'r') as fp: + content = fp.read() + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + content, re.M) + if version_match: + return version_match.group(1) + + raise RuntimeError('Unable to find version string in %s.' % (init_file)) + + +# alias for get_version_string +parse_version_string = get_version_string diff --git a/st2actions/dist_utils.py b/st2actions/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2actions/dist_utils.py +++ b/st2actions/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2api/dist_utils.py b/st2api/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2api/dist_utils.py +++ b/st2api/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2auth/dist_utils.py b/st2auth/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2auth/dist_utils.py +++ b/st2auth/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2client/dist_utils.py b/st2client/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2client/dist_utils.py +++ b/st2client/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2common/dist_utils.py b/st2common/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2common/dist_utils.py +++ b/st2common/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2common/tests/fixtures/requirements-used-for-tests.txt b/st2common/tests/fixtures/requirements-used-for-tests.txt new file mode 100644 index 0000000000..58352b169c --- /dev/null +++ b/st2common/tests/fixtures/requirements-used-for-tests.txt @@ -0,0 +1,23 @@ +# Don't edit this file. It's generated automatically! +RandomWords +amqp==2.4.2 +argcomplete +bcrypt==3.1.6 +flex==6.14.0 +# some +# more +# commments.... +git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper +git+https://github.com/StackStorm/orquesta.git@224c1a589a6007eb0598a62ee99d674e7836d369#egg=orquesta +git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient +git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file +-e git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper-editable +git+https://github.com/StackStorm/st2.git#egg=python_runner&subdirectory=contrib/runners/python_runner +hg+https://hg.repo/some_pkg.git#egg=SomePackageHq +svn+svn://svn.repo/some_pkg/trunk/@ma-branch#egg=SomePackageSvn +gitpython==2.1.11 +ose-timer==0.7.5 +oslo.config<1.13,>=1.12.1 +requests[security]<2.23.0,>=2.22.0 +retrying==1.3.3 +zake==0.2.2 diff --git a/st2common/tests/unit/test_dist_utils.py b/st2common/tests/unit/test_dist_utils.py new file mode 100644 index 0000000000..72f7aeb694 --- /dev/null +++ b/st2common/tests/unit/test_dist_utils.py @@ -0,0 +1,89 @@ +# Copyright 2019 Extreme Networks, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +import unittest2 + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +SCRIPTS_PATH = os.path.join(BASE_DIR, '../../../scripts/') + +# Add scripts/ which contain main dist_utils.py to PYTHONPATH +sys.path.insert(0, SCRIPTS_PATH) + +from dist_utils import fetch_requirements +from dist_utils_old import fetch_requirements as old_fetch_requirements + +__all__ = [ + 'DistUtilsTestCase' +] + +REQUIREMENTS_PATH_1 = os.path.join(BASE_DIR, '../fixtures/requirements-used-for-tests.txt') +REQUIREMENTS_PATH_2 = os.path.join(BASE_DIR, '../../../requirements.txt') + + +class DistUtilsTestCase(unittest2.TestCase): + def test_fetch_requirements(self): + expected_reqs = [ + 'RandomWords', + 'amqp==2.4.2', + 'argcomplete', + 'bcrypt==3.1.6', + 'flex==6.14.0', + 'logshipper', + 'orquesta', + 'python-mistralclient', + 'st2-auth-backend-flat-file', + 'logshipper-editable', + 'python_runner', + 'SomePackageHq', + 'SomePackageSvn', + 'gitpython==2.1.11', + 'ose-timer==0.7.5', + 'oslo.config<1.13,>=1.12.1', + 'requests[security]<2.23.0,>=2.22.0', + 'retrying==1.3.3', + 'zake==0.2.2' + ] + expected_links = [ + 'git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper', + 'git+https://github.com/StackStorm/orquesta.git@224c1a589a6007eb0598a62ee99d674e7836d369#egg=orquesta', # NOQA + 'git+https://github.com/StackStorm/python-mistralclient.git#egg=python-mistralclient', + 'git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file', # NOQA + 'git+https://github.com/Kami/logshipper.git@stackstorm_patched#egg=logshipper-editable', + 'git+https://github.com/StackStorm/st2.git#egg=python_runner&subdirectory=contrib/runners/python_runner', # NOQA + 'hg+https://hg.repo/some_pkg.git#egg=SomePackageHq', + 'svn+svn://svn.repo/some_pkg/trunk/@ma-branch#egg=SomePackageSvn' + ] + + reqs, links = fetch_requirements(REQUIREMENTS_PATH_1) + self.assertEqual(reqs, expected_reqs) + self.assertEqual(links, expected_links) + + # Verify output of old and new function is the same + reqs_old, links_old = old_fetch_requirements(REQUIREMENTS_PATH_1) + + self.assertEqual(reqs_old, expected_reqs) + self.assertEqual(links_old, expected_links) + + self.assertEqual(reqs_old, reqs) + self.assertEqual(links_old, links) + + # Also test it on requirements.txt in repo root + reqs, links = fetch_requirements(REQUIREMENTS_PATH_2) + reqs_old, links_old = old_fetch_requirements(REQUIREMENTS_PATH_2) + + self.assertEqual(reqs_old, reqs) + self.assertEqual(links_old, links) diff --git a/st2debug/dist_utils.py b/st2debug/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2debug/dist_utils.py +++ b/st2debug/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2exporter/dist_utils.py b/st2exporter/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2exporter/dist_utils.py +++ b/st2exporter/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2reactor/dist_utils.py b/st2reactor/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2reactor/dist_utils.py +++ b/st2reactor/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2stream/dist_utils.py b/st2stream/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2stream/dist_utils.py +++ b/st2stream/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links) diff --git a/st2tests/dist_utils.py b/st2tests/dist_utils.py index 63b868b145..370d5e0550 100644 --- a/st2tests/dist_utils.py +++ b/st2tests/dist_utils.py @@ -1,3 +1,4 @@ +# NOTE: This file is auto-generated - DO NOT EDIT MANUALLY # -*- coding: utf-8 -*- # Copyright 2019 Extreme Networks, Inc. # @@ -27,31 +28,18 @@ if PY3: text_type = str else: - text_type = unicode + text_type = unicode # NOQA GET_PIP = 'curl https://bootstrap.pypa.io/get-pip.py | python' try: import pip - from pip import __version__ as pip_version except ImportError as e: print('Failed to import pip: %s' % (text_type(e))) print('') print('Download pip:\n%s' % (GET_PIP)) sys.exit(1) -try: - # pip < 10.0 - from pip.req import parse_requirements -except ImportError: - # pip >= 10.0 - - try: - from pip._internal.req.req_file import parse_requirements - except ImportError as e: - print('Failed to import parse_requirements from pip: %s' % (text_type(e))) - print('Using pip: %s' % (str(pip_version))) - sys.exit(1) __all__ = [ 'check_pip_version', @@ -80,12 +68,43 @@ def fetch_requirements(requirements_file_path): """ links = [] reqs = [] - for req in parse_requirements(requirements_file_path, session=False): - # Note: req.url was used before 9.0.0 and req.link is used in all the recent versions - link = getattr(req, 'link', getattr(req, 'url', None)) - if link: - links.append(str(link)) - reqs.append(str(req.req)) + + def _get_link(line): + vcs_prefixes = ['git+', 'svn+', 'hg+', 'bzr+'] + + for vcs_prefix in vcs_prefixes: + if line.startswith(vcs_prefix) or line.startswith('-e %s' % (vcs_prefix)): + req_name = re.findall('.*#egg=(.+)([&|@]).*$', line) + + if not req_name: + req_name = re.findall('.*#egg=(.+?)$', line) + else: + req_name = req_name[0] + + if not req_name: + raise ValueError('Line "%s" is missing "#egg="' % (line)) + + link = line.replace('-e ', '').strip() + return link, req_name[0] + + return None, None + + with open(requirements_file_path, 'r') as fp: + for line in fp.readlines(): + line = line.strip() + + if line.startswith('#') or not line: + continue + + link, req_name = _get_link(line=line) + + if link: + links.append(link) + else: + req_name = line + + reqs.append(req_name) + return (reqs, links)