From b79cc6921d9e0c2adf9f4524979411bafdec7dc5 Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Tue, 5 Mar 2019 00:51:42 -0500 Subject: [PATCH 1/5] Added a shlex.split() wrapper to have a common way of calling it. Replaced parse_quoted_string with _get_command_arg_list. --- cmd2/cmd2.py | 83 ++++++++++++++++++++++++------------------ cmd2/parsing.py | 12 +++++- tests/test_argparse.py | 12 ------ 3 files changed, 58 insertions(+), 49 deletions(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 66de84736..070849b72 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -150,24 +150,6 @@ def categorize(func: Union[Callable, Iterable], category: str) -> None: setattr(func, HELP_CATEGORY, category) -def parse_quoted_string(string: str, preserve_quotes: bool) -> List[str]: - """ - Parse a quoted string into a list of arguments - :param string: the string being parsed - :param preserve_quotes: if True, then quotes will not be stripped - """ - if isinstance(string, list): - # arguments are already a list, return the list we were passed - lexed_arglist = string - else: - # Use shlex to split the command line into a list of arguments based on shell rules - lexed_arglist = shlex.split(string, comments=False, posix=False) - - if not preserve_quotes: - lexed_arglist = [utils.strip_quotes(arg) for arg in lexed_arglist] - return lexed_arglist - - def with_category(category: str) -> Callable: """A decorator to apply a category to a command function.""" def cat_decorator(func): @@ -176,10 +158,37 @@ def cat_decorator(func): return cat_decorator +def _get_command_arg_list(to_parse: Union[str, Statement], preserve_quotes: bool) -> List[str]: + """ + Called by the argument_list and argparse wrappers to retrieve just the arguments being + passed to their do_* methods as a list. + + :param to_parse: what is being passed to the do_* method. It can be one of two types: + 1. An already parsed Statement + 2. An argument string in cases where a do_* method is explicitly called + e.g.: Calling do_help('alias create') would cause to_parse to be 'alias create' + + :param preserve_quotes: if True, then quotes will not be stripped from the arguments + :return: the arguments in a list + """ + if isinstance(to_parse, Statement): + # In the case of a Statement, we already have what we need + if preserve_quotes: + return to_parse.arg_list + else: + return to_parse.argv[1:] + else: + # We only have the argument string. Use the parser to split this string. + parsed_arglist = StatementParser.shlex_split(to_parse) + if not preserve_quotes: + parsed_arglist = [utils.strip_quotes(arg) for arg in parsed_arglist] + + return parsed_arglist + + def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) -> Callable[[List], Optional[bool]]: """A decorator to alter the arguments passed to a do_* cmd2 method. Default passes a string of whatever the user - typed. With this decorator, the decorated method will receive a list of arguments parsed from user input using - shlex.split(). + typed. With this decorator, the decorated method will receive a list of arguments parsed from user input. :param args: Single-element positional argument list containing do_* method this decorator is wrapping :param preserve_quotes: if True, then argument quotes will not be stripped @@ -189,9 +198,9 @@ def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) -> def arg_decorator(func: Callable): @functools.wraps(func) - def cmd_wrapper(self, cmdline): - lexed_arglist = parse_quoted_string(cmdline, preserve_quotes) - return func(self, lexed_arglist) + def cmd_wrapper(cmd2_instance, statement: Union[str, Statement]): + parsed_arglist = _get_command_arg_list(statement, preserve_quotes) + return func(cmd2_instance, parsed_arglist) cmd_wrapper.__doc__ = func.__doc__ return cmd_wrapper @@ -214,16 +223,17 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, preserve import functools # noinspection PyProtectedMember - def arg_decorator(func: Callable[[Statement], Optional[bool]]): + def arg_decorator(func: Callable): @functools.wraps(func) - def cmd_wrapper(instance, cmdline): - lexed_arglist = parse_quoted_string(cmdline, preserve_quotes) + def cmd_wrapper(cmd2_instance, statement: Union[str, Statement]): + parsed_arglist = _get_command_arg_list(statement, preserve_quotes) + try: - args, unknown = argparser.parse_known_args(lexed_arglist) + args, unknown = argparser.parse_known_args(parsed_arglist) except SystemExit: return else: - return func(instance, args, unknown) + return func(cmd2_instance, args, unknown) # argparser defaults the program name to sys.argv[0] # we want it to be the name of our command @@ -256,16 +266,18 @@ def with_argparser(argparser: argparse.ArgumentParser, import functools # noinspection PyProtectedMember - def arg_decorator(func: Callable[[Statement], Optional[bool]]): + def arg_decorator(func: Callable): @functools.wraps(func) - def cmd_wrapper(instance, cmdline): - lexed_arglist = parse_quoted_string(cmdline, preserve_quotes) + def cmd_wrapper(cmd2_instance, statement: Union[str, Statement]): + + parsed_arglist = _get_command_arg_list(statement, preserve_quotes) + try: - args = argparser.parse_args(lexed_arglist) + args = argparser.parse_args(parsed_arglist) except SystemExit: return else: - return func(instance, args) + return func(cmd2_instance, args) # argparser defaults the program name to sys.argv[0] # we want it to be the name of our command @@ -742,8 +754,7 @@ def tokens_for_completion(self, line: str, begidx: int, endidx: int) -> Tuple[Li # Parse the line into tokens while True: try: - # Use non-POSIX parsing to keep the quotes around the tokens - initial_tokens = shlex.split(tmp_line[:tmp_endidx], comments=False, posix=False) + initial_tokens = StatementParser.shlex_split(tmp_line[:tmp_endidx]) # If the cursor is at an empty token outside of a quoted string, # then that is the token being completed. Add it to the list. @@ -1735,7 +1746,7 @@ def _run_cmdfinalization_hooks(self, stop: bool, statement: Optional[Statement]) # Fix those annoying problems that occur with terminal programs like "less" when you pipe to them if self.stdin.isatty(): import subprocess - proc = subprocess.Popen(shlex.split('stty sane')) + proc = subprocess.Popen(['stty', 'sane']) proc.communicate() try: diff --git a/cmd2/parsing.py b/cmd2/parsing.py index 5ec13fb7e..86227b5b4 100644 --- a/cmd2/parsing.py +++ b/cmd2/parsing.py @@ -349,7 +349,7 @@ def tokenize(self, line: str) -> List[str]: return [] # split on whitespace - tokens = shlex.split(line, comments=False, posix=False) + tokens = StatementParser.shlex_split(line) # custom lexing tokens = self._split_on_punctuation(tokens) @@ -607,6 +607,16 @@ def _command_and_args(tokens: List[str]) -> Tuple[str, str]: return command, args + @staticmethod + def shlex_split(str_to_split: str) -> List[str]: + """ + A wrapper around shlex.split() that uses cmd2's preferred arguments + This allows other classes to easily call split() the same way StatementParser does + :param str_to_split: the string being split + :return: A list of tokens + """ + return shlex.split(str_to_split, comments=False, posix=False) + def _split_on_punctuation(self, tokens: List[str]) -> List[str]: """Further splits tokens from a command line using punctuation characters diff --git a/tests/test_argparse.py b/tests/test_argparse.py index f5948f031..74a03e288 100644 --- a/tests/test_argparse.py +++ b/tests/test_argparse.py @@ -72,14 +72,6 @@ def do_arglist(self, arglist): def do_preservelist(self, arglist): self.stdout.write('{}'.format(arglist)) - @cmd2.with_argument_list - @cmd2.with_argument_list - def do_arglisttwice(self, arglist): - if isinstance(arglist, list): - self.stdout.write(' '.join(arglist)) - else: - self.stdout.write('False') - known_parser = argparse.ArgumentParser() known_parser.add_argument('-p', '--piglatin', action='store_true', help='atinLay') known_parser.add_argument('-s', '--shout', action='store_true', help='N00B EMULATION MODE') @@ -178,10 +170,6 @@ def test_preservelist(argparse_app): out = run_cmd(argparse_app, 'preservelist foo "bar baz"') assert out[0] == "['foo', '\"bar baz\"']" -def test_arglist_decorator_twice(argparse_app): - out = run_cmd(argparse_app, 'arglisttwice "we should" get these') - assert out[0] == 'we should get these' - class SubcommandApp(cmd2.Cmd): """ Example cmd2 application where we a base command which has a couple sub-commands.""" From be14802dc008f09d20235356978b527d30178d40 Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Tue, 5 Mar 2019 01:10:18 -0500 Subject: [PATCH 2/5] Reversed arguments in a Union --- cmd2/cmd2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 070849b72..889640615 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -158,7 +158,7 @@ def cat_decorator(func): return cat_decorator -def _get_command_arg_list(to_parse: Union[str, Statement], preserve_quotes: bool) -> List[str]: +def _get_command_arg_list(to_parse: Union[Statement, str], preserve_quotes: bool) -> List[str]: """ Called by the argument_list and argparse wrappers to retrieve just the arguments being passed to their do_* methods as a list. @@ -198,7 +198,7 @@ def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) -> def arg_decorator(func: Callable): @functools.wraps(func) - def cmd_wrapper(cmd2_instance, statement: Union[str, Statement]): + def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): parsed_arglist = _get_command_arg_list(statement, preserve_quotes) return func(cmd2_instance, parsed_arglist) @@ -225,7 +225,7 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, preserve # noinspection PyProtectedMember def arg_decorator(func: Callable): @functools.wraps(func) - def cmd_wrapper(cmd2_instance, statement: Union[str, Statement]): + def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): parsed_arglist = _get_command_arg_list(statement, preserve_quotes) try: @@ -268,7 +268,7 @@ def with_argparser(argparser: argparse.ArgumentParser, # noinspection PyProtectedMember def arg_decorator(func: Callable): @functools.wraps(func) - def cmd_wrapper(cmd2_instance, statement: Union[str, Statement]): + def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): parsed_arglist = _get_command_arg_list(statement, preserve_quotes) From aea0220ceac4927a77377c3549ed2e8118079af4 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Tue, 5 Mar 2019 20:11:56 -0500 Subject: [PATCH 3/5] Fix flake8 failure --- cmd2/cmd2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 889640615..fc8953c91 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -35,7 +35,6 @@ import inspect import os import re -import shlex import sys import threading from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Type, Union, IO From 3d52ee70ae4612b3a32ee3a37cf2324f7ddb0828 Mon Sep 17 00:00:00 2001 From: Todd Leonhardt Date: Tue, 5 Mar 2019 20:39:25 -0500 Subject: [PATCH 4/5] Moved some utility functions from cmd2.py to parsing.py --- cmd2/cmd2.py | 38 +++++------------------------------- cmd2/parsing.py | 52 +++++++++++++++++++++++++++++++++++++------------ 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index fc8953c91..6ac44df29 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -47,7 +47,7 @@ from . import utils from .argparse_completer import AutoCompleter, ACArgumentParser, ACTION_ARG_CHOICES from .clipboard import can_clip, get_paste_buffer, write_to_paste_buffer -from .parsing import StatementParser, Statement, Macro, MacroArg +from .parsing import StatementParser, Statement, Macro, MacroArg, shlex_split, get_command_arg_list from .history import History, HistoryItem # Set up readline @@ -157,34 +157,6 @@ def cat_decorator(func): return cat_decorator -def _get_command_arg_list(to_parse: Union[Statement, str], preserve_quotes: bool) -> List[str]: - """ - Called by the argument_list and argparse wrappers to retrieve just the arguments being - passed to their do_* methods as a list. - - :param to_parse: what is being passed to the do_* method. It can be one of two types: - 1. An already parsed Statement - 2. An argument string in cases where a do_* method is explicitly called - e.g.: Calling do_help('alias create') would cause to_parse to be 'alias create' - - :param preserve_quotes: if True, then quotes will not be stripped from the arguments - :return: the arguments in a list - """ - if isinstance(to_parse, Statement): - # In the case of a Statement, we already have what we need - if preserve_quotes: - return to_parse.arg_list - else: - return to_parse.argv[1:] - else: - # We only have the argument string. Use the parser to split this string. - parsed_arglist = StatementParser.shlex_split(to_parse) - if not preserve_quotes: - parsed_arglist = [utils.strip_quotes(arg) for arg in parsed_arglist] - - return parsed_arglist - - def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) -> Callable[[List], Optional[bool]]: """A decorator to alter the arguments passed to a do_* cmd2 method. Default passes a string of whatever the user typed. With this decorator, the decorated method will receive a list of arguments parsed from user input. @@ -198,7 +170,7 @@ def with_argument_list(*args: List[Callable], preserve_quotes: bool = False) -> def arg_decorator(func: Callable): @functools.wraps(func) def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): - parsed_arglist = _get_command_arg_list(statement, preserve_quotes) + parsed_arglist = get_command_arg_list(statement, preserve_quotes) return func(cmd2_instance, parsed_arglist) cmd_wrapper.__doc__ = func.__doc__ @@ -225,7 +197,7 @@ def with_argparser_and_unknown_args(argparser: argparse.ArgumentParser, preserve def arg_decorator(func: Callable): @functools.wraps(func) def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): - parsed_arglist = _get_command_arg_list(statement, preserve_quotes) + parsed_arglist = get_command_arg_list(statement, preserve_quotes) try: args, unknown = argparser.parse_known_args(parsed_arglist) @@ -269,7 +241,7 @@ def arg_decorator(func: Callable): @functools.wraps(func) def cmd_wrapper(cmd2_instance, statement: Union[Statement, str]): - parsed_arglist = _get_command_arg_list(statement, preserve_quotes) + parsed_arglist = get_command_arg_list(statement, preserve_quotes) try: args = argparser.parse_args(parsed_arglist) @@ -753,7 +725,7 @@ def tokens_for_completion(self, line: str, begidx: int, endidx: int) -> Tuple[Li # Parse the line into tokens while True: try: - initial_tokens = StatementParser.shlex_split(tmp_line[:tmp_endidx]) + initial_tokens = shlex_split(tmp_line[:tmp_endidx]) # If the cursor is at an empty token outside of a quoted string, # then that is the token being completed. Add it to the list. diff --git a/cmd2/parsing.py b/cmd2/parsing.py index 86227b5b4..9c95ba75b 100644 --- a/cmd2/parsing.py +++ b/cmd2/parsing.py @@ -5,7 +5,7 @@ import os import re import shlex -from typing import List, Tuple, Dict +from typing import Dict, List, Tuple, Union import attr @@ -13,6 +13,16 @@ from . import utils +def shlex_split(str_to_split: str) -> List[str]: + """A wrapper around shlex.split() that uses cmd2's preferred arguments. + + This allows other classes to easily call split() the same way StatementParser does + :param str_to_split: the string being split + :return: A list of tokens + """ + return shlex.split(str_to_split, comments=False, posix=False) + + @attr.s(frozen=True) class MacroArg: """ @@ -226,6 +236,34 @@ def argv(self) -> List[str]: return rtn +def get_command_arg_list(to_parse: Union[Statement, str], preserve_quotes: bool) -> List[str]: + """ + Called by the argument_list and argparse wrappers to retrieve just the arguments being + passed to their do_* methods as a list. + + :param to_parse: what is being passed to the do_* method. It can be one of two types: + 1. An already parsed Statement + 2. An argument string in cases where a do_* method is explicitly called + e.g.: Calling do_help('alias create') would cause to_parse to be 'alias create' + + :param preserve_quotes: if True, then quotes will not be stripped from the arguments + :return: the arguments in a list + """ + if isinstance(to_parse, Statement): + # In the case of a Statement, we already have what we need + if preserve_quotes: + return to_parse.arg_list + else: + return to_parse.argv[1:] + else: + # We only have the argument string. Use the parser to split this string. + parsed_arglist = shlex_split(to_parse) + if not preserve_quotes: + parsed_arglist = [utils.strip_quotes(arg) for arg in parsed_arglist] + + return parsed_arglist + + class StatementParser: """Parse raw text into command components. @@ -349,7 +387,7 @@ def tokenize(self, line: str) -> List[str]: return [] # split on whitespace - tokens = StatementParser.shlex_split(line) + tokens = shlex_split(line) # custom lexing tokens = self._split_on_punctuation(tokens) @@ -607,16 +645,6 @@ def _command_and_args(tokens: List[str]) -> Tuple[str, str]: return command, args - @staticmethod - def shlex_split(str_to_split: str) -> List[str]: - """ - A wrapper around shlex.split() that uses cmd2's preferred arguments - This allows other classes to easily call split() the same way StatementParser does - :param str_to_split: the string being split - :return: A list of tokens - """ - return shlex.split(str_to_split, comments=False, posix=False) - def _split_on_punctuation(self, tokens: List[str]) -> List[str]: """Further splits tokens from a command line using punctuation characters From 38804d740f718c6e1db1e96845fae87b597a456e Mon Sep 17 00:00:00 2001 From: Kevin Van Brunt Date: Wed, 6 Mar 2019 08:43:17 -0500 Subject: [PATCH 5/5] Updated a comment and moved an import --- cmd2/cmd2.py | 2 +- cmd2/parsing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd2/cmd2.py b/cmd2/cmd2.py index 6ac44df29..b3a61212d 100644 --- a/cmd2/cmd2.py +++ b/cmd2/cmd2.py @@ -47,8 +47,8 @@ from . import utils from .argparse_completer import AutoCompleter, ACArgumentParser, ACTION_ARG_CHOICES from .clipboard import can_clip, get_paste_buffer, write_to_paste_buffer -from .parsing import StatementParser, Statement, Macro, MacroArg, shlex_split, get_command_arg_list from .history import History, HistoryItem +from .parsing import StatementParser, Statement, Macro, MacroArg, shlex_split, get_command_arg_list # Set up readline from .rl_utils import rl_type, RlType, rl_get_point, rl_set_prompt, vt100_support, rl_make_safe_prompt diff --git a/cmd2/parsing.py b/cmd2/parsing.py index 9c95ba75b..d72ca4ec5 100644 --- a/cmd2/parsing.py +++ b/cmd2/parsing.py @@ -256,7 +256,7 @@ def get_command_arg_list(to_parse: Union[Statement, str], preserve_quotes: bool) else: return to_parse.argv[1:] else: - # We only have the argument string. Use the parser to split this string. + # We have the arguments in a string. Use shlex to split it. parsed_arglist = shlex_split(to_parse) if not preserve_quotes: parsed_arglist = [utils.strip_quotes(arg) for arg in parsed_arglist]