From d31c5d784bc0cb6101bca95f4e423a88c7a83e89 Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 07:05:45 -0600 Subject: [PATCH 1/9] Call Python with sys.executable in tests so they work with venv on Windows --- layerstack/tests/test_layer.py | 6 +++++- layerstack/tests/test_layerbase.py | 2 +- setup.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/layerstack/tests/test_layer.py b/layerstack/tests/test_layer.py index bf9478b..4528c88 100644 --- a/layerstack/tests/test_layer.py +++ b/layerstack/tests/test_layer.py @@ -22,6 +22,7 @@ import json import subprocess +import sys import pytest @@ -77,7 +78,10 @@ def create_layer_library_dir(manage_outdir): def test_layer_base(): _layer_dir = Layer.create('Test Layer Base', created_layers_library_dir) # should be able to run the layer as-is - subprocess.check_call(['python', str(created_layers_library_dir / 'test_layer_base' / 'layer.py'), 'dummy_arg']) + subprocess.check_call([ + sys.executable, + str(created_layers_library_dir / 'test_layer_base' / 'layer.py'), + 'dummy_arg']) def test_model_dependent_args_kwargs(): diff --git a/layerstack/tests/test_layerbase.py b/layerstack/tests/test_layerbase.py index 7f5ab5a..937d578 100644 --- a/layerstack/tests/test_layerbase.py +++ b/layerstack/tests/test_layerbase.py @@ -32,7 +32,7 @@ def test_layer_cli(): test_list = ['1', '2', '3'] - args = ['python', str(layer_library_dir / 'test_list_args' / 'layer.py')] + args = [sys.executable, str(layer_library_dir / 'test_list_args' / 'layer.py')] args += test_list out_list = subprocess.Popen(args, stdout=PIPE, stderr=PIPE) diff --git a/setup.py b/setup.py index 8ea511e..3d041dc 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,8 @@ 'pytest-ordering', 'sphinx', 'sphinx_rtd_theme', - 'twine' + 'twine', + 'wheel' ] }, entry_points={ From a39463214705928aa6cf8a756f6bde027178130b Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 07:11:19 -0600 Subject: [PATCH 2/9] Fixes #22 --- layerstack/layer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/layerstack/layer.py b/layerstack/layer.py index 3b3f827..970e0c3 100644 --- a/layerstack/layer.py +++ b/layerstack/layer.py @@ -444,13 +444,13 @@ def create(cls, name, parent_dir, desc=None, layer_base_class=LayerBase): Parameters ---------- - name : 'str' + name : str Layer name - parent_dir : 'str' + parent_dir : str or pathlib.Path Parent directory for layer - desc : 'str' + desc : str Layer description - layer_base_class : 'LayerBase|ModelLayerBase' + layer_base_class : LayerBase or child class Base class on which to build layer Returns @@ -460,6 +460,7 @@ def create(cls, name, parent_dir, desc=None, layer_base_class=LayerBase): """ # Create the directory + parent_dir = Path(parent_dir) if not parent_dir.exists(): raise LayerStackError(f"The parent_dir {parent_dir} does not exist.") # maynot need the msg_begin here dir_name = name.lower().replace(" ", "_") From 3454b42db64a87ba40516534dd100c378681d941 Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 07:34:34 -0600 Subject: [PATCH 3/9] Fixing deprecation warning for collections.abc.MutableSequence --- layerstack/stack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/layerstack/stack.py b/layerstack/stack.py index 3ad7bb0..ff527d1 100644 --- a/layerstack/stack.py +++ b/layerstack/stack.py @@ -26,7 +26,8 @@ from __future__ import print_function, division, absolute_import import argparse -from collections import MutableSequence, OrderedDict +from collections import OrderedDict +from collections.abc import MutableSequence import json import logging from pathlib import Path From 6598bc45ca8a1c3b3ba3f9fa6aece76aca304fdf Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 07:35:14 -0600 Subject: [PATCH 4/9] Documenting #23 --- .../test_kwarg_name_clashes/layer.py | 62 +++++++++++++++++++ layerstack/tests/test_layerbase.py | 20 +++++- 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py diff --git a/layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py b/layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py new file mode 100644 index 0000000..25176a5 --- /dev/null +++ b/layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py @@ -0,0 +1,62 @@ +from __future__ import print_function, division, absolute_import + +from builtins import super +import logging +from uuid import UUID + +from layerstack.args import Arg, Kwarg +from layerstack.layer import LayerBase + +logger = logging.getLogger('layerstack.layers.TestManyKwargNameClashes') + + +class TestKwargNameClashes(LayerBase): + name = "Test kwarg name clashes" + uuid = UUID("88cfcb69-27c0-4664-a295-c909a61ad832") + version = '0.1.0' + desc = None + + @classmethod + def args(cls, model=None): + arg_list = super().args() + return arg_list + + @classmethod + def kwargs(cls): + ''' + Each layer must define its keyword arguments by populating and returning + a KwargDict object. + + Returns + ------- + KwargDict + KwargDict object describing the layer's keyword arguments. Keyword + argument specifications in the apply method should match what is + defined in this method (i.e., be equivalent to + Kwarg.name=Kwarg.default). + ''' + kwarg_dict = super().kwargs() + kwarg_dict['hit_rate'] = Kwarg( + description="Kwargs starting with h should be allowed") + return kwarg_dict + + @classmethod + def apply(cls, stack, hit_rate = None): + """ + No logic required--just testing kwarg-passing. + """ + return True + + +if __name__ == '__main__': + # Single-layer command-line interface entry point. + + # Parameters + # ---------- + # log_format : str + # custom logging format to use with the logging package via + # layerstack.start_console_log + # + TestKwargNameClashes.main() + + \ No newline at end of file diff --git a/layerstack/tests/test_layerbase.py b/layerstack/tests/test_layerbase.py index 937d578..2b68c85 100644 --- a/layerstack/tests/test_layerbase.py +++ b/layerstack/tests/test_layerbase.py @@ -43,5 +43,23 @@ def test_layer_cli(): assert stderr[-15:] == str(test_list), f"stdout:\n{stdout}\nstderr:\n{stderr}" -# *** perform similar split to main and parser as in stack.py for layer.py? *** +def test_kwarg_name_clashes(): + args = [sys.executable, str(layer_library_dir / 'test_kwarg_name_clashes' / 'layer.py')] + + # run help + ret = subprocess.run(args + ["--help"], capture_output=True) + assert not ret.returncode, print(ret.stderr) + logger.debug(f"In test_kwarg_name_clashes, stdout:\n{ret.stdout}\nstderr:\n" + f"{ret.stderr}\nafter calling --help") + + to_call = [ + "-hr", 0.2 + ] + + # run layer + ret = subprocess.Popen(args + to_call, stdout=PIPE, stderr=PIPE) + assert not ret.returncode, print(ret.stderr) + logger.debug(f"In test_kwarg_name_clashes, stdout:\n{ret.stdout}\nstderr:\n" + f"{ret.stderr}\nafter calling with kwargs") + From eceb3f05b216f5c355add84731c9ee01ce759e52 Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 07:53:09 -0600 Subject: [PATCH 5/9] Fixes #23 --- layerstack/layer.py | 3 +-- layerstack/tests/__init__.py | 5 +++++ layerstack/tests/test_layerbase.py | 16 ++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/layerstack/layer.py b/layerstack/layer.py index 970e0c3..fae5b36 100644 --- a/layerstack/layer.py +++ b/layerstack/layer.py @@ -133,8 +133,7 @@ def main(cls, log_format=DEFAULT_LOG_FORMAT): arg_list = cls.args() arg_list.add_arguments(parser) kwarg_dict = cls.kwargs() - # *** issue 23, likely just need to add an h to this list - kwarg_dict.add_arguments(parser, short_names=['r', 'd']) + kwarg_dict.add_arguments(parser, short_names=['r', 'd', 'h']) # Parse args and set values cli_args = parser.parse_args() diff --git a/layerstack/tests/__init__.py b/layerstack/tests/__init__.py index d2fa983..2391b99 100644 --- a/layerstack/tests/__init__.py +++ b/layerstack/tests/__init__.py @@ -5,3 +5,8 @@ here = Path(__file__).parent outdir = here / 'outputs' layer_library_dir = here / 'layer_library' + +def get_output_str(stdout_stderr): + if not isinstance(stdout_stderr, bytes): + stdout_stderr = stdout_stderr.read() # _io.BufferedReader + return stdout_stderr.decode("utf-8") diff --git a/layerstack/tests/test_layerbase.py b/layerstack/tests/test_layerbase.py index 2b68c85..e6cd48f 100644 --- a/layerstack/tests/test_layerbase.py +++ b/layerstack/tests/test_layerbase.py @@ -24,7 +24,7 @@ from subprocess import Popen, PIPE import sys -from layerstack.tests import layer_library_dir +from layerstack.tests import layer_library_dir, get_output_str logger = logging.getLogger(__name__) @@ -48,18 +48,18 @@ def test_kwarg_name_clashes(): # run help ret = subprocess.run(args + ["--help"], capture_output=True) - assert not ret.returncode, print(ret.stderr) - logger.debug(f"In test_kwarg_name_clashes, stdout:\n{ret.stdout}\nstderr:\n" - f"{ret.stderr}\nafter calling --help") + assert not ret.returncode, get_output_str(ret.stderr) + logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\nstderr:\n" + f"{get_output_str(ret.stderr)}\nafter calling --help") to_call = [ - "-hr", 0.2 + "-hr", str(0.2) ] # run layer ret = subprocess.Popen(args + to_call, stdout=PIPE, stderr=PIPE) - assert not ret.returncode, print(ret.stderr) - logger.debug(f"In test_kwarg_name_clashes, stdout:\n{ret.stdout}\nstderr:\n" - f"{ret.stderr}\nafter calling with kwargs") + assert not ret.returncode, get_output_str(ret.stderr) + logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\nstderr:\n" + f"{get_output_str(ret.stderr)}\nafter calling with kwargs") From 2bd6791d12a6fb19213801a5ebd15fd179af2219 Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 11:36:26 -0600 Subject: [PATCH 6/9] Generalizes get_short_name and fixes #24 --- layerstack/args.py | 101 ++++++++++++------ layerstack/layer.py | 1 - .../test_kwarg_name_clashes/layer.py | 12 ++- .../test_kwargs_with_dashes/layer.py | 84 +++++++++++++++ layerstack/tests/test_layerbase.py | 43 ++++++-- 5 files changed, 201 insertions(+), 40 deletions(-) create mode 100644 layerstack/tests/layer_library/test_kwargs_with_dashes/layer.py diff --git a/layerstack/args.py b/layerstack/args.py index cb3ebea..b35941b 100644 --- a/layerstack/args.py +++ b/layerstack/args.py @@ -100,6 +100,11 @@ def is_list(self): """ return self._is_list + def get_name(self, cleaned=True): + if cleaned: + return self.name.replace("-", "_") + return self.name + def _set_value(self, value): """ Called by derived classes to set their _value attribute. @@ -505,15 +510,16 @@ def set_args(self, cli_args): unexpected behavior. """ if not self.mode == ArgMode.USE: - raise LayerStackRuntimeError("{} ".format(self.__class__.__name__) + + raise LayerStackRuntimeError(f"{self.__class__.__name__} " "must be in ArgMode.USE to set values.") try: self.mode = ArgMode.DESC for arg in self: try: - temp_value = eval('cli_args.' + arg.name) + temp_value = getattr(cli_args, arg.get_name(cleaned=False)) except Exception as e: - raise LayerStackRuntimeError("{} not found in cli_args, {}".format(arg.name,e)) + raise LayerStackRuntimeError(f"{arg.get_name(cleaned=False)} not found in " + f"cli_args:\n{cli_args},\nbecause {e}") arg.value = temp_value # may throw if temp_value is bad per arg.parser, etc. self.mode = ArgMode.USE except Exception as e: @@ -621,7 +627,7 @@ def items(self): (if mode == ArgMode.DESC), or its .value (if mode == ArgMode.USE) """ if self.mode == ArgMode.USE: - return [(name, kwarg.value) for name, kwarg in super().items()] + return [(kwarg.get_name(), kwarg.value) for kwarg in super().values()] return super().items() def iteritems(self): @@ -636,7 +642,7 @@ def iteritems(self): ArgMode.USE) """ for name, kwarg in super().items(): - yield (name, kwarg) if self.mode == ArgMode.DESC else (name, kwarg.value) + yield (name, kwarg) if self.mode == ArgMode.DESC else (kwarg.get_name(), kwarg.value) def values(self): """ @@ -726,30 +732,8 @@ def add_arguments(self, parser, short_names=[]): raise LayerStackRuntimeError("{} ".format(self.__class__.__name__) + "must be in ArgMode.DESC to add arguments to an argparse parser.") - def get_short_name(name): - short_name = None; n = 0 - - # first try splitting on '_' to get good letters - n = 0 - chars = "".join([x[0] for x in name.split('_')]) - while not short_name: - n += 1 - if chars[:n] not in short_names: - short_name = chars[:n] - - # now just use all characters - n = 1 - while not short_name: - n += 1 - if name[:n] not in short_names: - short_name = name[:n] - - assert short_name - short_names.append(short_name) - return short_name - for name, kwarg in self.items(): - short_name = get_short_name(name) + short_name = get_short_name(name, short_names=short_names) kwarg_kwargs = kwarg.add_argument_kwargs() try: parser.add_argument('-' + short_name, '--' + name, @@ -790,12 +774,67 @@ def set_kwargs(self, cli_args): self.mode = ArgMode.DESC for key, kwarg in self.items(): try: - temp_value = eval('cli_args.' + key) + temp_value = getattr(cli_args, kwarg.get_name()) except Exception as e: - raise LayerStackRuntimeError("{} not found in cli_args, {}".format(key,e)) + raise LayerStackRuntimeError(f"{kwarg.get_name()} not found in " + f"cli_args:\n{cli_args},\nbecause {e}") kwarg.value = temp_value # may throw if temp_value is bad per kwarg.parser, etc. self.mode = ArgMode.USE except Exception as e: self.mode = ArgMode.USE raise e - \ No newline at end of file + + +def get_short_name(name, short_names = [], seps = ['_', '-']): + short_name = None + + def multisplit(astr, seps): + parts = None + for sep in seps: + if parts is None: + parts = astr.split(sep) + continue + tmp = [] + for part in parts: + tmp.extend(part.split(sep)) + parts = tmp + return parts + + parts = multisplit(name, seps) + len_parts = [len(part) for part in parts] + + M = 1; N = len(parts); candidates = set(); k = 0 + logger.debug(parts) + while not short_name: + if k and (len(candidates) == k): + raise LayerStackRuntimeError("Unable to find a short name " + "for {name}. Current short names:\n[\n {short_names}\n]" + "\nStarted with parts = {parts},\nevaluated " + "candidates:\n[\n {candidates}\n]".format( + name = name, + short_names = ",\n ".join([repr(sn) for sn in short_names]), + parts = parts, + candidates = ",\n ".join([repr(candi) for candi in candidates]) + )) + n = 1; k = len(candidates) + logger.debug(f"M = {M}; N = {N}; k = {k}") + while n <= N: + candidate = '' + for i in range(n): + m = min(len_parts[i], M) + candidate += parts[i][:m] + logger.debug(f" n = {n}; i = {i}; m = {m}; {candidate}") + for i in range(n,N): + mm1 = min(len_parts[i], M - 1) + candidate += parts[i][:mm1] + logger.debug(f" n = {n}; N = {N}; i = {i}; mm1 = {mm1}; {candidate}") + if candidate not in short_names: + short_name = candidate + break + candidates.add(candidate) + n += 1 + M += 1 + + assert short_name + short_names.append(short_name) + return short_name diff --git a/layerstack/layer.py b/layerstack/layer.py index fae5b36..d7d217f 100644 --- a/layerstack/layer.py +++ b/layerstack/layer.py @@ -199,7 +199,6 @@ def _main_apply(cls, cli_args, arg_list, kwarg_dict): assert arg_list.mode == ArgMode.USE assert kwarg_dict.mode == ArgMode.USE from layerstack.stack import Stack - # TODO: Fix KwargDict so **kwarg_dict works natively return cls.apply(Stack(run_dir=cli_args.run_dir), *arg_list, **{k: v for k, v in kwarg_dict.items()}) diff --git a/layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py b/layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py index 25176a5..daf71c2 100644 --- a/layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py +++ b/layerstack/tests/layer_library/test_kwarg_name_clashes/layer.py @@ -38,10 +38,20 @@ def kwargs(cls): kwarg_dict = super().kwargs() kwarg_dict['hit_rate'] = Kwarg( description="Kwargs starting with h should be allowed") + kwarg_dict['hearth_rug_dog'] = Kwarg( + description="Short name should be -hrd" + ) + kwarg_dict['heart_rate'] = Kwarg( + description="Short name should be -her" + ) + kwarg_dict['herself_running_dearly'] = Kwarg( + description="Look deep for a name that works" + ) return kwarg_dict @classmethod - def apply(cls, stack, hit_rate = None): + def apply(cls, stack, hit_rate = None, hearth_rug_dog = None, + heart_rate = None, herself_running_dearly = None): """ No logic required--just testing kwarg-passing. """ diff --git a/layerstack/tests/layer_library/test_kwargs_with_dashes/layer.py b/layerstack/tests/layer_library/test_kwargs_with_dashes/layer.py new file mode 100644 index 0000000..ec8afbc --- /dev/null +++ b/layerstack/tests/layer_library/test_kwargs_with_dashes/layer.py @@ -0,0 +1,84 @@ +from __future__ import print_function, division, absolute_import + +from builtins import super +import logging +from uuid import UUID + +from layerstack.args import Arg, Kwarg +from layerstack.layer import LayerBase + +logger = logging.getLogger('layerstack.layers.TestKwargsWithDashes') + + +class TestKwargsWithDashes(LayerBase): + name = "Test kwargs with dashes" + uuid = UUID("03369f30-656c-4154-9ec3-4b5f67736324") + version = '0.1.0' + desc = None + + @classmethod + def args(cls, model=None): + ''' + Each layer must define its positional arguments by populating and + returning an ArgList object. + + Returns + ------- + ArgList + ArgList object describing the layer's positional arguments. Arg + names should appear as positional arguments in the apply method in + the same order as they are defined here. + ''' + arg_list = super().args() + arg_list.append(Arg('positional-arg')) + return arg_list + + @classmethod + def kwargs(cls): + ''' + Each layer must define its keyword arguments by populating and returning + a KwargDict object. + + Returns + ------- + KwargDict + KwargDict object describing the layer's keyword arguments. Keyword + argument specifications in the apply method should match what is + defined in this method (i.e., be equivalent to + Kwarg.name=Kwarg.default). + ''' + kwarg_dict = super().kwargs() + kwarg_dict['hit-rate'] = Kwarg( + description="Kwargs starting with h should be allowed") + kwarg_dict['hearth-rug-dog'] = Kwarg( + description="Short name should be -hrd" + ) + kwarg_dict['heart_rate'] = Kwarg( + description="Short name should be -her" + ) + kwarg_dict['herself_running-dearly'] = Kwarg( + description="Look deep for a name that works" + ) + return kwarg_dict + + @classmethod + def apply(cls, stack, positional_arg, hit_rate = None, hearth_rug_dog = None, + heart_rate = None, herself_running_dearly = None): + ''' + No logic required--just testing arg and kwarg passing. + ''' + return True + + +if __name__ == '__main__': + # Single-layer command-line interface entry point. + + # Parameters + # ---------- + # log_format : str + # custom logging format to use with the logging package via + # layerstack.start_console_log + # + TestKwargsWithDashes.main() + + \ No newline at end of file diff --git a/layerstack/tests/test_layerbase.py b/layerstack/tests/test_layerbase.py index e6cd48f..002a71f 100644 --- a/layerstack/tests/test_layerbase.py +++ b/layerstack/tests/test_layerbase.py @@ -48,18 +48,47 @@ def test_kwarg_name_clashes(): # run help ret = subprocess.run(args + ["--help"], capture_output=True) + logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\n" + f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" + "after calling --help") assert not ret.returncode, get_output_str(ret.stderr) - logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\nstderr:\n" - f"{get_output_str(ret.stderr)}\nafter calling --help") - + to_call = [ - "-hr", str(0.2) + "-hr", str(0.2), + "-hrd", "Rufus", + "-her", str(85), + "-herd", "Anne" ] # run layer - ret = subprocess.Popen(args + to_call, stdout=PIPE, stderr=PIPE) + ret = subprocess.run(args + to_call, capture_output=True) + logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\n" + f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" + "after calling with kwargs") assert not ret.returncode, get_output_str(ret.stderr) - logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\nstderr:\n" - f"{get_output_str(ret.stderr)}\nafter calling with kwargs") +def test_kwargs_with_dashes(): + args = [sys.executable, str(layer_library_dir / 'test_kwargs_with_dashes' / 'layer.py')] + + # run help + ret = subprocess.run(args + ["--help"], capture_output=True) + logger.debug(f"In test_kwargs_with_dashes, stdout:\n{get_output_str(ret.stdout)}\n" + f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" + "after calling --help") + assert not ret.returncode, get_output_str(ret.stderr) + + to_call = [ + "-hr", str(0.2), + "-hrd", "Rufus", + "-her", str(85), + "-herd", "Anne", + str(734) + ] + + # run layer + ret = subprocess.run(args + to_call, capture_output=True) + logger.debug(f"In test_kwargs_with_dashes, stdout:\n{get_output_str(ret.stdout)}\n" + f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" + "after calling with kwargs") + assert not ret.returncode, get_output_str(ret.stderr) From 4bb5023d16054cfa9c0b11ef8cf7f6a39d3fdaa1 Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 11:37:29 -0600 Subject: [PATCH 7/9] Fixes #28 --- layerstack/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layerstack/stack.py b/layerstack/stack.py index ff527d1..cd80dff 100644 --- a/layerstack/stack.py +++ b/layerstack/stack.py @@ -696,7 +696,7 @@ def run(self, save_path=None, log_level=logging.INFO, archive=True): end_file_log(logfile) except: chdir(old_cur_dir) - logger.info(f"Stack failed after {timer_str(timer() - start)}") + logger.error(f"Stack failed after {timer_str(timer() - start)}") end_file_log(logfile) raise From 8e8d1212d600296e7e2e7aa116d14e1d680c7423 Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 11:55:08 -0600 Subject: [PATCH 8/9] Making tests Python 3.6 compatible --- layerstack/tests/__init__.py | 12 +++++++-- layerstack/tests/test_layerbase.py | 42 ++++++++++-------------------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/layerstack/tests/__init__.py b/layerstack/tests/__init__.py index 2391b99..2210117 100644 --- a/layerstack/tests/__init__.py +++ b/layerstack/tests/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import subprocess from pathlib import Path here = Path(__file__).parent @@ -9,4 +9,12 @@ def get_output_str(stdout_stderr): if not isinstance(stdout_stderr, bytes): stdout_stderr = stdout_stderr.read() # _io.BufferedReader - return stdout_stderr.decode("utf-8") + return stdout_stderr.decode('ascii').rstrip() + +def run_command(args, logger, test_name, msg_postfix): + ret = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = ret.communicate() + stdout = get_output_str(stdout); stderr = get_output_str(stderr) + logger.debug(f"In {test_name}, stdout:\n{stdout}\nstderr:\n{stderr}\n" + f"returncode = {ret.returncode}\n{msg_postfix}") + return ret, stdout, stderr \ No newline at end of file diff --git a/layerstack/tests/test_layerbase.py b/layerstack/tests/test_layerbase.py index 002a71f..94ce768 100644 --- a/layerstack/tests/test_layerbase.py +++ b/layerstack/tests/test_layerbase.py @@ -24,7 +24,7 @@ from subprocess import Popen, PIPE import sys -from layerstack.tests import layer_library_dir, get_output_str +from layerstack.tests import layer_library_dir, run_command logger = logging.getLogger(__name__) @@ -35,23 +35,16 @@ def test_layer_cli(): args = [sys.executable, str(layer_library_dir / 'test_list_args' / 'layer.py')] args += test_list - out_list = subprocess.Popen(args, stdout=PIPE, stderr=PIPE) - stdout, stderr = out_list.communicate() - - stderr = stderr.decode('ascii').rstrip() - logger.debug(f"In test_layer_cli, stdout:\n{stdout}\nstderr:\n{stderr}") + ret, stdout, stderr = run_command(args, logger, "test_layer_cli", "") assert stderr[-15:] == str(test_list), f"stdout:\n{stdout}\nstderr:\n{stderr}" - def test_kwarg_name_clashes(): args = [sys.executable, str(layer_library_dir / 'test_kwarg_name_clashes' / 'layer.py')] # run help - ret = subprocess.run(args + ["--help"], capture_output=True) - logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\n" - f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" - "after calling --help") - assert not ret.returncode, get_output_str(ret.stderr) + ret, stdout, stderr = run_command(args + ["--help"], logger, + "test_kwarg_name_clashes", "after calling --help") + assert not ret.returncode, stderr to_call = [ "-hr", str(0.2), @@ -61,22 +54,17 @@ def test_kwarg_name_clashes(): ] # run layer - ret = subprocess.run(args + to_call, capture_output=True) - logger.debug(f"In test_kwarg_name_clashes, stdout:\n{get_output_str(ret.stdout)}\n" - f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" - "after calling with kwargs") - assert not ret.returncode, get_output_str(ret.stderr) - + ret, stdout, stderr = run_command(args + to_call, logger, + "test_kwarg_name_clashes", "after calling with kwargs") + assert not ret.returncode, ret.stderr def test_kwargs_with_dashes(): args = [sys.executable, str(layer_library_dir / 'test_kwargs_with_dashes' / 'layer.py')] # run help - ret = subprocess.run(args + ["--help"], capture_output=True) - logger.debug(f"In test_kwargs_with_dashes, stdout:\n{get_output_str(ret.stdout)}\n" - f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" - "after calling --help") - assert not ret.returncode, get_output_str(ret.stderr) + ret, stdout, stderr = run_command(args + ["--help"], logger, + "test_kwargs_with_dashes", "after calling --help") + assert not ret.returncode, ret.stderr to_call = [ "-hr", str(0.2), @@ -87,8 +75,6 @@ def test_kwargs_with_dashes(): ] # run layer - ret = subprocess.run(args + to_call, capture_output=True) - logger.debug(f"In test_kwargs_with_dashes, stdout:\n{get_output_str(ret.stdout)}\n" - f"stderr:\n{get_output_str(ret.stderr)}\nreturncode = {ret.returncode}\n" - "after calling with kwargs") - assert not ret.returncode, get_output_str(ret.stderr) + ret, stdout, stderr = run_command(args + to_call, logger, + "test_kwargs_with_dashes", "after calling with kwargs") + assert not ret.returncode, ret.stderr From 37f6a03cec1848139278d7bd395e47e69b053069 Mon Sep 17 00:00:00 2001 From: Elaine Hale Date: Thu, 5 Aug 2021 13:59:32 -0600 Subject: [PATCH 9/9] Add some easy tests and get ready to release new version --- CHANGES.txt | 1 + LICENSE | 2 +- layerstack/tests/test_args_kwargs.py | 24 ++++++++++++++++++++---- layerstack/tests/test_layerbase.py | 2 -- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2d96804..e70cf7e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,4 @@ +v0.4.0, 08/05/21 -- Layer constructor accepts optional model argument; improved arg and kwarg name handling; additional bug fix, documentation, and testing improvements v0.3.1, 09/02/20 -- Ensure files required to run tests get installed v0.3.0, 09/02/20 -- Stack.load and CLI functions can look in alternate locations for Layers; ArgMode, Layer and Stack can be imported directly from layerstack; Stack.save more robust to different object types; testing and documentation improvements v0.2.0, 10/30/19 -- adding layerstack_stack CLI, ArgMode can be set with str, bug and documentation fixes diff --git a/LICENSE b/LICENSE index 4bc758a..418fa43 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2020 Alliance for Sustainable Energy, LLC, All Rights Reserved +Copyright (c) 2021 Alliance for Sustainable Energy, LLC, All Rights Reserved Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/layerstack/tests/test_args_kwargs.py b/layerstack/tests/test_args_kwargs.py index 6a68db6..7a0bfdf 100644 --- a/layerstack/tests/test_args_kwargs.py +++ b/layerstack/tests/test_args_kwargs.py @@ -19,9 +19,10 @@ [/LICENSE] ''' +from typing import KeysView import pytest -from layerstack.args import ArgMode, Arg, Kwarg, ArgList, KwargDict +from layerstack.args import ArgMode, Arg, Kwarg, ArgList, KwargDict, get_short_name def test_arglist_creation(): @@ -50,13 +51,28 @@ def test_kwarg_creation(): kwargs['max_pv_systems'] = Kwarg(parser=int,description='Maximum number of PV systems to install.') kwargs['fix_tap_changers'] = Kwarg(parser=bool,description='Whether to fix tap changer positions', action='store_true',default=False) - assert len(kwargs) == 2 + kwargs['change_rate'] = Kwarg(parser=float,description="Rate of change (fraction)", + default=0.01) + assert len(kwargs) == 3 assert kwargs['max_pv_systems'].name == 'max_pv_systems' kwargs = KwargDict([('max_pv',kwargs['max_pv_systems']), - ('fix_taps',kwargs['fix_tap_changers'])]) - assert len(kwargs) == 2 + ('fix_taps',kwargs['fix_tap_changers']), + ('change-rate', kwargs['change_rate'])]) + assert len(kwargs) == 3 assert kwargs['max_pv'].name == 'max_pv' + assert kwargs['max_pv'].get_name() == 'max_pv' + assert kwargs['change-rate'].name == 'change-rate' + assert kwargs['change-rate'].get_name() == 'change_rate' + assert kwargs['change-rate'].get_name(cleaned=False) == 'change-rate' with pytest.raises(Exception): KwargDict(kwargs['max_pv']) + + +def test_get_short_name(): + short_names = ['r', 'd', 'h'] + + assert get_short_name('dredge', short_names=short_names) == 'dr' + assert get_short_name('rude-awakening', short_names=short_names) == 'ra' + assert get_short_name('recharge_area', short_names=short_names) == 'rea' diff --git a/layerstack/tests/test_layerbase.py b/layerstack/tests/test_layerbase.py index 94ce768..6810907 100644 --- a/layerstack/tests/test_layerbase.py +++ b/layerstack/tests/test_layerbase.py @@ -20,8 +20,6 @@ ''' import logging from pathlib import Path -import subprocess -from subprocess import Popen, PIPE import sys from layerstack.tests import layer_library_dir, run_command