From d32ddbfa4808d89501f712cc715f34e4b10158e0 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Wed, 6 May 2020 13:57:46 +0100 Subject: [PATCH 1/9] added basic script which given a valid resource type checks for available resources locally and remotely --- validphys2/setup.py | 1 + validphys2/src/validphys/scripts/vp_list.py | 92 +++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 validphys2/src/validphys/scripts/vp_list.py diff --git a/validphys2/setup.py b/validphys2/setup.py index 88ff006d27..be84de879b 100644 --- a/validphys2/setup.py +++ b/validphys2/setup.py @@ -30,6 +30,7 @@ 'vp-checktheory = validphys.scripts.vp_checktheory:main', 'vp-rebuild-data = validphys.scripts.vp_rebuild_data:main', 'vp-pdfrename = validphys.scripts.vp_pdfrename:main', + 'vp-list = validphys.scripts.vp_list:main', ]}, package_dir = {'': 'src'}, packages = find_packages('src'), diff --git a/validphys2/src/validphys/scripts/vp_list.py b/validphys2/src/validphys/scripts/vp_list.py new file mode 100644 index 0000000000..9b23a59fb8 --- /dev/null +++ b/validphys2/src/validphys/scripts/vp_list.py @@ -0,0 +1,92 @@ +""" +vp-list + +Script which lists available resources locally and remotely + +""" +import argparse +import sys +import inspect +import re +import logging + +from reportengine import colors + +from validphys.loader import FallbackLoader as L +from validphys.config import ConfigError + +log = logging.getLogger() +log.setLevel(logging.INFO) +log.addHandler(colors.ColorHandler()) + + +def atoi(text): + """convert string to integer, if possible""" + return int(text) if text.isdigit() else text + + +def natural_keys(text): + """ + sort a list according to natural ordering + http://nedbatchelder.com/blog/200712/human_sorting.html + + taken directly from https://stackoverflow.com/a/5967539 + + """ + return [atoi(c) for c in re.split(r"(\d+)", text)] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "resource", + type=str, + help=( + "The type of resource to check availability for " + "(locally and remotely). Alternatively type `list` to see what " + "resources can be checked" + ), + ) + args = parser.parse_args() + l = L() + attrs = inspect.getmembers(l) + available = { + key.lstrip("available_"): val + for (key, val) in attrs + if key.startswith("available") + } + downloadable = { + key.lstrip("downloadable_"): val + for (key, val) in attrs + if key.startswith("downloadable") + } + if args.resource == "list": + log.info( + "You can check the availability (locally and remotely) of the following resources:\n- " + + "\n- ".join(list({*available.keys(), *downloadable.keys()})) + ) + sys.exit(0) + + if args.resource not in available and args.resource not in downloadable: + e = ConfigError( + f"Couldn't find that type of resource in available or downloadable resources.", + bad_item=args.resource, + alternatives={*available.keys(), *downloadable.keys()}, + ) + log.error(e) + print("Alternatively run `vp-list list` to see resources which can be checked.") + sys.exit(1) + if args.resource in available and available[args.resource]: + log.info( + f"The following {args.resource} are available locally:\n- " + + "\n- ".join(sorted(available[args.resource], key=natural_keys)) + ) + if args.resource in downloadable and downloadable[args.resource]: + log.info( + f"The following {args.resource} are downloadable:\n- " + + "\n- ".join(sorted(downloadable[args.resource], key=natural_keys)) + ) + + +if __name__ == "__main__": + main() From 2644d121360715e3935e6bb6318961aec228b6d4 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 8 May 2020 13:45:28 +0100 Subject: [PATCH 2/9] cleaned up script and added option to just check locally or remotely --- validphys2/src/validphys/scripts/vp_list.py | 108 ++++++++++++++------ 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/validphys2/src/validphys/scripts/vp_list.py b/validphys2/src/validphys/scripts/vp_list.py index 9b23a59fb8..30ad65e9fb 100644 --- a/validphys2/src/validphys/scripts/vp_list.py +++ b/validphys2/src/validphys/scripts/vp_list.py @@ -4,9 +4,9 @@ Script which lists available resources locally and remotely """ -import argparse import sys -import inspect +import argparse +from functools import partial import re import logging @@ -19,6 +19,29 @@ log.setLevel(logging.INFO) log.addHandler(colors.ColorHandler()) +REMOTE_TOKEN = "downloadable_" +LOCAL_TOKEN = "available_" + + +class ListAction(argparse.Action): + def __init__(self, *args, loader, **kwargs): + self.loader = loader + super().__init__(*args, **kwargs) + + def __call__(self, parser, *args, **kwargs): + attrs = dir(self.loader) + available = [ + attr.lstrip(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN) + ] + downloadable = [ + attr.lstrip(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) + ] + print( + "You can check the availability (locally and remotely) of the following resources:\n- " + + "\n- ".join(list({*available, *downloadable})) + ) + parser.exit() + def atoi(text): """convert string to integer, if possible""" @@ -36,6 +59,9 @@ def natural_keys(text): return [atoi(c) for c in re.split(r"(\d+)", text)] +sane_order = partial(sorted, key=natural_keys) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -43,29 +69,42 @@ def main(): type=str, help=( "The type of resource to check availability for " - "(locally and remotely). Alternatively type `list` to see what " - "resources can be checked" + "(locally and remotely). See --list for a list of resource types." ), ) + parser.add_argument( + "--list", + action=ListAction, + loader=L, + nargs=0, + help="List available resources and exit.", + ) + g = parser.add_mutually_exclusive_group() + g.add_argument( + "-r", + "--remote-only", + dest="remote", + action="store_true", + default=False, + help="Only list remote resources", + ) + g.add_argument( + "-l", + "--local-only", + dest="local", + action="store_true", + default=False, + help="Only list local resources", + ) args = parser.parse_args() - l = L() - attrs = inspect.getmembers(l) - available = { - key.lstrip("available_"): val - for (key, val) in attrs - if key.startswith("available") - } - downloadable = { - key.lstrip("downloadable_"): val - for (key, val) in attrs - if key.startswith("downloadable") - } - if args.resource == "list": - log.info( - "You can check the availability (locally and remotely) of the following resources:\n- " - + "\n- ".join(list({*available.keys(), *downloadable.keys()})) - ) - sys.exit(0) + attrs = dir(L) + + available = [ + attr.lstrip(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN) + ] + downloadable = [ + attr.lstrip(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) + ] if args.resource not in available and args.resource not in downloadable: e = ConfigError( @@ -76,16 +115,21 @@ def main(): log.error(e) print("Alternatively run `vp-list list` to see resources which can be checked.") sys.exit(1) - if args.resource in available and available[args.resource]: - log.info( - f"The following {args.resource} are available locally:\n- " - + "\n- ".join(sorted(available[args.resource], key=natural_keys)) - ) - if args.resource in downloadable and downloadable[args.resource]: - log.info( - f"The following {args.resource} are downloadable:\n- " - + "\n- ".join(sorted(downloadable[args.resource], key=natural_keys)) - ) + l = L() + if not args.remote: + local_res = getattr(l, LOCAL_TOKEN + args.resource, None) + if args.resource in available and local_res: + log.info("The following %s are available locally", args.resource) + print("- " + "\n- ".join(sane_order(local_res))) + else: + log.info("No %s are available locally.", args.resource) + if not args.local: + remote_res = getattr(l, REMOTE_TOKEN + args.resource, None) + if args.resource in downloadable and remote_res: + log.info("The following %s are downloadable:", args.resource) + print("- " + "\n- ".join(sane_order(remote_res))) + else: + log.info("No %s are available to download.", args.resource) if __name__ == "__main__": From cb7a0c31cbcbe2efa82c4401d5ebcc396742f013 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 8 May 2020 14:09:01 +0100 Subject: [PATCH 3/9] update how to check available fits to include using vp-list to check all resources - keep old docs as example for checking manually --- doc/sphinx/source/tutorials/list-fits.md | 39 ++++++++++++++++++++---- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/doc/sphinx/source/tutorials/list-fits.md b/doc/sphinx/source/tutorials/list-fits.md index 22583c6a1f..75151d6d1c 100644 --- a/doc/sphinx/source/tutorials/list-fits.md +++ b/doc/sphinx/source/tutorials/list-fits.md @@ -1,8 +1,35 @@ -# How to list the available fits +# How to list the available resources -The bulk of the available fits can be found by going to the -[fits folder](https://data.nnpdf.science/fits/) of the NNPDF data server. Some other fits may be -found in standalone folders in the [home folder](https://data.nnpdf.science/) of this server, but -of course finding a specific fit here may require some digging. For help in accessing the server, -please see [here](NNPDF-server). For information on how to download fits and other resources, +## Using `vp-list` + +In order to check what resources are available locally and for download, use +`vp-list` which will print out the names of resources. + +```bash +vp-list +``` + +The options for resource type can be seen with `vp-list --list`. + +```bash +$ vp-list --list +You can check the availability (locally and remotely) of the following resources: +- pdfs +- fits +- datasets +- theories +``` + +You can use the options `-l/--local-only` or `-r/--remote-only` to only check +for resources available locally or remotely respectively. + +## Manually checking server - example with fits + +You can also check manually on the storage servers for these resources. For example, +the bulk of the available fits can be found by going to the fits folder of the +NNPDF data server. Some other fits may be found in standalone folders in the home +folder of this server, but of course finding a specific fit here may require some +digging. For help in accessing the server, +please see [here](NNPDF-server). For information on how to download fits and +other resources, please see the [Downloading resources](download) section of the vp-guide. \ No newline at end of file From af2367ede10b472d713420574d6d72ff48889534 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Mon, 11 May 2020 11:18:39 +0100 Subject: [PATCH 4/9] address review points - rename doc and clean info strings for vp-list --- doc/sphinx/source/tutorials/{list-fits.md => list-resources.md} | 0 validphys2/src/validphys/scripts/vp_list.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename doc/sphinx/source/tutorials/{list-fits.md => list-resources.md} (100%) diff --git a/doc/sphinx/source/tutorials/list-fits.md b/doc/sphinx/source/tutorials/list-resources.md similarity index 100% rename from doc/sphinx/source/tutorials/list-fits.md rename to doc/sphinx/source/tutorials/list-resources.md diff --git a/validphys2/src/validphys/scripts/vp_list.py b/validphys2/src/validphys/scripts/vp_list.py index 30ad65e9fb..ca0fa9d747 100644 --- a/validphys2/src/validphys/scripts/vp_list.py +++ b/validphys2/src/validphys/scripts/vp_list.py @@ -119,7 +119,7 @@ def main(): if not args.remote: local_res = getattr(l, LOCAL_TOKEN + args.resource, None) if args.resource in available and local_res: - log.info("The following %s are available locally", args.resource) + log.info("The following %s are available locally:", args.resource) print("- " + "\n- ".join(sane_order(local_res))) else: log.info("No %s are available locally.", args.resource) From 757bfc9cb5dacf4cf7d32249d4a747471733f9c3 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Mon, 11 May 2020 11:30:17 +0100 Subject: [PATCH 5/9] ensure renamed docs is in index --- doc/sphinx/source/tutorials/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/tutorials/index.rst b/doc/sphinx/source/tutorials/index.rst index 4dbd56f193..6e0e7b6079 100644 --- a/doc/sphinx/source/tutorials/index.rst +++ b/doc/sphinx/source/tutorials/index.rst @@ -6,7 +6,7 @@ Tutorials ./runafit.md ./compare-fits.md - ./list-fits.md + ./list-resources.md ./report.md ./buildmaster.md ./APPLgrids.md From 5e59908d1ad117c984a4eab0022757d5fd64ed6c Mon Sep 17 00:00:00 2001 From: wilsonm Date: Mon, 11 May 2020 12:14:35 +0100 Subject: [PATCH 6/9] uses argparse choices and print available options in help --- validphys2/src/validphys/scripts/vp_list.py | 64 ++++++--------------- 1 file changed, 16 insertions(+), 48 deletions(-) diff --git a/validphys2/src/validphys/scripts/vp_list.py b/validphys2/src/validphys/scripts/vp_list.py index ca0fa9d747..ac770bf9dd 100644 --- a/validphys2/src/validphys/scripts/vp_list.py +++ b/validphys2/src/validphys/scripts/vp_list.py @@ -4,7 +4,6 @@ Script which lists available resources locally and remotely """ -import sys import argparse from functools import partial import re @@ -13,7 +12,6 @@ from reportengine import colors from validphys.loader import FallbackLoader as L -from validphys.config import ConfigError log = logging.getLogger() log.setLevel(logging.INFO) @@ -23,26 +21,6 @@ LOCAL_TOKEN = "available_" -class ListAction(argparse.Action): - def __init__(self, *args, loader, **kwargs): - self.loader = loader - super().__init__(*args, **kwargs) - - def __call__(self, parser, *args, **kwargs): - attrs = dir(self.loader) - available = [ - attr.lstrip(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN) - ] - downloadable = [ - attr.lstrip(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) - ] - print( - "You can check the availability (locally and remotely) of the following resources:\n- " - + "\n- ".join(list({*available, *downloadable})) - ) - parser.exit() - - def atoi(text): """convert string to integer, if possible""" return int(text) if text.isdigit() else text @@ -64,20 +42,27 @@ def natural_keys(text): def main(): parser = argparse.ArgumentParser(description=__doc__) + + attrs = dir(L) + + available = [ + attr.lstrip(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN) + ] + downloadable = [ + attr.lstrip(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) + ] + # set metavar and print choices in help string - otherwise looks ugly. parser.add_argument( "resource", type=str, + choices={*available, *downloadable}, help=( - "The type of resource to check availability for " - "(locally and remotely). See --list for a list of resource types." + "The type of resource to check availability for (locally and/or remotely). " + + "Choose from: " + + ", ".join(list({*available, *downloadable})) + + "." ), - ) - parser.add_argument( - "--list", - action=ListAction, - loader=L, - nargs=0, - help="List available resources and exit.", + metavar="resource", ) g = parser.add_mutually_exclusive_group() g.add_argument( @@ -97,24 +82,7 @@ def main(): help="Only list local resources", ) args = parser.parse_args() - attrs = dir(L) - - available = [ - attr.lstrip(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN) - ] - downloadable = [ - attr.lstrip(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) - ] - if args.resource not in available and args.resource not in downloadable: - e = ConfigError( - f"Couldn't find that type of resource in available or downloadable resources.", - bad_item=args.resource, - alternatives={*available.keys(), *downloadable.keys()}, - ) - log.error(e) - print("Alternatively run `vp-list list` to see resources which can be checked.") - sys.exit(1) l = L() if not args.remote: local_res = getattr(l, LOCAL_TOKEN + args.resource, None) From 2b27365cfaae4b71d8a7b2a15b3bcf275adcc1f8 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Mon, 11 May 2020 12:19:06 +0100 Subject: [PATCH 7/9] update docs to reflect change to help --- doc/sphinx/source/tutorials/list-resources.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/sphinx/source/tutorials/list-resources.md b/doc/sphinx/source/tutorials/list-resources.md index 75151d6d1c..dc96cf0648 100644 --- a/doc/sphinx/source/tutorials/list-resources.md +++ b/doc/sphinx/source/tutorials/list-resources.md @@ -9,15 +9,19 @@ In order to check what resources are available locally and for download, use vp-list ``` -The options for resource type can be seen with `vp-list --list`. +The options for resource type can be seen with `vp-list --help`. ```bash -$ vp-list --list -You can check the availability (locally and remotely) of the following resources: -- pdfs -- fits -- datasets -- theories +$ vp-list --help +usage: vp-list [-h] [-r | -l] resource + +vp-list Script which lists available resources locally and remotely + +positional arguments: + resource The type of resource to check availability for (locally + and/or remotely). Choose from: theories, fits, pdfs, + datasets. + ``` You can use the options `-l/--local-only` or `-r/--remote-only` to only check From e3e664709735f0919c47489b74c9cff1226296bb Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 15 May 2020 16:11:15 +0100 Subject: [PATCH 8/9] added basic testing suite for vp-list --- validphys2/src/validphys/scripts/vp_list.py | 4 +- .../src/validphys/tests/test_vplistscript.py | 64 +++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 validphys2/src/validphys/tests/test_vplistscript.py diff --git a/validphys2/src/validphys/scripts/vp_list.py b/validphys2/src/validphys/scripts/vp_list.py index ac770bf9dd..b10f170ed5 100644 --- a/validphys2/src/validphys/scripts/vp_list.py +++ b/validphys2/src/validphys/scripts/vp_list.py @@ -40,7 +40,7 @@ def natural_keys(text): sane_order = partial(sorted, key=natural_keys) -def main(): +def main(command_line=None): parser = argparse.ArgumentParser(description=__doc__) attrs = dir(L) @@ -81,7 +81,7 @@ def main(): default=False, help="Only list local resources", ) - args = parser.parse_args() + args = parser.parse_args(command_line) l = L() if not args.remote: diff --git a/validphys2/src/validphys/tests/test_vplistscript.py b/validphys2/src/validphys/tests/test_vplistscript.py new file mode 100644 index 0000000000..81e70080fd --- /dev/null +++ b/validphys2/src/validphys/tests/test_vplistscript.py @@ -0,0 +1,64 @@ +""" +test_vplistscript.py + +Module for testing vp-list. The output of which is dynamic and so we just check +that the script runs and gives some output +""" +from contextlib import redirect_stdout +import io + +from validphys.scripts.vp_list import main + +def test_listfits(): + """Checks listing fits returns output""" + f = io.StringIO() + cmd = ["fits"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_listpdfs(): + """Checks listing pdfs returns output""" + f = io.StringIO() + cmd = ["pdfs"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_listtheories(): + """Checks listing theories returns output""" + f = io.StringIO() + cmd = ["theories"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_listdatasets(): + """Checks listing datasets returns output""" + f = io.StringIO() + cmd = ["pdfs"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_local(): + """Check local flag""" + f = io.StringIO() + cmd = ["datasets", "-l"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_remote(): + """Test remote flag on both datasets (which should return empty string) and pdfs + which returns output + + """ + f = io.StringIO() + cmd_data = ["datasets", "-r"] + cmd_pdfs = ["pdfs", "-r"] + with redirect_stdout(f): + main(cmd_data) + assert not f.getvalue() + main(cmd_pdfs) + assert f.getvalue() From 8e98fc969360fa1058f417a0c689e974dc1ea6c6 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Mon, 18 May 2020 11:28:35 +0100 Subject: [PATCH 9/9] fix vp-list test which lists datasets to actually list datasets --- validphys2/src/validphys/tests/test_vplistscript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/tests/test_vplistscript.py b/validphys2/src/validphys/tests/test_vplistscript.py index 81e70080fd..2fe2be3802 100644 --- a/validphys2/src/validphys/tests/test_vplistscript.py +++ b/validphys2/src/validphys/tests/test_vplistscript.py @@ -36,7 +36,7 @@ def test_listtheories(): def test_listdatasets(): """Checks listing datasets returns output""" f = io.StringIO() - cmd = ["pdfs"] + cmd = ["datasets"] with redirect_stdout(f): main(cmd) assert f.getvalue()