diff --git a/doc/sphinx/source/tutorials/index.rst b/doc/sphinx/source/tutorials/index.rst index 4dbd56f193..6e0e7b6079 100644 --- a/doc/sphinx/source/tutorials/index.rst +++ b/doc/sphinx/source/tutorials/index.rst @@ -6,7 +6,7 @@ Tutorials ./runafit.md ./compare-fits.md - ./list-fits.md + ./list-resources.md ./report.md ./buildmaster.md ./APPLgrids.md diff --git a/doc/sphinx/source/tutorials/list-fits.md b/doc/sphinx/source/tutorials/list-fits.md deleted file mode 100644 index 22583c6a1f..0000000000 --- a/doc/sphinx/source/tutorials/list-fits.md +++ /dev/null @@ -1,8 +0,0 @@ -# How to list the available fits - -The bulk of the available fits can be found by going to the -[fits folder](https://data.nnpdf.science/fits/) of the NNPDF data server. Some other fits may be -found in standalone folders in the [home folder](https://data.nnpdf.science/) of this server, but -of course finding a specific fit here may require some digging. For help in accessing the server, -please see [here](NNPDF-server). For information on how to download fits and other resources, -please see the [Downloading resources](download) section of the vp-guide. \ No newline at end of file diff --git a/doc/sphinx/source/tutorials/list-resources.md b/doc/sphinx/source/tutorials/list-resources.md new file mode 100644 index 0000000000..dc96cf0648 --- /dev/null +++ b/doc/sphinx/source/tutorials/list-resources.md @@ -0,0 +1,39 @@ +# How to list the available resources + +## Using `vp-list` + +In order to check what resources are available locally and for download, use +`vp-list` which will print out the names of resources. + +```bash +vp-list +``` + +The options for resource type can be seen with `vp-list --help`. + +```bash +$ vp-list --help +usage: vp-list [-h] [-r | -l] resource + +vp-list Script which lists available resources locally and remotely + +positional arguments: + resource The type of resource to check availability for (locally + and/or remotely). Choose from: theories, fits, pdfs, + datasets. + +``` + +You can use the options `-l/--local-only` or `-r/--remote-only` to only check +for resources available locally or remotely respectively. + +## Manually checking server - example with fits + +You can also check manually on the storage servers for these resources. For example, +the bulk of the available fits can be found by going to the fits folder of the +NNPDF data server. Some other fits may be found in standalone folders in the home +folder of this server, but of course finding a specific fit here may require some +digging. For help in accessing the server, +please see [here](NNPDF-server). For information on how to download fits and +other resources, +please see the [Downloading resources](download) section of the vp-guide. \ No newline at end of file diff --git a/validphys2/setup.py b/validphys2/setup.py index 88ff006d27..be84de879b 100644 --- a/validphys2/setup.py +++ b/validphys2/setup.py @@ -30,6 +30,7 @@ 'vp-checktheory = validphys.scripts.vp_checktheory:main', 'vp-rebuild-data = validphys.scripts.vp_rebuild_data:main', 'vp-pdfrename = validphys.scripts.vp_pdfrename:main', + 'vp-list = validphys.scripts.vp_list:main', ]}, package_dir = {'': 'src'}, packages = find_packages('src'), diff --git a/validphys2/src/validphys/scripts/vp_list.py b/validphys2/src/validphys/scripts/vp_list.py new file mode 100644 index 0000000000..b10f170ed5 --- /dev/null +++ b/validphys2/src/validphys/scripts/vp_list.py @@ -0,0 +1,104 @@ +""" +vp-list + +Script which lists available resources locally and remotely + +""" +import argparse +from functools import partial +import re +import logging + +from reportengine import colors + +from validphys.loader import FallbackLoader as L + +log = logging.getLogger() +log.setLevel(logging.INFO) +log.addHandler(colors.ColorHandler()) + +REMOTE_TOKEN = "downloadable_" +LOCAL_TOKEN = "available_" + + +def atoi(text): + """convert string to integer, if possible""" + return int(text) if text.isdigit() else text + + +def natural_keys(text): + """ + sort a list according to natural ordering + http://nedbatchelder.com/blog/200712/human_sorting.html + + taken directly from https://stackoverflow.com/a/5967539 + + """ + return [atoi(c) for c in re.split(r"(\d+)", text)] + + +sane_order = partial(sorted, key=natural_keys) + + +def main(command_line=None): + parser = argparse.ArgumentParser(description=__doc__) + + attrs = dir(L) + + available = [ + attr.lstrip(LOCAL_TOKEN) for attr in attrs if attr.startswith(LOCAL_TOKEN) + ] + downloadable = [ + attr.lstrip(REMOTE_TOKEN) for attr in attrs if attr.startswith(REMOTE_TOKEN) + ] + # set metavar and print choices in help string - otherwise looks ugly. + parser.add_argument( + "resource", + type=str, + choices={*available, *downloadable}, + help=( + "The type of resource to check availability for (locally and/or remotely). " + + "Choose from: " + + ", ".join(list({*available, *downloadable})) + + "." + ), + metavar="resource", + ) + g = parser.add_mutually_exclusive_group() + g.add_argument( + "-r", + "--remote-only", + dest="remote", + action="store_true", + default=False, + help="Only list remote resources", + ) + g.add_argument( + "-l", + "--local-only", + dest="local", + action="store_true", + default=False, + help="Only list local resources", + ) + args = parser.parse_args(command_line) + + l = L() + if not args.remote: + local_res = getattr(l, LOCAL_TOKEN + args.resource, None) + if args.resource in available and local_res: + log.info("The following %s are available locally:", args.resource) + print("- " + "\n- ".join(sane_order(local_res))) + else: + log.info("No %s are available locally.", args.resource) + if not args.local: + remote_res = getattr(l, REMOTE_TOKEN + args.resource, None) + if args.resource in downloadable and remote_res: + log.info("The following %s are downloadable:", args.resource) + print("- " + "\n- ".join(sane_order(remote_res))) + else: + log.info("No %s are available to download.", args.resource) + + +if __name__ == "__main__": + main() diff --git a/validphys2/src/validphys/tests/test_vplistscript.py b/validphys2/src/validphys/tests/test_vplistscript.py new file mode 100644 index 0000000000..2fe2be3802 --- /dev/null +++ b/validphys2/src/validphys/tests/test_vplistscript.py @@ -0,0 +1,64 @@ +""" +test_vplistscript.py + +Module for testing vp-list. The output of which is dynamic and so we just check +that the script runs and gives some output +""" +from contextlib import redirect_stdout +import io + +from validphys.scripts.vp_list import main + +def test_listfits(): + """Checks listing fits returns output""" + f = io.StringIO() + cmd = ["fits"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_listpdfs(): + """Checks listing pdfs returns output""" + f = io.StringIO() + cmd = ["pdfs"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_listtheories(): + """Checks listing theories returns output""" + f = io.StringIO() + cmd = ["theories"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_listdatasets(): + """Checks listing datasets returns output""" + f = io.StringIO() + cmd = ["datasets"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_local(): + """Check local flag""" + f = io.StringIO() + cmd = ["datasets", "-l"] + with redirect_stdout(f): + main(cmd) + assert f.getvalue() + +def test_remote(): + """Test remote flag on both datasets (which should return empty string) and pdfs + which returns output + + """ + f = io.StringIO() + cmd_data = ["datasets", "-r"] + cmd_pdfs = ["pdfs", "-r"] + with redirect_stdout(f): + main(cmd_data) + assert not f.getvalue() + main(cmd_pdfs) + assert f.getvalue()