From 673ec6b2d118ea7cc4448a37f4f82f2e1cfcf119 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:40 -0400 Subject: [PATCH 01/21] Require `asciitree`. This dependency is able to build some really nice looking trees in ASCII using Python dictionaries. Thus it would be really useful for giving hierarchical listings of the contents of groups inside Zarr. It is for this reason that we add it here. --- requirements.txt | 1 + requirements_dev.txt | 1 + requirements_rtfd.txt | 1 + setup.py | 1 + 4 files changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index 8427764e04..2aaf58dff2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +asciitree numpy fasteners numcodecs diff --git a/requirements_dev.txt b/requirements_dev.txt index d273262177..f54c565d2e 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,6 @@ appdirs==1.4.3 args==0.1.0 +asciitree==0.3.3 certifi==2017.7.27.1 chardet==3.0.4 clint==0.5.1 diff --git a/requirements_rtfd.txt b/requirements_rtfd.txt index 86091ed956..55730713f7 100644 --- a/requirements_rtfd.txt +++ b/requirements_rtfd.txt @@ -1,3 +1,4 @@ +asciitree setuptools setuptools_scm sphinx diff --git a/setup.py b/setup.py index 11523fd229..c420174a75 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ 'setuptools-scm>1.5.4' ], install_requires=[ + 'asciitree', 'numpy>=1.7', 'fasteners', 'numcodecs>=0.2.0', From 44fdd5212ee04142b654bc4acb252de34471e246 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:41 -0400 Subject: [PATCH 02/21] Add the `tree` method to `Group`. Provides a method for `Group` that provides a pretty-printed listing of everything below it. This designed to be similar to the output of the Unix `tree` command. --- docs/api/hierarchy.rst | 1 + zarr/hierarchy.py | 54 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/api/hierarchy.rst b/docs/api/hierarchy.rst index 799657c8d0..5e0a22507c 100644 --- a/docs/api/hierarchy.rst +++ b/docs/api/hierarchy.rst @@ -19,6 +19,7 @@ Groups (``zarr.hierarchy``) .. automethod:: visitkeys .. automethod:: visitvalues .. automethod:: visititems + .. automethod:: tree .. automethod:: create_group .. automethod:: require_group .. automethod:: create_groups diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 937c53d12c..2b47277e38 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division -from collections import MutableMapping +from collections import OrderedDict, MutableMapping from itertools import islice +from asciitree import BoxStyle, LeftAligned +from asciitree.drawing import BOX_LIGHT import numpy as np @@ -60,6 +62,7 @@ class Group(MutableMapping): visitkeys visitvalues visititems + tree create_group require_group create_groups @@ -523,6 +526,55 @@ def visititems(self, func): base_len = len(self.name) return self.visitvalues(lambda o: func(o.name[base_len:].lstrip("/"), o)) + def tree(self): + """Provide a ``print`-able display of the hierarchy. + + Examples + -------- + >>> import zarr + >>> g1 = zarr.group() + >>> g2 = g1.create_group('foo') + >>> g3 = g1.create_group('bar') + >>> g4 = g3.create_group('baz') + >>> g5 = g3.create_group('quux') + >>> d1 = g5.create_dataset('baz', shape=100, chunks=10) + >>> print(g1.tree()) + / + ├── bar + │ ├── baz + │ └── quux + │ └── baz[...] + └── foo + >>> print(g3.tree()) + bar + ├── baz + └── quux + └── baz[...] + """ + + def gen_tree(g): + r = OrderedDict() + n = self.name.strip("/") + n = n if n else "/" + d = r.setdefault(n, OrderedDict()) + + def _gen_branch(p, o): + sd = d + n = p.strip("/") + g = n.split("/") + g[-1] += "[...]" if isinstance(o, Array) else "" + for e in g: + sd = sd.setdefault(e, OrderedDict()) + + g.visititems(_gen_branch) + + return r + + box_sty = BoxStyle(gfx=BOX_LIGHT, horiz_len=2, label_space=1) + box_tr = LeftAligned(draw=box_sty) + + return box_tr(gen_tree(self)) + def _write_op(self, f, *args, **kwargs): # guard condition From b242d53a3ce602cccb7ab15e30ed59577b71d073 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:42 -0400 Subject: [PATCH 03/21] Test the `tree` method of `Group`. Add some unit tests in addition to the doctests to verify that `tree` provides the expected output. --- zarr/tests/test_hierarchy.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index ae7adf380b..e3ac703310 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -4,6 +4,7 @@ import tempfile import atexit import shutil +import textwrap import os import pickle @@ -608,6 +609,36 @@ def visitor1(val, *args): eq(True, g1.visitvalues(visitor1)) eq(True, g1.visititems(visitor1)) + def test_tree(self): + # setup + g1 = self.create_group() + g2 = g1.create_group('foo') + g3 = g1.create_group('bar') + g3.create_group('baz') + g5 = g3.create_group('quux') + g5.create_dataset('baz', shape=100, chunks=10) + + # test + sg1 = textwrap.dedent(u"""\ + / + ├── bar + │ ├── baz + │ └── quux + │ └── baz[...] + └── foo""") + eq(sg1, g1.tree()) + + sg2 = textwrap.dedent(u"""\ + foo""") + eq(sg2, g2.tree()) + + sg3 = textwrap.dedent(u"""\ + bar + ├── baz + └── quux + └── baz[...]""") + eq(sg3, g3.tree()) + def test_empty_getitem_contains_iterators(self): # setup g = self.create_group() From 830a52db2ec08342628d80d7153e97176b541f11 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:43 -0400 Subject: [PATCH 04/21] Add TreeHierarchy object Provides an easy to use representation of a dictionary. --- zarr/util.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/zarr/util.py b/zarr/util.py index 51b65d06d2..18b077f329 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -4,6 +4,8 @@ from textwrap import TextWrapper import numbers +from asciitree import BoxStyle, LeftAligned +from asciitree.drawing import BOX_LIGHT import numpy as np @@ -365,3 +367,22 @@ def __repr__(self): def _repr_html_(self): items = self.obj.info_items() return info_html_report(items) + + +class TreeHierarchy(object): + + def __init__(self, hier, ascii_kwargs={}): + self.hier = hier + + self.ascii_kwargs = dict( + gfx=BOX_LIGHT, horiz_len=2, label_space=1, indent=1 + ) + self.update_ascii_kwargs(ascii_kwargs) + + def update_ascii_kwargs(self, ascii_kwargs={}): + self.ascii_kwargs.update(ascii_kwargs) + self.ascii_draw = LeftAligned(draw=BoxStyle(**self.ascii_kwargs)) + return self + + def __repr__(self): + return self.ascii_draw(self.hier) From f9e8435a0bbba4d74c5498485fde888ae976520b Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:44 -0400 Subject: [PATCH 05/21] Return a TreeHierarchy from tree Instead of providing a specific configuration of the group representation, return a customizable object that can be used immediately for representation or can be configured after the fact. --- zarr/hierarchy.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 2b47277e38..728a157c4a 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -3,9 +3,6 @@ from collections import OrderedDict, MutableMapping from itertools import islice -from asciitree import BoxStyle, LeftAligned -from asciitree.drawing import BOX_LIGHT - import numpy as np @@ -15,7 +12,7 @@ DictStore, DirectoryStore, group_meta_key, attrs_key, listdir, rmdir from zarr.creation import array, create, empty, zeros, ones, full, \ empty_like, zeros_like, ones_like, full_like -from zarr.util import normalize_storage_path, normalize_shape, InfoReporter +from zarr.util import normalize_storage_path, normalize_shape, InfoReporter, TreeHierarchy from zarr.errors import err_contains_array, err_contains_group, err_group_not_found, err_read_only from zarr.meta import decode_group_metadata @@ -570,10 +567,7 @@ def _gen_branch(p, o): return r - box_sty = BoxStyle(gfx=BOX_LIGHT, horiz_len=2, label_space=1) - box_tr = LeftAligned(draw=box_sty) - - return box_tr(gen_tree(self)) + return TreeHierarchy(gen_tree(self)) def _write_op(self, f, *args, **kwargs): From 1365a806fb6befecc0a97d0481d0926d671badef Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:44 -0400 Subject: [PATCH 06/21] Refresh tree tests to use repr As we now return a `TreeHierarchy` object instead of a `str`, we have to take the `repr` of this object to get a `str` that we can compare in the old `tree` tests. Hence we tweak the tests in this way to get them passing again. --- zarr/tests/test_hierarchy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index e3ac703310..49767bc547 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -626,18 +626,18 @@ def test_tree(self): │ └── quux │ └── baz[...] └── foo""") - eq(sg1, g1.tree()) + eq(sg1, repr(g1.tree())) sg2 = textwrap.dedent(u"""\ foo""") - eq(sg2, g2.tree()) + eq(sg2, repr(g2.tree())) sg3 = textwrap.dedent(u"""\ bar ├── baz └── quux └── baz[...]""") - eq(sg3, g3.tree()) + eq(sg3, repr(g3.tree())) def test_empty_getitem_contains_iterators(self): # setup From 1975711f6efb3ae82b655a6daf7affe7b8d0c551 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:45 -0400 Subject: [PATCH 07/21] Pick unicode/ascii agnostic tree characters These are needed to workaround the constraint that Python 2 can only place `bytes` in `__repr__` results and Python 3 can only place `unicode` in `__repr__` results. As such, we need to chose characters that will work well in both ASCII and UTF-8. Hence we change the symbols that `asciitree` uses to be a set that match this criteria. Unfortunately they cannot be imported from `asciitree` directly as they force all of their characters to UTF-8. --- zarr/hierarchy.py | 16 ++++++++-------- zarr/tests/test_hierarchy.py | 16 ++++++++-------- zarr/util.py | 11 +++++++++-- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 728a157c4a..aaf4d95963 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -537,16 +537,16 @@ def tree(self): >>> d1 = g5.create_dataset('baz', shape=100, chunks=10) >>> print(g1.tree()) / - ├── bar - │ ├── baz - │ └── quux - │ └── baz[...] - └── foo + +-- bar + | +-- baz + | +-- quux + | +-- baz[...] + +-- foo >>> print(g3.tree()) bar - ├── baz - └── quux - └── baz[...] + +-- baz + +-- quux + +-- baz[...] """ def gen_tree(g): diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 49767bc547..71d767820f 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -621,11 +621,11 @@ def test_tree(self): # test sg1 = textwrap.dedent(u"""\ / - ├── bar - │ ├── baz - │ └── quux - │ └── baz[...] - └── foo""") + +-- bar + | +-- baz + | +-- quux + | +-- baz[...] + +-- foo""") eq(sg1, repr(g1.tree())) sg2 = textwrap.dedent(u"""\ @@ -634,9 +634,9 @@ def test_tree(self): sg3 = textwrap.dedent(u"""\ bar - ├── baz - └── quux - └── baz[...]""") + +-- baz + +-- quux + +-- baz[...]""") eq(sg3, repr(g3.tree())) def test_empty_getitem_contains_iterators(self): diff --git a/zarr/util.py b/zarr/util.py index 18b077f329..e3d203f4a4 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -5,7 +5,6 @@ import numbers from asciitree import BoxStyle, LeftAligned -from asciitree.drawing import BOX_LIGHT import numpy as np @@ -375,7 +374,15 @@ def __init__(self, hier, ascii_kwargs={}): self.hier = hier self.ascii_kwargs = dict( - gfx=BOX_LIGHT, horiz_len=2, label_space=1, indent=1 + gfx=dict( + UP_AND_RIGHT="+", + HORIZONTAL="-", + VERTICAL="|", + VERTICAL_AND_RIGHT="+" + ), + horiz_len=2, + label_space=1, + indent=1 ) self.update_ascii_kwargs(ascii_kwargs) From f709e9ec651659212dd94360cee89ee068a7c0ad Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:46 -0400 Subject: [PATCH 08/21] Use custom traverser with tree Instead of trying to incorporate all the traversal logic into the `tree` method in Zarr's `Group` class, make a custom traverser that simply acts on the Zarr objects to construct the tree. This simplifies the code and makes it more flexible. --- zarr/hierarchy.py | 22 ++-------------------- zarr/util.py | 26 ++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index aaf4d95963..1c0a89dc07 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division -from collections import OrderedDict, MutableMapping +from collections import MutableMapping from itertools import islice import numpy as np @@ -549,25 +549,7 @@ def tree(self): +-- baz[...] """ - def gen_tree(g): - r = OrderedDict() - n = self.name.strip("/") - n = n if n else "/" - d = r.setdefault(n, OrderedDict()) - - def _gen_branch(p, o): - sd = d - n = p.strip("/") - g = n.split("/") - g[-1] += "[...]" if isinstance(o, Array) else "" - for e in g: - sd = sd.setdefault(e, OrderedDict()) - - g.visititems(_gen_branch) - - return r - - return TreeHierarchy(gen_tree(self)) + return TreeHierarchy(self) def _write_op(self, f, *args, **kwargs): diff --git a/zarr/util.py b/zarr/util.py index e3d203f4a4..e01937d5f9 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -5,6 +5,7 @@ import numbers from asciitree import BoxStyle, LeftAligned +from asciitree.traversal import Traversal import numpy as np @@ -368,10 +369,24 @@ def _repr_html_(self): return info_html_report(items) +class ZarrGroupTraversal(Traversal): + + def get_children(self, node): + return getattr(node, "values", lambda: [])() + + def get_root(self, tree): + return tree + + def get_text(self, node): + name = node.name.split("/")[-1] or "/" + name += "[...]" if hasattr(node, "dtype") else "" + return name + + class TreeHierarchy(object): - def __init__(self, hier, ascii_kwargs={}): - self.hier = hier + def __init__(self, group, ascii_kwargs={}): + self.group = group self.ascii_kwargs = dict( gfx=dict( @@ -388,8 +403,11 @@ def __init__(self, hier, ascii_kwargs={}): def update_ascii_kwargs(self, ascii_kwargs={}): self.ascii_kwargs.update(ascii_kwargs) - self.ascii_draw = LeftAligned(draw=BoxStyle(**self.ascii_kwargs)) + self.ascii_draw = LeftAligned( + traverse=ZarrGroupTraversal(), + draw=BoxStyle(**self.ascii_kwargs) + ) return self def __repr__(self): - return self.ascii_draw(self.hier) + return self.ascii_draw(self.group) From 0037f2f8af210a956a19623412fa2f62ff3eed03 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:47 -0400 Subject: [PATCH 09/21] Drop `ascii_kwargs` from `TreeHierarchy` constructor We don't currently provide a way to set `ascii_kwargs` from `tree`. So maybe we don't need the argument in `TreeHierarchy`'s constructor. Besides it is trivial to set the `ascii_kwargs` after the fact. --- zarr/util.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zarr/util.py b/zarr/util.py index e01937d5f9..dbe67225bd 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -385,10 +385,11 @@ def get_text(self, node): class TreeHierarchy(object): - def __init__(self, group, ascii_kwargs={}): + def __init__(self, group): self.group = group + self.ascii_kwargs = dict() - self.ascii_kwargs = dict( + self.update_ascii_kwargs(dict( gfx=dict( UP_AND_RIGHT="+", HORIZONTAL="-", @@ -398,8 +399,7 @@ def __init__(self, group, ascii_kwargs={}): horiz_len=2, label_space=1, indent=1 - ) - self.update_ascii_kwargs(ascii_kwargs) + )) def update_ascii_kwargs(self, ascii_kwargs={}): self.ascii_kwargs.update(ascii_kwargs) From 67e24077a656965ebd2516f373c7dfe4183938ca Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:48 -0400 Subject: [PATCH 10/21] Collect arbitrary kwargs in `update_ascii_kwargs` Should improve usability and reduce the amount of typing required to tweak the structure. --- zarr/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zarr/util.py b/zarr/util.py index dbe67225bd..0cbdeb5d53 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -389,7 +389,7 @@ def __init__(self, group): self.group = group self.ascii_kwargs = dict() - self.update_ascii_kwargs(dict( + self.update_ascii_kwargs( gfx=dict( UP_AND_RIGHT="+", HORIZONTAL="-", @@ -399,9 +399,9 @@ def __init__(self, group): horiz_len=2, label_space=1, indent=1 - )) + ) - def update_ascii_kwargs(self, ascii_kwargs={}): + def update_ascii_kwargs(self, **ascii_kwargs): self.ascii_kwargs.update(ascii_kwargs) self.ascii_draw = LeftAligned( traverse=ZarrGroupTraversal(), From 58486be8e3723a6e6d689296e8c1f187bfcd91b7 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:49 -0400 Subject: [PATCH 11/21] Add an HTML representation of TreeHierarchy Provides an HTML representation of TreeHierarchy that can be used for representation of Zarr's `tree` method in the notebook. --- zarr/util.py | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/zarr/util.py b/zarr/util.py index 0cbdeb5d53..4ff63fccfe 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division import operator -from textwrap import TextWrapper +from textwrap import TextWrapper, dedent import numbers from asciitree import BoxStyle, LeftAligned @@ -383,6 +383,104 @@ def get_text(self, node): return name +def custom_html_sublist(group, indent): + traverser = ZarrGroupTraversal(tree=group) + result = "" + + result += ( + """{0}
  • {1}
    """.format( + indent, traverser.get_text(group) + ) + ) + + children = traverser.get_children(group) + if children: + result += """\n{0}{0}
      \n""".format(indent) + for c in children: + for l in custom_html_sublist(c, indent).splitlines(): + result += "{0}{0}{1}\n".format(indent, l) + if children: + result += "{0}{0}
    \n{0}".format(indent) + + result += ( + """
    • \n""".format( + indent, traverser.get_text(group) + ) + ) + + return result + + +def custom_html_list(group, indent=" "): + result = "" + + # Add custom CSS style for our HTML list + result += """\n\n" + + # Insert the HTML list + result += """
    \n""" + result += "
      \n" + result += custom_html_sublist(group, indent=indent) + result += "
    \n" + result += "
    \n" + + return result + + class TreeHierarchy(object): def __init__(self, group): @@ -411,3 +509,6 @@ def update_ascii_kwargs(self, **ascii_kwargs): def __repr__(self): return self.ascii_draw(self.group) + + def _repr_html_(self): + return custom_html_list(self.group) From 0f67052066942a252dcbec99798e5221ec7e07e5 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:50 -0400 Subject: [PATCH 12/21] Test the HTML representation --- zarr/tests/test_hierarchy.py | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 71d767820f..bd0706eba9 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -627,10 +627,38 @@ def test_tree(self): | +-- baz[...] +-- foo""") eq(sg1, repr(g1.tree())) + hg1 = textwrap.dedent(u"""\ +
    +
      +
    • /
      +
        +
      • bar
        +
          +
        • baz
          • +
        • quux
          +
            +
          • baz[...]
            • +
          +
          • +
        +
        • +
      • foo
        • +
      +
      • +
    +
    """) + eq(hg1, g1.tree()._repr_html_().split("")[1].strip()) sg2 = textwrap.dedent(u"""\ foo""") eq(sg2, repr(g2.tree())) + hg2 = textwrap.dedent(u"""\ +
    +
      +
    • foo
      • +
    +
    """) + eq(hg2, g2.tree()._repr_html_().split("")[1].strip()) sg3 = textwrap.dedent(u"""\ bar @@ -638,6 +666,22 @@ def test_tree(self): +-- quux +-- baz[...]""") eq(sg3, repr(g3.tree())) + hg3 = textwrap.dedent(u"""\ +
    +
      +
    • bar
      +
        +
      • baz
        • +
      • quux
        +
          +
        • baz[...]
          • +
        +
        • +
      +
      • +
    +
    """) + eq(hg3, g3.tree()._repr_html_().split("")[1].strip()) def test_empty_getitem_contains_iterators(self): # setup From f9170cd5c803a59b3807621363dfc9615c41e3d1 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:51 -0400 Subject: [PATCH 13/21] Include notebook demonstrating `tree` Provides a quick demo showing how to use `tree` to get text-based and HTML-based representations of a Zarr `Group` object. --- notebooks/repr_tree.ipynb | 177 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 notebooks/repr_tree.ipynb diff --git a/notebooks/repr_tree.ipynb b/notebooks/repr_tree.ipynb new file mode 100644 index 0000000000..73affc1e0f --- /dev/null +++ b/notebooks/repr_tree.ipynb @@ -0,0 +1,177 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import zarr" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "g1 = zarr.group()\n", + "g3 = g1.create_group('bar')\n", + "g3.create_group('baz')\n", + "g5 = g3.create_group('quux')\n", + "g5.create_dataset('baz', shape=100, chunks=10)\n", + "g7 = g3.create_group('zoo')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/\n", + " +-- bar\n", + " +-- baz\n", + " +-- quux\n", + " | +-- baz[...]\n", + " +-- zoo\n" + ] + } + ], + "source": [ + "print(g1.tree())" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "
    \n", + "
      \n", + "
    • /
      \n", + "
        \n", + "
      • bar
        \n", + "
          \n", + "
        • baz
          • \n", + "
        • quux
          \n", + "
            \n", + "
          • baz[...]
            • \n", + "
          \n", + "
          • \n", + "
        • zoo
          • \n", + "
        \n", + "
        • \n", + "
      \n", + "
      • \n", + "
    \n", + "
    \n" + ], + "text/plain": [ + "/\n", + " +-- bar\n", + " +-- baz\n", + " +-- quux\n", + " | +-- baz[...]\n", + " +-- zoo" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "g1.tree()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From f5ce2ee7cff949996442db3c961f0273af763c1f Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:52 -0400 Subject: [PATCH 14/21] Support unicode or bytes with TreeHierarchy Works around some Python 2/3 pains by implementing `__bytes__` and `__unicode__`. This allows a user to get a nicer Unicode text-based representation of `tree` on Python 2 even though `__repr__` will not permit it. Also allows Python 3 to get the nicer Unicode representation for free. Though users can still force the `bytes` representation should they prefer it for some reason. --- zarr/hierarchy.py | 16 +++++----- zarr/tests/test_hierarchy.py | 40 ++++++++++++++++++++--- zarr/util.py | 61 +++++++++++++++++++++++++++--------- 3 files changed, 89 insertions(+), 28 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 1c0a89dc07..cad8c74886 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -537,16 +537,16 @@ def tree(self): >>> d1 = g5.create_dataset('baz', shape=100, chunks=10) >>> print(g1.tree()) / - +-- bar - | +-- baz - | +-- quux - | +-- baz[...] - +-- foo + ├── bar + │ ├── baz + │ └── quux + │ └── baz[...] + └── foo >>> print(g3.tree()) bar - +-- baz - +-- quux - +-- baz[...] + ├── baz + └── quux + └── baz[...] """ return TreeHierarchy(self) diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index bd0706eba9..3adfb2ac23 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -18,6 +18,7 @@ from zarr.storage import DictStore, DirectoryStore, ZipStore, init_group, \ init_array, attrs_key, array_meta_key, group_meta_key, atexit_rmtree from zarr.core import Array +from zarr.compat import PY2, text_type from zarr.hierarchy import Group, group, open_group from zarr.attrs import Attributes from zarr.errors import PermissionError @@ -619,13 +620,25 @@ def test_tree(self): g5.create_dataset('baz', shape=100, chunks=10) # test - sg1 = textwrap.dedent(u"""\ + bg1 = textwrap.dedent(u"""\ / +-- bar | +-- baz | +-- quux | +-- baz[...] - +-- foo""") + +-- foo""").encode() + eq(bg1, bytes(g1.tree())) + ug1 = textwrap.dedent(u"""\ + / + ├── bar + │ ├── baz + │ └── quux + │ └── baz[...] + └── foo""") + eq(ug1, text_type(g1.tree())) + sg1 = ug1 + if PY2: + sg1 = bg1 eq(sg1, repr(g1.tree())) hg1 = textwrap.dedent(u"""\
    @@ -649,8 +662,15 @@ def test_tree(self):
    """) eq(hg1, g1.tree()._repr_html_().split("")[1].strip()) - sg2 = textwrap.dedent(u"""\ + bg2 = textwrap.dedent(u"""\ + foo""").encode() + eq(bg2, bytes(g2.tree())) + ug2 = textwrap.dedent(u"""\ foo""") + eq(ug2, text_type(g2.tree())) + sg2 = ug2 + if PY2: + sg2 = bg2 eq(sg2, repr(g2.tree())) hg2 = textwrap.dedent(u"""\
    @@ -660,11 +680,21 @@ def test_tree(self):
    """) eq(hg2, g2.tree()._repr_html_().split("")[1].strip()) - sg3 = textwrap.dedent(u"""\ + bg3 = textwrap.dedent(u"""\ bar +-- baz +-- quux - +-- baz[...]""") + +-- baz[...]""").encode() + eq(bg3, bytes(g3.tree())) + ug3 = textwrap.dedent(u"""\ + bar + ├── baz + └── quux + └── baz[...]""") + eq(ug3, text_type(g3.tree())) + sg3 = ug3 + if PY2: + sg3 = bg3 eq(sg3, repr(g3.tree())) hg3 = textwrap.dedent(u"""\
    diff --git a/zarr/util.py b/zarr/util.py index 4ff63fccfe..1945fe993e 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -485,30 +485,61 @@ class TreeHierarchy(object): def __init__(self, group): self.group = group - self.ascii_kwargs = dict() - - self.update_ascii_kwargs( - gfx=dict( - UP_AND_RIGHT="+", - HORIZONTAL="-", - VERTICAL="|", - VERTICAL_AND_RIGHT="+" - ), + + self.text_kwargs = dict( horiz_len=2, label_space=1, indent=1 ) - def update_ascii_kwargs(self, **ascii_kwargs): - self.ascii_kwargs.update(ascii_kwargs) - self.ascii_draw = LeftAligned( + self.bytes_kwargs = dict( + UP_AND_RIGHT="+", + HORIZONTAL="-", + VERTICAL="|", + VERTICAL_AND_RIGHT="+" + ) + + self.unicode_kwargs = dict( + UP_AND_RIGHT=u"\u2514", + HORIZONTAL=u"\u2500", + VERTICAL=u"\u2502", + VERTICAL_AND_RIGHT=u"\u251C" + ) + + def __bytes__(self): + drawer = LeftAligned( traverse=ZarrGroupTraversal(), - draw=BoxStyle(**self.ascii_kwargs) + draw=BoxStyle(gfx=self.bytes_kwargs, **self.text_kwargs) ) - return self + + result = drawer(self.group) + + # Unicode characters slip in on Python 3. + # So we need to straighten that out first. + if not PY2: + result = result.encode() + + return result + + def __unicode__(self): + drawer = LeftAligned( + traverse=ZarrGroupTraversal(), + draw=BoxStyle(gfx=self.unicode_kwargs, **self.text_kwargs) + ) + + return drawer(self.group) + + def __str__(self): + if PY2: + return self.__bytes__() + else: + return self.__unicode__() def __repr__(self): - return self.ascii_draw(self.group) + if PY2: + return self.__bytes__() + else: + return self.__unicode__() def _repr_html_(self): return custom_html_list(self.group) From 590f98d3dda9004f77a6a7411786254f3852e3c0 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:53 -0400 Subject: [PATCH 15/21] Try unicode and bytes in notebook --- notebooks/repr_tree.ipynb | 43 +++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/notebooks/repr_tree.ipynb b/notebooks/repr_tree.ipynb index 73affc1e0f..053cf94196 100644 --- a/notebooks/repr_tree.ipynb +++ b/notebooks/repr_tree.ipynb @@ -6,7 +6,13 @@ "metadata": {}, "outputs": [], "source": [ - "import zarr" + "import zarr\n", + "\n", + "# Python 2/3 trick\n", + "try:\n", + " unicode\n", + "except NameError:\n", + " unicode = str" ] }, { @@ -42,13 +48,34 @@ } ], "source": [ - "print(g1.tree())" + "print(bytes(g1.tree()).decode())" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "bar\n", + " ├── baz\n", + " ├── quux\n", + " │ └── baz[...]\n", + " └── zoo\n" + ] + } + ], + "source": [ + "print(unicode(g3.tree()))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, "outputs": [ { "data": { @@ -129,14 +156,14 @@ ], "text/plain": [ "/\n", - " +-- bar\n", - " +-- baz\n", - " +-- quux\n", - " | +-- baz[...]\n", - " +-- zoo" + " └── bar\n", + " ├── baz\n", + " ├── quux\n", + " │ └── baz[...]\n", + " └── zoo" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } From d9682de30275eb2fa5234034be85733910fd3c46 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:53 -0400 Subject: [PATCH 16/21] Fix inline code quote in tree docstring --- zarr/hierarchy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index cad8c74886..8e3a0046f5 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -524,7 +524,7 @@ def visititems(self, func): return self.visitvalues(lambda o: func(o.name[base_len:].lstrip("/"), o)) def tree(self): - """Provide a ``print`-able display of the hierarchy. + """Provide a ``print``-able display of the hierarchy. Examples -------- From 7b9a6ae55ec578f48bc49dcad9d773477d0365ba Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:54 -0400 Subject: [PATCH 17/21] Consolidate `if`s in `tree`'s HTML sublist build --- zarr/util.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zarr/util.py b/zarr/util.py index 1945fe993e..a31f7f6715 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -396,10 +396,9 @@ def custom_html_sublist(group, indent): children = traverser.get_children(group) if children: result += """\n{0}{0}
      \n""".format(indent) - for c in children: - for l in custom_html_sublist(c, indent).splitlines(): - result += "{0}{0}{1}\n".format(indent, l) - if children: + for c in children: + for l in custom_html_sublist(c, indent).splitlines(): + result += "{0}{0}{1}\n".format(indent, l) result += "{0}{0}
    \n{0}".format(indent) result += ( From b7a5fd93c77d255d6e41a365f4aa8612e261f00b Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:57 -0400 Subject: [PATCH 18/21] Drop unneeded formatting at tree's HTML list end --- zarr/util.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/zarr/util.py b/zarr/util.py index a31f7f6715..12f51041d2 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -401,11 +401,7 @@ def custom_html_sublist(group, indent): result += "{0}{0}{1}\n".format(indent, l) result += "{0}{0}\n{0}".format(indent) - result += ( - """\n""".format( - indent, traverser.get_text(group) - ) - ) + result += "\n" return result From 8da154a3795f24b04c05c931f425022bc0b8a5b9 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:58 -0400 Subject: [PATCH 19/21] Change zarrTree to zarr-tree --- notebooks/repr_tree.ipynb | 26 +++++++++++++------------- zarr/tests/test_hierarchy.py | 6 +++--- zarr/util.py | 26 +++++++++++++------------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/notebooks/repr_tree.ipynb b/notebooks/repr_tree.ipynb index 053cf94196..a45162d720 100644 --- a/notebooks/repr_tree.ipynb +++ b/notebooks/repr_tree.ipynb @@ -81,36 +81,36 @@ "data": { "text/html": [ "\n", "\n", - "
    \n", + "
    \n", "
      \n", "
    • /
      \n", "
        \n", diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 3adfb2ac23..f71f749d7b 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -641,7 +641,7 @@ def test_tree(self): sg1 = bg1 eq(sg1, repr(g1.tree())) hg1 = textwrap.dedent(u"""\ -
        +
        • /
            @@ -673,7 +673,7 @@ def test_tree(self): sg2 = bg2 eq(sg2, repr(g2.tree())) hg2 = textwrap.dedent(u"""\ -
            +
            • foo
            @@ -697,7 +697,7 @@ def test_tree(self): sg3 = bg3 eq(sg3, repr(g3.tree())) hg3 = textwrap.dedent(u"""\ -
            +
            • bar
                diff --git a/zarr/util.py b/zarr/util.py index 12f51041d2..3238ac4a30 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -412,36 +412,36 @@ def custom_html_list(group, indent=" "): # Add custom CSS style for our HTML list result += """\n\n" # Insert the HTML list - result += """
                \n""" + result += """
                \n""" result += "
                  \n" result += custom_html_sublist(group, indent=indent) result += "
                \n" From 3a4c65cf5cc3cfea61a82a86af872a7c09d947ec Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:12:59 -0400 Subject: [PATCH 20/21] Change TreeHierarchy to TreeViewer --- zarr/hierarchy.py | 4 ++-- zarr/util.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 8e3a0046f5..a7d35d1b92 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -12,7 +12,7 @@ DictStore, DirectoryStore, group_meta_key, attrs_key, listdir, rmdir from zarr.creation import array, create, empty, zeros, ones, full, \ empty_like, zeros_like, ones_like, full_like -from zarr.util import normalize_storage_path, normalize_shape, InfoReporter, TreeHierarchy +from zarr.util import normalize_storage_path, normalize_shape, InfoReporter, TreeViewer from zarr.errors import err_contains_array, err_contains_group, err_group_not_found, err_read_only from zarr.meta import decode_group_metadata @@ -549,7 +549,7 @@ def tree(self): └── baz[...] """ - return TreeHierarchy(self) + return TreeViewer(self) def _write_op(self, f, *args, **kwargs): diff --git a/zarr/util.py b/zarr/util.py index 3238ac4a30..b88d0ce3b4 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -476,7 +476,7 @@ def custom_html_list(group, indent=" "): return result -class TreeHierarchy(object): +class TreeViewer(object): def __init__(self, group): self.group = group From d3d06db251ce71b40d3f54dbc7da819433634bd0 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 26 Oct 2017 20:13:01 -0400 Subject: [PATCH 21/21] Drop `TreeViewer`'s `__str__` As all Python `object`s will fallback to `__repr__` if `__str__` is undefined on both Python 2 and Python 3, there is no need for us to implement `__str__` in basically the same way as `__repr__`. So this just drops `__str__` so that `__repr__` can fill in as needed. --- zarr/util.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zarr/util.py b/zarr/util.py index b88d0ce3b4..86247f8a37 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -524,12 +524,6 @@ def __unicode__(self): return drawer(self.group) - def __str__(self): - if PY2: - return self.__bytes__() - else: - return self.__unicode__() - def __repr__(self): if PY2: return self.__bytes__()