diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..d386d4d4d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,37 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: 'bug, awaiting-triage' +assignees: '' + +--- + +## Bug Report + +### Description +A clear and concise description of what is the overall operation that is intended to be performed that resulted in an error. + +### Reproducibility +Include: +- OS (WIN | MACOS | Linux) +- Python Version OR MATLAB Version +- MySQL Version +- MySQL Deployment Strategy (local-native | local-docker | remote) +- DataJoint Version +- Minimum number of steps to reliably reproduce the issue +- Complete error stack as a result of evaluating the above steps + +### Expected Behavior +A clear and concise description of what you expected to happen. + +### Screenshots +If applicable, add screenshots to help explain your problem. + +### Additional Research and Context +Add any additional research or context that was conducted in creating this report. + +For example: +- Related GitHub issues and PR's either within this repository or in other relevant repositories. +- Specific links to specific lines or a focus within source code. +- Relevant summary of Maintainers development meetings, milestones, projects, etc. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..4d4eeffd9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,46 @@ +--- +name: Feature request +about: Suggest an idea for a new feature +title: '' +labels: 'enhancement, awaiting-triage' +assignees: '' + +--- + +## Feature Request + +### Problem +A clear and concise description how this idea has manifested and the context. Elaborate on the need for this feature and/or what could be improved. Ex. I'm always frustrated when [...] + +### Requirements +A clear and concise description of the requirements to satisfy the new feature. Detail what you expect from a successful implementation of the feature. Ex. When using this feature, it should [...] + +### Justification +Provide the key benefits in making this a supported feature. Ex. Adding support for this feature would ensure [...] + +### Alternative Considerations +Do you currently have a work-around for this? Provide any alternative solutions or features you've considered. + +### Related Errors +Add any errors as a direct result of not exposing this feature. + +Please include steps to reproduce provided errors as follows: +- OS (WIN | MACOS | Linux) +- Python Version OR MATLAB Version +- MySQL Version +- MySQL Deployment Strategy (local-native | local-docker | remote) +- DataJoint Version +- Minimum number of steps to reliably reproduce the issue +- Complete error stack as a result of evaluating the above steps + +### Screenshots +If applicable, add screenshots to help explain your feature. + +### Additional Research and Context +Add any additional research or context that was conducted in creating this feature request. + +For example: +- Related GitHub issues and PR's either within this repository or in other relevant repositories. +- Specific links to specific line or focus within source code. +- Relevant summary of Maintainers development meetings, milestones, projects, etc. +- Any additional supplemental web references or links that would further justify this feature request. diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml new file mode 100644 index 000000000..55ced7eb9 --- /dev/null +++ b/.github/workflows/development.yaml @@ -0,0 +1,53 @@ +name: Development +on: + push: + branches: + - '**' # every branch + - '!stage*' # exclude branches beginning with stage + pull_request: + branches: + - '**' # every branch + - '!stage*' # exclude branches beginning with stage +jobs: + test: + if: github.event_name == 'push' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + strategy: + matrix: + py_ver: ["3.8"] + mysql_ver: ["8.0", "5.7", "5.6"] + include: + - py_ver: "3.7" + mysql_ver: "5.7" + - py_ver: "3.6" + mysql_ver: "5.7" + - py_ver: "3.5" + mysql_ver: "5.7" + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{matrix.py_ver}} + uses: actions/setup-python@v2 + with: + python-version: ${{matrix.py_ver}} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + - name: Run syntax tests + run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Run primary tests + env: + UID: "1001" + GID: "116" + PY_VER: ${{matrix.py_ver}} + MYSQL_VER: ${{matrix.mysql_ver}} + ALPINE_VER: "3.10" + MINIO_VER: RELEASE.2019-09-26T19-42-35Z + COMPOSE_HTTP_TIMEOUT: "120" + COVERALLS_SERVICE_NAME: travis-ci + COVERALLS_REPO_TOKEN: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY + run: docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app + - name: Run style tests + run: | + flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \ + --count --max-complexity=62 --max-line-length=127 --statistics \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 72c11490b..000000000 --- a/.travis.yml +++ /dev/null @@ -1,59 +0,0 @@ -sudo: required -env: - global: - - MINIO_VER="RELEASE.2019-09-26T19-42-35Z" - - ALPINE_VER="3.10" - - COMPOSE_HTTP_TIMEOUT="300" - - UID="2000" - - GID="2000" - - COVERALLS_SERVICE_NAME="travis-ci" - - COVERALLS_REPO_TOKEN="fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY" -services: -- docker -main: &main - stage: "Tests & Coverage: Alpine" - os: linux - dist: xenial # precise, trusty, xenial, bionic - language: shell - script: - - docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app -jobs: - include: - - stage: "Lint: Syntax" - language: python - install: - - pip install flake8 - script: - - flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics - - <<: *main - env: - - PY_VER: "3.8" - - MYSQL_VER: "5.7" - - <<: *main - env: - - PY_VER: "3.7" - - MYSQL_VER: "5.7" - - <<: *main - env: - - PY_VER: "3.6" - - MYSQL_VER: "5.7" - - <<: *main - env: - - PY_VER: "3.5" - - MYSQL_VER: "5.7" - - <<: *main - env: - - PY_VER: "3.8" - - MYSQL_VER: "8.0" - - <<: *main - env: - - PY_VER: "3.8" - - MYSQL_VER: "5.6" - - stage: "Lint: Style" - language: python - install: - - pip install flake8 - script: - - | - flake8 --ignore=E121,E123,E126,E226,E24,E704,W503,W504,E722,F401,W605 datajoint \ - --count --max-complexity=62 --max-line-length=127 --statistics \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 6920ad1f7..4b276ed07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,20 @@ ## Release notes -### 0.13.0 -- Dec 15, 2020 +### 0.13.0 -- Jan 11, 2020 * Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484). PR #754 * Re-implement cascading deletes for better performance. PR #839. * Add table method `.update1` to update an existing row in its table. * Python datatypes are now enabled by default in blobs (#761). PR #785 * Added permissive join and restriction operators `@` and `^` (#785) PR #754 -### 0.12.8 -- Nov 30, 2020 -* table.children, .parents, .descendents, and ancestors optionally return queryable objects. PR #833 +### 0.12.8 -- Dec 22, 2020 +* table.children, .parents, .descendents, and ancestors can return queryable objects. PR #833 +* Load dependencies before querying dependencies. (#179) PR #833 +* Fix display of part tables in `schema.save`. (#821) PR #833 +* Add `schema.list_tables`. (#838) PR #844 +* Fix minio new version regression. PR #847 +* Add more S3 logging for debugging. (#831) PR #832 +* Convert testing framework from TravisCI to GitHub Actions (#841) PR #840 ### 0.12.7 -- Oct 27, 2020 * Fix case sensitivity issues to adapt to MySQL 8+. PR #819 diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py index c1b0c4653..c44787eac 100644 --- a/datajoint/autopopulate.py +++ b/datajoint/autopopulate.py @@ -7,7 +7,6 @@ from tqdm import tqdm from .expression import QueryExpression, AndList from .errors import DataJointError, LostConnectionError -from .table import FreeTable import signal # noinspection PyExceptionInherit,PyCallingNonCallable @@ -27,30 +26,24 @@ class AutoPopulate: @property def key_source(self): """ - :return: the query whose primary key values are passed, sequentially, to the - `make` method when populate() is called. + :return: the relation whose primary key values are passed, sequentially, to the + ``make`` method when populate() is called. The default value is the join of the parent relations. Users may override to change the granularity or the scope of populate() calls. """ - def parent_gen(): - if self.target.full_table_name not in self.connection.dependencies: - self.connection.dependencies.load() - for parent_name, fk_props in self.target.parents(primary=True).items(): - if not parent_name.isdigit(): # simple foreign key - yield FreeTable(self.connection, parent_name).proj() - else: - grandparent = list(self.connection.dependencies.in_edges(parent_name))[0][0] - yield FreeTable(self.connection, grandparent).proj(**{ - attr: ref for attr, ref in fk_props['attr_map'].items() if ref != attr}) + def _rename_attributes(table, props): + return (table.proj( + **{attr: ref for attr, ref in props['attr_map'].items() if attr != ref}) + if props['aliased'] else table) if self._key_source is None: - parents = parent_gen() - try: - self._key_source = next(parents) - except StopIteration: - raise DataJointError('A relation must have primary dependencies for auto-populate to work') from None - for q in parents: - self._key_source *= q + parents = self.target.parents(primary=True, as_objects=True, foreign_key_info=True) + if not parents: + raise DataJointError( + 'A relation must have primary dependencies for auto-populate to work') from None + self._key_source = _rename_attributes(*parents[0]) + for q in parents[1:]: + self._key_source *= _rename_attributes(*q) return self._key_source def make(self, key): diff --git a/datajoint/dependencies.py b/datajoint/dependencies.py index 170eeac80..a4e1afb0c 100644 --- a/datajoint/dependencies.py +++ b/datajoint/dependencies.py @@ -1,9 +1,36 @@ import networkx as nx import itertools +import re from collections import defaultdict, OrderedDict from .errors import DataJointError +def unite_master_parts(lst): + """ + re-order a list of table names so that part tables immediately follow their master tables without breaking + the topological order. + Without this correction, a simple topological sort may insert other descendants between master and parts. + The input list must be topologically sorted. + :example: + unite_master_parts( + ['`s`.`a`', '`s`.`a__q`', '`s`.`b`', '`s`.`c`', '`s`.`c__q`', '`s`.`b__q`', '`s`.`d`', '`s`.`a__r`']) -> + ['`s`.`a`', '`s`.`a__q`', '`s`.`a__r`', '`s`.`b`', '`s`.`b__q`', '`s`.`c`', '`s`.`c__q`', '`s`.`d`'] + """ + for i in range(2, len(lst)): + name = lst[i] + match = re.match(r'(?P`\w+`.`\w+)__\w+`', name) + if match: # name is a part table + master = match.group('master') + for j in range(i-1, -1, -1): + if lst[j] == master + '`' or lst[j].startswith(master + '__'): + # move from the ith position to the (j+1)th position + lst[j+1:i+1] = [name] + lst[j+1:i] + break + else: + raise DataJointError("Found a part table {name} without its master table.".format(name=name)) + return lst + + class Dependencies(nx.DiGraph): """ The graph of dependencies (foreign keys) between loaded tables. @@ -118,8 +145,8 @@ def descendants(self, full_table_name): self.load(force=False) nodes = self.subgraph( nx.algorithms.dag.descendants(self, full_table_name)) - return [full_table_name] + list( - nx.algorithms.dag.topological_sort(nodes)) + return unite_master_parts([full_table_name] + list( + nx.algorithms.dag.topological_sort(nodes))) def ancestors(self, full_table_name): """ @@ -129,5 +156,5 @@ def ancestors(self, full_table_name): self.load(force=False) nodes = self.subgraph( nx.algorithms.dag.ancestors(self, full_table_name)) - return [full_table_name] + list(reversed(list( - nx.algorithms.dag.topological_sort(nodes)))) + return list(reversed(unite_master_parts(list( + nx.algorithms.dag.topological_sort(nodes)) + [full_table_name]))) diff --git a/datajoint/diagram.py b/datajoint/diagram.py index e610fa3f2..dd48e7b17 100644 --- a/datajoint/diagram.py +++ b/datajoint/diagram.py @@ -5,6 +5,7 @@ import warnings import inspect from .table import Table +from .dependencies import unite_master_parts try: from matplotlib import pyplot as plt @@ -155,29 +156,8 @@ def is_part(part, master): return self def topological_sort(self): - """ - :return: list of nodes in topological order - """ - - def _unite(lst): - """ - reorder list so that parts immediately follow their masters without breaking the topological order. - Without this correction, simple topological sort may insert other descendants between master and parts - :example: - _unite(['a', 'a__q', 'b', 'c', 'c__q', 'b__q', 'd', 'a__r']) - -> ['a', 'a__q', 'a__r', 'b', 'b__q', 'c', 'c__q', 'd'] - """ - if len(lst) <= 2: - return lst - el = lst.pop() - lst = _unite(lst) - if '__' in el: - master = el.split('__')[0] - if not lst[-1].startswith(master): - return _unite(lst[:-1] + [el, lst[-1]]) - return lst + [el] - - return _unite(list(nx.algorithms.dag.topological_sort( + """ :return: list of nodes in topological order """ + return unite_master_parts(list(nx.algorithms.dag.topological_sort( nx.DiGraph(self).subgraph(self.nodes_to_show)))) def __add__(self, arg): diff --git a/datajoint/expression.py b/datajoint/expression.py index 1cd731b92..7ea62996f 100644 --- a/datajoint/expression.py +++ b/datajoint/expression.py @@ -34,7 +34,6 @@ class QueryExpression: 2. A projection is applied remapping remapped attributes 3. Subclasses: Join, Aggregation, and Union have additional specific rules. """ - _restriction = None _restriction_attributes = None _left = [] # True for left joins, False for inner joins @@ -80,11 +79,11 @@ def restriction_attributes(self): def primary_key(self): return self.heading.primary_key - __subquery_alias_count = count() # count for alias names used in from_clause + _subquery_alias_count = count() # count for alias names used in from_clause def from_clause(self): support = ('(' + src.make_sql() + ') as `_s%x`' % next( - self.__subquery_alias_count) if isinstance(src, QueryExpression) else src for src in self.support) + self._subquery_alias_count) if isinstance(src, QueryExpression) else src for src in self.support) clause = next(support) for s, a, left in zip(support, self._join_attributes, self._left): clause += '{left} JOIN {clause}{using}'.format( @@ -93,9 +92,9 @@ def from_clause(self): using="" if not a else " USING (%s)" % ",".join('`%s`' % _ for _ in a)) return clause - @property def where_clause(self): - return '' if not self.restriction else ' WHERE(%s)' % ')AND('.join(str(s) for s in self.restriction) + return '' if not self.restriction else ' WHERE(%s)' % ')AND('.join( + str(s) for s in self.restriction) def make_sql(self, fields=None): """ @@ -106,7 +105,7 @@ def make_sql(self, fields=None): return 'SELECT {distinct}{fields} FROM {from_}{where}'.format( distinct="DISTINCT " if distinct else "", fields=self.heading.as_sql(fields or self.heading.names), - from_=self.from_clause(), where=self.where_clause) + from_=self.from_clause(), where=self.where_clause()) # --------- query operators ----------- def make_subquery(self): @@ -180,7 +179,7 @@ def restrict(self, restriction): result = self.make_subquery() else: result = copy.copy(self) - result._restriction = AndList(self.restriction) # make a copy to protect the original + result._restriction = AndList(self.restriction) # copy to preserve the original result.restriction.append(new_condition) result.restriction_attributes.update(attributes) return result @@ -385,6 +384,11 @@ def aggr(self, group, *attributes, keep_all_rows=False, **named_attributes): :param named_attributes: computations of the form new_attribute="sql expression on attributes of group" :return: The derived query expression """ + if Ellipsis in attributes: + # expand ellipsis to include only attributes from the left table + attributes = set(attributes) + attributes.discard(Ellipsis) + attributes.update(self.heading.secondary_attributes) return Aggregation.create( self, group=group, keep_all_rows=keep_all_rows).proj(*attributes, **named_attributes) @@ -421,25 +425,27 @@ def tail(self, limit=25, **fetch_kwargs): def __len__(self): """ :return: number of elements in the result set """ - what = '*' if set(self.heading.names) != set(self.primary_key) else 'DISTINCT %s' % ','.join( - (self.heading[k].attribute_expression or '`%s`' % k for k in self.primary_key)) return self.connection.query( - 'SELECT count({what}) FROM {from_}{where}'.format( - what=what, + 'SELECT count(DISTINCT {fields}) FROM {from_}{where}'.format( + fields=self.heading.as_sql(self.primary_key, include_aliases=False), from_=self.from_clause(), - where=self.where_clause)).fetchone()[0] + where=self.where_clause())).fetchone()[0] def __bool__(self): """ - :return: True if the result is not empty. Equivalent to len(rel)>0 but may be more efficient. + :return: True if the result is not empty. Equivalent to len(self) > 0 but often faster. """ - return len(self) > 0 + return bool(self.connection.query( + 'SELECT EXISTS(SELECT 1 FROM {from_}{where})'.format( + from_=self.from_clause(), + where=self.where_clause())).fetchone()[0]) def __contains__(self, item): """ returns True if item is found in the . :param item: any restriction - (item in query_expression) is equivalent to bool(query_expression & item) but may be executed more efficiently. + (item in query_expression) is equivalent to bool(query_expression & item) but may be + executed more efficiently. """ return bool(self & item) # May be optimized e.g. using an EXISTS query @@ -453,7 +459,8 @@ def __next__(self): key = self._iter_keys.pop(0) except AttributeError: # self._iter_keys is missing because __iter__ has not been called. - raise TypeError("'QueryExpression' object is not an iterator. Use iter(obj) to create an iterator.") + raise TypeError("A QueryExpression object is not an iterator. " + "Use iter(obj) to create an iterator.") except IndexError: raise StopIteration else: @@ -463,7 +470,8 @@ def __next__(self): try: return (self & key).fetch1() except DataJointError: - # The data may have been deleted since the moment the keys were fetched -- move on to next entry. + # The data may have been deleted since the moment the keys were fetched + # -- move on to next entry. return next(self) def cursor(self, offset=0, limit=None, order_by=None, as_dict=False): @@ -495,13 +503,15 @@ def _repr_html_(self): class Aggregation(QueryExpression): """ - Aggregation(rel, comp1='expr1', ..., compn='exprn') yields an entity set with the primary key specified by rel.heading. - The computed arguments comp1, ..., compn use aggregation operators on the attributes of rel. + Aggregation.create(arg, group, comp1='calc1', ..., compn='calcn') yields an entity set + with primary key from arg. + The computed arguments comp1, ..., compn use aggregation calculations on the attributes of + group or simple projections and calculations on the attributes of arg. Aggregation is used QueryExpression.aggr and U.aggr. Aggregation is a private class in DataJoint, not exposed to users. """ _left_restrict = None # the pre-GROUP BY conditions for the WHERE clause - __subquery_alias_count = count() + _subquery_alias_count = count() @classmethod def create(cls, arg, group, keep_all_rows=False): @@ -517,12 +527,16 @@ def create(cls, arg, group, keep_all_rows=False): result._support = join.support result._join_attributes = join._join_attributes result._left = join._left - result.initial_restriction = join.restriction # WHERE clause applied before GROUP BY + result._left_restrict = join.restriction # WHERE clause applied before GROUP BY result._grouping_attributes = result.primary_key + return result + def where_clause(self): + return '' if not self._left_restrict else ' WHERE (%s)' % ')AND('.join( + str(s) for s in self._left_restrict) + def make_sql(self, fields=None): - where = '' if not self._left_restrict else ' WHERE (%s)' % ')AND('.join(self._left_restrict) fields = self.heading.as_sql(fields or self.heading.names) assert self._grouping_attributes or not self.restriction distinct = set(self.heading.names) == set(self.primary_key) @@ -530,18 +544,20 @@ def make_sql(self, fields=None): distinct="DISTINCT " if distinct else "", fields=fields, from_=self.from_clause(), - where=where, + where=self.where_clause(), group_by="" if not self.primary_key else ( " GROUP BY `%s`" % '`,`'.join(self._grouping_attributes) + ("" if not self.restriction else ' HAVING (%s)' % ')AND('.join(self.restriction)))) def __len__(self): - what = '*' if set(self.heading.names) != set(self.primary_key) else 'DISTINCT `%s`' % '`,`'.join(self.primary_key) return self.connection.query( - 'SELECT count({what}) FROM ({subquery}) as `${alias:x}`'.format( - what=what, + 'SELECT count(1) FROM ({subquery}) `${alias:x}`'.format( subquery=self.make_sql(), - alias=next(self.__subquery_alias_count))).fetchone()[0] + alias=next(self._subquery_alias_count))).fetchone()[0] + + def __bool__(self): + return bool(self.connection.query( + 'SELECT EXISTS({sql})'.format(sql=self.make_sql()))) class Union(QueryExpression): @@ -553,36 +569,54 @@ def create(cls, arg1, arg2): if inspect.isclass(arg2) and issubclass(arg2, QueryExpression): arg2 = arg2() # instantiate if a class if not isinstance(arg2, QueryExpression): - raise DataJointError('A QueryExpression can only be unioned with another QueryExpression') + raise DataJointError( + "A QueryExpression can only be unioned with another QueryExpression") if arg1.connection != arg2.connection: - raise DataJointError("Cannot operate on QueryExpressions originating from different connections.") + raise DataJointError( + "Cannot operate on QueryExpressions originating from different connections.") if set(arg1.primary_key) != set(arg2.primary_key): raise DataJointError("The operands of a union must share the same primary key.") if set(arg1.heading.secondary_attributes) & set(arg2.heading.secondary_attributes): - raise DataJointError("The operands of a union must not share any secondary attributes.") + raise DataJointError( + "The operands of a union must not share any secondary attributes.") result = cls() result._connection = arg1.connection result._heading = arg1.heading.join(arg2.heading) result._support = [arg1, arg2] return result - def make_sql(self, select_fields=None): + def make_sql(self): arg1, arg2 = self._support - if not arg1.heading.secondary_attributes and not arg2.heading.secondary_attributes: # use UNION DISTINCT - fields = select_fields or arg1.primary_key - return "({sql1}) UNION ({sql2})".format(sql1=arg1.make_sql(fields), sql2=arg2.make_sql(fields)) - fields = select_fields or self.heading.names + if not arg1.heading.secondary_attributes and not arg2.heading.secondary_attributes: + # no secondary attributes: use UNION DISTINCT + fields = arg1.primary_key + return "({sql1}) UNION ({sql2})".format( + sql1=arg1.make_sql(fields), + sql2=arg2.make_sql(fields)) + # with secondary attributes, use union of left join with antijoin + fields = self.heading.names sql1 = arg1.join(arg2, left=True).make_sql(fields) - sql2 = (arg2 - arg1).proj(..., **{k: 'NULL' for k in arg1.heading.secondary_attributes}).make_sql(fields) + sql2 = (arg2 - arg1).proj( + ..., **{k: 'NULL' for k in arg1.heading.secondary_attributes}).make_sql(fields) return "({sql1}) UNION ({sql2})".format(sql1=sql1, sql2=sql2) def from_clause(self): - """In Union, the select clause can be used as the WHERE clause and make_sql() does not call from_clause""" - return self.make_sql() + """ The union does not use a FROM clause """ + assert False + + def where_clause(self): + """ The union does not use a WHERE clause """ + assert False def __len__(self): return self.connection.query( - 'SELECT count(*) FROM ({sql}) `$sub`'.format(sql=self.make_sql())).fetchone()[0] + 'SELECT count(1) FROM ({subquery}) `${alias:x}`'.format( + subquery=self.make_sql(), + alias=next(QueryExpression._subquery_alias_count))).fetchone()[0] + + def __bool__(self): + return bool(self.connection.query( + 'SELECT EXISTS({sql})'.format(sql=self.make_sql()))) class U: @@ -688,7 +722,8 @@ def aggr(self, group, **named_attributes): :return: The derived query expression """ if named_attributes.get('keep_all_rows', False): - raise DataJointError('Cannot set keep_all_rows=True when aggregating on a universal set.') + raise DataJointError( + 'Cannot set keep_all_rows=True when aggregating on a universal set.') return Aggregation.create(self, group=group, keep_all_rows=False).proj(**named_attributes) aggregate = aggr # alias for aggr diff --git a/datajoint/heading.py b/datajoint/heading.py index 6e4db0134..6a929b337 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -148,13 +148,14 @@ def as_dtype(self): names=self.names, formats=[v.dtype for v in self.attributes.values()])) - def as_sql(self, fields): + def as_sql(self, fields, include_aliases=True): """ represent heading as the SQL SELECT clause. """ - return ','.join('`%s`' % name if self.attributes[name].attribute_expression is None - else '%s as `%s`' % (self.attributes[name].attribute_expression, name) - for name in fields) + return ','.join( + '`%s`' % name if self.attributes[name].attribute_expression is None + else self.attributes[name].attribute_expression + (' as `%s`' % name if include_aliases else '') + for name in fields) def __iter__(self): return iter(self.attributes) diff --git a/datajoint/s3.py b/datajoint/s3.py index 72e0f4f06..0e3efb2dd 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -37,8 +37,11 @@ def get(self, name): logger.debug('get: {}:{}'.format(self.bucket, name)) try: return self.client.get_object(self.bucket, str(name)).data - except minio.error.NoSuchKey: - raise errors.MissingExternalFile('Missing s3 key %s' % name) from None + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + raise errors.MissingExternalFile('Missing s3 key %s' % name) from None + else: + raise e def fget(self, name, local_filepath): """get file from object name to local filepath""" @@ -59,16 +62,22 @@ def exists(self, name): logger.debug('exists: {}:{}'.format(self.bucket, name)) try: self.client.stat_object(self.bucket, str(name)) - except minio.error.NoSuchKey: - return False + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + return False + else: + raise e return True def get_size(self, name): logger.debug('get_size: {}:{}'.format(self.bucket, name)) try: return self.client.stat_object(self.bucket, str(name)).size - except minio.error.NoSuchKey: - raise errors.MissingExternalFile from None + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + raise errors.MissingExternalFile from None + else: + raise e def remove_object(self, name): logger.debug('remove_object: {}:{}'.format(self.bucket, name)) diff --git a/datajoint/schemas.py b/datajoint/schemas.py index c8c80dbcb..b72502cba 100644 --- a/datajoint/schemas.py +++ b/datajoint/schemas.py @@ -196,9 +196,9 @@ def _decorate_table(self, table_class, context, assert_declared=False): contents = list(instance.contents) if len(contents) > len(instance): if instance.heading.has_autoincrement: - warnings.warn( - 'Contents has changed but cannot be inserted because {table} has autoincrement.'.format( - table=instance.__class__.__name__)) + warnings.warn(('Contents has changed but cannot be inserted because ' + '{table} has autoincrement.').format( + table=instance.__class__.__name__)) else: instance.insert(contents, skip_duplicates=True) @@ -329,26 +329,29 @@ def make_class_definition(table): class_name = table.split('.')[1].strip('`') indent = '' if tier == 'Part': - class_name = class_name.split('__')[1] + class_name = class_name.split('__')[-1] indent += ' ' class_name = to_camel_case(class_name) def replace(s): - d, tab = s.group(1), s.group(2) - return ('' if d == db else (module_lookup[d]+'.')) + to_camel_case(tab) - - return ('' if tier == 'Part' else '@schema\n') + \ - '{indent}class {class_name}(dj.{tier}):\n{indent} definition = """\n{indent} {defi}"""'.format( + d, tabs = s.group(1), s.group(2) + return ('' if d == db else (module_lookup[d]+'.')) + '.'.join( + to_camel_case(tab) for tab in tabs.lstrip('__').split('__')) + + return ('' if tier == 'Part' else '\n@schema\n') + ( + '{indent}class {class_name}(dj.{tier}):\n' + '{indent} definition = """\n' + '{indent} {defi}"""').format( class_name=class_name, indent=indent, tier=tier, - defi=re.sub( - r'`([^`]+)`.`([^`]+)`', replace, - FreeTable(self.connection, table).describe(printout=False).replace('\n', '\n ' + indent))) + defi=re.sub(r'`([^`]+)`.`([^`]+)`', replace, + FreeTable(self.connection, table).describe(printout=False) + ).replace('\n', '\n ' + indent)) diagram = Diagram(self) - body = '\n\n\n'.join(make_class_definition(table) for table in diagram.topological_sort()) - python_code = '\n\n\n'.join(( + body = '\n\n'.join(make_class_definition(table) for table in diagram.topological_sort()) + python_code = '\n\n'.join(( '"""This module was auto-generated by datajoint from an existing schema"""', "import datajoint as dj\n\nschema = dj.Schema('{db}')".format(db=db), '\n'.join("{module} = dj.VirtualModule('{module}', '{schema_name}')".format(module=v, schema_name=k) @@ -359,6 +362,17 @@ def replace(s): with open(python_filename, 'wt') as f: f.write(python_code) + def list_tables(self): + """ + Return a list of all tables in the schema except tables with ~ in first character such + as ~logs and ~job + :return: A list of table names in their raw datajoint naming convection form + """ + + return [table_name for (table_name,) in self.connection.query(""" + SELECT table_name FROM information_schema.tables + WHERE table_schema = %s and table_name NOT LIKE '~%%'""", args=(self.database))] + class VirtualModule(types.ModuleType): """ diff --git a/datajoint/table.py b/datajoint/table.py index 3935b370b..bdcc6a6f0 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -13,7 +13,7 @@ from .condition import make_condition from .expression import QueryExpression from . import blob -from .utils import user_choice +from .utils import user_choice, OrderedDict from .heading import Heading from .errors import DuplicateError, AccessError, DataJointError, UnknownAttributeError, IntegrityError from .version import __version__ as version @@ -58,7 +58,8 @@ def definition(self): def declare(self, context=None): """ Declare the table in the schema based on self.definition. - :param context: the context for foreign key resolution. If None, foreign keys are not allowed. + :param context: the context for foreign key resolution. If None, foreign keys are + not allowed. """ if self.connection.in_transaction: raise DataJointError('Cannot declare new tables inside a transaction, ' @@ -121,38 +122,59 @@ def get_select_fields(self, select_fields=None): """ return '*' if select_fields is None else self.heading.project(select_fields).as_sql - def parents(self, primary=None, as_objects=False): + def parents(self, primary=None, as_objects=False, foreign_key_info=False): """ :param primary: if None, then all parents are returned. If True, then only foreign keys composed of - primary key attributes are considered. If False, the only foreign keys including at least one non-primary - attribute are considered. - :param as_objects: if False (default), the output is a dict describing the foreign keys. If True, return table objects. - :return: dict of tables referenced with self's foreign keys or list of table objects if as_objects=True - """ - parents = self.connection.dependencies.parents(self.full_table_name, primary) + primary key attributes are considered. If False, return foreign keys including at least one + secondary attribute. + :param as_objects: if False, return table names. If True, return table objects. + :param foreign_key_info: if True, each element in result also includes foreign key info. + :return: list of parents as table names or table objects + with (optional) foreign key information. + """ + get_edge = self.connection.dependencies.parents + nodes = [next(iter(get_edge(name).items())) if name.isdigit() else (name, props) + for name, props in get_edge(self.full_table_name, primary).items()] if as_objects: - parents = [FreeTable(self.connection, c) for c in parents] - return parents + nodes = [(FreeTable(self.connection, name), props) for name, props in nodes] + if not foreign_key_info: + nodes = [name for name, props in nodes] + return nodes - def children(self, primary=None, as_objects=False): + def children(self, primary=None, as_objects=False, foreign_key_info=False): """ :param primary: if None, then all children are returned. If True, then only foreign keys composed of - primary key attributes are considered. If False, the only foreign keys including at least one non-primary - attribute are considered. - :param as_objects: if False (default), the output is a dict describing the foreign keys. If True, return table objects. - :return: dict of tables with foreign keys referencing self or list of table objects if as_objects=True - """ - nodes = dict((next(iter(self.connection.dependencies.children(k).items())) if k.isdigit() else (k, v)) - for k, v in self.connection.dependencies.children(self.full_table_name, primary).items()) + primary key attributes are considered. If False, return foreign keys including at least one + secondary attribute. + :param as_objects: if False, return table names. If True, return table objects. + :param foreign_key_info: if True, each element in result also includes foreign key info. + :return: list of children as table names or table objects + with (optional) foreign key information. + """ + get_edge = self.connection.dependencies.children + nodes = [next(iter(get_edge(name).items())) if name.isdigit() else (name, props) + for name, props in get_edge(self.full_table_name, primary).items()] if as_objects: - nodes = [FreeTable(self.connection, c) for c in nodes] + nodes = [(FreeTable(self.connection, name), props) for name, props in nodes] + if not foreign_key_info: + nodes = [name for name, props in nodes] return nodes def descendants(self, as_objects=False): - nodes = [node for node in self.connection.dependencies.descendants(self.full_table_name) if not node.isdigit()] - if as_objects: - nodes = [FreeTable(self.connection, c) for c in nodes] - return nodes + """ + :param as_objects: False - a list of table names; True - a list of table objects. + :return: list of tables descendants in topological order. + """ + return [FreeTable(self.connection, node) if as_objects else node + for node in self.connection.dependencies.descendants(self.full_table_name) if not node.isdigit()] + + def ancestors(self, as_objects=False): + """ + :param as_objects: False - a list of table names; True - a list of table objects. + :return: list of tables ancestors in topological order. + """ + return [FreeTable(self.connection, node) if as_objects else node + for node in self.connection.dependencies.ancestors(self.full_table_name) if not node.isdigit()] def parts(self, as_objects=False): """ @@ -161,15 +183,7 @@ def parts(self, as_objects=False): """ nodes = [node for node in self.connection.dependencies.nodes if not node.isdigit() and node.startswith(self.full_table_name[:-1] + '__')] - if as_objects: - nodes = [FreeTable(self.connection, c) for c in nodes] - return nodes - - def ancestors(self, as_objects=False): - nodes = [node for node in self.connection.dependencies.ancestors(self.full_table_name) if not node.isdigit()] - if as_objects: - nodes = [FreeTable(self.connection, c) for c in nodes] - return nodes + return [FreeTable(self.connection, c) for c in nodes] if as_objects else nodes @property def is_declared(self): @@ -242,7 +256,6 @@ def insert1(self, row, **kwargs): def insert(self, rows, replace=False, skip_duplicates=False, ignore_extra_fields=False, allow_direct_insert=None): """ Insert a collection of rows. - :param rows: An iterable where an element is a numpy record, a dict-like object, a pandas.DataFrame, a sequence, or a query expression with the same heading as table self. :param replace: If True, replaces the existing tuple. @@ -250,13 +263,11 @@ def insert(self, rows, replace=False, skip_duplicates=False, ignore_extra_fields :param ignore_extra_fields: If False, fields that are not in the heading raise error. :param allow_direct_insert: applies only in auto-populated tables. If False (default), insert are allowed only from inside the make callback. - Example:: >>> relation.insert([ >>> dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"), >>> dict(subject_id=8, species="mouse", date_of_birth="2014-09-02")]) """ - if isinstance(rows, pandas.DataFrame): # drop 'extra' synthetic index for 1-field index case - # frames with more advanced indices should be prepared by user. @@ -316,7 +327,7 @@ def delete_quick(self, get_count=False): Deletes the table without cascading and without user prompt. If this table has populated dependent tables, this will fail. """ - query = 'DELETE FROM ' + self.full_table_name + self.where_clause + query = 'DELETE FROM ' + self.full_table_name + self.where_clause() self.connection.query(query) count = self.connection.query("SELECT ROW_COUNT()").fetchone()[0] if get_count else None self._log(query[:255]) @@ -459,7 +470,7 @@ def describe(self, context=None, printout=True): del frame if self.full_table_name not in self.connection.dependencies: self.connection.dependencies.load() - parents = self.parents() + parents = self.parents(foreign_key_info=True) in_key = True definition = ('# ' + self.heading.table_status['comment'] + '\n' if self.heading.table_status['comment'] else '') @@ -472,11 +483,10 @@ def describe(self, context=None, printout=True): in_key = False attributes_thus_far.add(attr.name) do_include = True - for parent_name, fk_props in list(parents.items()): # need list() to force a copy + for parent_name, fk_props in parents: if attr.name in fk_props['attr_map']: do_include = False if attributes_thus_far.issuperset(fk_props['attr_map']): - parents.pop(parent_name) # foreign key properties try: index_props = indexes.pop(tuple(fk_props['attr_map'])) @@ -486,19 +496,19 @@ def describe(self, context=None, printout=True): index_props = [k for k, v in index_props.items() if v] index_props = ' [{}]'.format(', '.join(index_props)) if index_props else '' - if not parent_name.isdigit(): + if not fk_props['aliased']: # simple foreign key definition += '->{props} {class_name}\n'.format( props=index_props, class_name=lookup_class_name(parent_name, context) or parent_name) else: # projected foreign key - parent_name = list(self.connection.dependencies.in_edges(parent_name))[0][0] - lst = [(attr, ref) for attr, ref in fk_props['attr_map'].items() if ref != attr] definition += '->{props} {class_name}.proj({proj_list})\n'.format( props=index_props, class_name=lookup_class_name(parent_name, context) or parent_name, - proj_list=','.join('{}="{}"'.format(a, b) for a, b in lst)) + proj_list=','.join( + '{}="{}"'.format(attr, ref) + for attr, ref in fk_props['attr_map'].items() if ref != attr)) attributes_declared.update(fk_props['attr_map']) if do_include: attributes_declared.add(attr.name) @@ -557,7 +567,7 @@ def _update(self, attrname, value=None): full_table_name=self.from_clause(), attrname=attrname, placeholder=placeholder, - where_clause=self.where_clause) + where_clause=self.where_clause()) self.connection.query(command, args=(value, ) if value is not None else ()) # --- private helper functions ---- diff --git a/datajoint/user_tables.py b/datajoint/user_tables.py index 24354ed81..e9cf68fb7 100644 --- a/datajoint/user_tables.py +++ b/datajoint/user_tables.py @@ -14,6 +14,7 @@ supported_class_attrs = { 'key_source', 'describe', 'alter', 'heading', 'populate', 'progress', 'primary_key', 'proj', 'aggr', 'join', 'fetch', 'fetch1', 'head', 'tail', + 'descendants', 'ancestors', 'parts', 'parents', 'children', 'insert', 'insert1', 'update1', 'drop', 'drop_quick', 'delete', 'delete_quick'} diff --git a/datajoint/utils.py b/datajoint/utils.py index 8ad6f4b48..a20f26059 100644 --- a/datajoint/utils.py +++ b/datajoint/utils.py @@ -45,13 +45,13 @@ def to_camel_case(s): :param s: string in under_score notation :returns: string in CamelCase notation Example: - >>> to_camel_case("table_name") # yields "TableName" + >>> to_camel_case("table_name") # returns "TableName" """ def to_upper(match): return match.group(0)[-1].upper() - return re.sub('(^|[_\W])+[a-zA-Z]', to_upper, s) + return re.sub(r'(^|[_\W])+[a-zA-Z]', to_upper, s) def from_camel_case(s): diff --git a/datajoint/version.py b/datajoint/version.py index 8b00db41c..fca233bf7 100644 --- a/datajoint/version.py +++ b/datajoint/version.py @@ -1,3 +1,3 @@ -__version__ = "0.13.dev1" +__version__ = "0.13.dev2" assert len(__version__) <= 10 # The log table limits version to the 10 characters diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index b98ec4602..d741434dd 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,4 +1,4 @@ -0.13.0 -- Dec 15, 2020 +0.13.0 -- Jan 11, 2020 ---------------------- * Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484). PR #754 * Re-implement cascading deletes for better performance. PR #839. @@ -6,15 +6,15 @@ * Python datatypes are now enabled by default in blobs (#761). PR #785 * Added permissive join and restriction operators `@` and `^` (#785) PR #754 -0.12.8 -- Nov 30, 2020 +0.12.8 -- Dec 22, 2020 ---------------------- -* table.children, .parents, .descendents, and ancestors optionally return queryable objects. PR #833 - -0.12.7 -- Oct 27, 2020 ----------------------- -* Fix case sensitivity issues to adapt to MySQL 8+. PR #819 -* Fix pymysql regression bug (#814) PR #816 -* Adapted attribute types now have dtype=object in all recarray results. PR #811 +* table.children, .parents, .descendents, and ancestors can return queryable objects. PR #833 +* Load dependencies before querying dependencies. (#179) PR #833 +* Fix display of part tables in `schema.save`. (#821) PR #833 +* Add `schema.list_tables`. (#838) PR #844 +* Fix minio new version regression. PR #847 +* Add more S3 logging for debugging. (#831) PR #832 +* Convert testing framework from TravisCI to GitHub Actions (#841) PR #840 0.12.7 -- Oct 27, 2020 ---------------------- diff --git a/requirements.txt b/requirements.txt index 67546bf5f..628e14e6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ pandas tqdm networkx pydot -minio<7.0.0 +minio>=7.0.0 matplotlib \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index bb0b814c4..6d578f95d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -136,8 +136,9 @@ def setup_package(): region = "us-east-1" try: minioClient.make_bucket(S3_MIGRATE_BUCKET, location=region) - except minio.error.BucketAlreadyOwnedByYou: - pass + except minio.error.S3Error as e: + if e.code != 'BucketAlreadyOwnedByYou': + raise e pathlist = Path(source).glob('**/*') for path in pathlist: @@ -149,8 +150,9 @@ def setup_package(): # Add S3 try: minioClient.make_bucket(S3_CONN_INFO['bucket'], location=region) - except minio.error.BucketAlreadyOwnedByYou: - pass + except minio.error.S3Error as e: + if e.code != 'BucketAlreadyOwnedByYou': + raise e # Add old File Content try: @@ -179,14 +181,14 @@ def teardown_package(): remove("dj_local_conf.json") # Remove old S3 - objs = list(minioClient.list_objects_v2( + objs = list(minioClient.list_objects( S3_MIGRATE_BUCKET, recursive=True)) objs = [minioClient.remove_object(S3_MIGRATE_BUCKET, o.object_name.encode('utf-8')) for o in objs] minioClient.remove_bucket(S3_MIGRATE_BUCKET) # Remove S3 - objs = list(minioClient.list_objects_v2(S3_CONN_INFO['bucket'], recursive=True)) + objs = list(minioClient.list_objects(S3_CONN_INFO['bucket'], recursive=True)) objs = [minioClient.remove_object(S3_CONN_INFO['bucket'], o.object_name.encode('utf-8')) for o in objs] minioClient.remove_bucket(S3_CONN_INFO['bucket']) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index ed986b94c..2b7687bb5 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,10 +1,32 @@ -from nose.tools import assert_true, assert_false, assert_equal, assert_list_equal, raises +from nose.tools import assert_true, raises, assert_list_equal from .schema import * +from datajoint.dependencies import unite_master_parts + + +def test_unite_master_parts(): + assert_list_equal(unite_master_parts( + ['`s`.`a`', '`s`.`a__q`', '`s`.`b`', '`s`.`c`', '`s`.`c__q`', '`s`.`b__q`', '`s`.`d`', '`s`.`a__r`']), + ['`s`.`a`', '`s`.`a__q`', '`s`.`a__r`', '`s`.`b`', '`s`.`b__q`', '`s`.`c`', '`s`.`c__q`', '`s`.`d`']) + assert_list_equal(unite_master_parts( + [ + '`cells`.`cell_analysis_method`', + '`cells`.`cell_analysis_method_task_type`', + '`cells`.`cell_analysis_method_users`', + '`cells`.`favorite_selection`', + '`cells`.`cell_analysis_method__cell_selection_params`', + '`cells`.`cell_analysis_method__field_detect_params`']), + [ + '`cells`.`cell_analysis_method`', + '`cells`.`cell_analysis_method__cell_selection_params`', + '`cells`.`cell_analysis_method__field_detect_params`', + '`cells`.`cell_analysis_method_task_type`', + '`cells`.`cell_analysis_method_users`', + '`cells`.`favorite_selection`' + ]) def test_nullable_dependency(): """test nullable unique foreign key""" - # Thing C has a nullable dependency on B whose primary key is composite a = ThingA() b = ThingB() diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 8c8d50549..c72c3c896 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -201,7 +201,7 @@ def test_fetch1_step3(self): self.lang.fetch1('name') def test_decimal(self): - """Tests that decimal fields are correctly fetched and used in restrictions, see issue #334""" + """ Tests that decimal fields are correctly fetched and used in restrictions, see issue #334""" rel = schema.DecimalPrimaryKey() rel.insert1([decimal.Decimal('3.1415926')]) keys = rel.fetch() diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 719ae7129..fbf023bb0 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,4 +1,4 @@ -from nose.tools import assert_equal, assert_false, assert_true, raises +from nose.tools import assert_equal, assert_false, assert_true from datajoint.declare import declare from . import schema_advanced diff --git a/tests/test_schema.py b/tests/test_schema.py index 98ec3e7f5..2d868b60d 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -4,6 +4,7 @@ from . import schema from . import schema_empty from . import PREFIX, CONN_INFO +from .schema_simple import schema as schema_simple def relation_selector(attr): @@ -105,6 +106,11 @@ class Unit(dj.Part): test_schema.drop() +def test_list_tables(): + assert(['#a', '#argmax_test', '#data_a', '#data_b', '#i_j', '#j_i', '#l',\ + '#t_test_update', '__b', '__b__c', '__d', '__e', '__e__f', 'f',\ + 'reserved_word'] == schema_simple.list_tables()) + def test_schema_save(): assert_true("class Experiment(dj.Imported)" in schema.schema.code) diff --git a/tests/test_university.py b/tests/test_university.py index 3dd9ce1ba..323889e34 100644 --- a/tests/test_university.py +++ b/tests/test_university.py @@ -1,4 +1,4 @@ -from nose.tools import assert_true, assert_list_equal, assert_set_equal, assert_equal +from nose.tools import assert_true, assert_list_equal, assert_false from .schema_university import * import hashlib @@ -83,7 +83,25 @@ def test_aggr(): assert_true(len(avg_grade_per_course) == 45) # GPA - student_gpa = Student.aggr(Course * Grade * LetterGrade, gpa='round(sum(points*credits)/sum(credits), 2)') + student_gpa = Student.aggr( + Course * Grade * LetterGrade, + gpa='round(sum(points*credits)/sum(credits), 2)') gpa = student_gpa.fetch('gpa') assert_true(len(gpa) == 261) assert_true(2 < gpa.mean() < 3) + + # Sections in biology department with zero students in them + section = (Section & {"dept": "BIOL"}).aggr( + Enroll, n='count(student_id)', keep_all_rows=True) & 'n=0' + assert_true(len(set(section.fetch('dept'))) == 1) + assert_true(len(section) == 17) + assert_true(bool(section)) + + # Test correct use of ellipses in a similar query + section = (Section & {"dept": "BIOL"}).aggr( + Grade, ..., n='count(student_id)', keep_all_rows=True) & 'n>1' + assert_false( + any(name in section.heading.names for name in Grade.heading.secondary_attributes)) + assert_true(len(set(section.fetch('dept'))) == 1) + assert_true(len(section) == 168) + assert_true(bool(section))