From e0d68ce45594225cb6fa5718c3cffcfa9367aad3 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 09:19:41 -0500 Subject: [PATCH 01/23] Add interval and day to reserved key words. --- datajoint/condition.py | 2 +- tests/test_relational_operand.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/datajoint/condition.py b/datajoint/condition.py index 7d921be4f..b7b322ca5 100644 --- a/datajoint/condition.py +++ b/datajoint/condition.py @@ -206,5 +206,5 @@ def extract_column_names(sql_expression): s = re.sub(r"(\b[a-z][a-z_0-9]*)\(", "(", s) remaining_tokens = set(re.findall(r"\b[a-z][a-z_0-9]*\b", s)) # update result removing reserved words - result.update(remaining_tokens - {"is", "in", "between", "like", "and", "or", "null", "not"}) + result.update(remaining_tokens - {"is", "in", "between", "like", "and", "or", "null", "not", "interval", "day"}) return result diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index f37dafb31..00521f53c 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -459,3 +459,13 @@ def test_permissive_join_basic(): def test_permissive_restriction_basic(): """Verify join compatibility check is skipped for restriction""" Child ^ Parent + + @staticmethod + def test_complex_date_restriction(): + # https://github.com/datajoint/datajoint-python/issues/892 + """Test a complex date restriction""" + date1 = datetime.date.today() - datetime.timedelta(days=15) + F.insert([dict(id=100, date=date1)]) + q = F & 'date between curdate() - interval 30 day and curdate()' + assert len(q) == 1 + q.delete() From edd2961eb048dab611c1342bf052c7c31ea31656 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 09:49:18 -0500 Subject: [PATCH 02/23] Fix styling and add to MySQL reserved words. --- datajoint/condition.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datajoint/condition.py b/datajoint/condition.py index b7b322ca5..b1e130530 100644 --- a/datajoint/condition.py +++ b/datajoint/condition.py @@ -206,5 +206,7 @@ def extract_column_names(sql_expression): s = re.sub(r"(\b[a-z][a-z_0-9]*)\(", "(", s) remaining_tokens = set(re.findall(r"\b[a-z][a-z_0-9]*\b", s)) # update result removing reserved words - result.update(remaining_tokens - {"is", "in", "between", "like", "and", "or", "null", "not", "interval", "day"}) + result.update(remaining_tokens - {"is", "in", "between", "like", "and", "or", "null", + "not", "interval", "day" + }) return result From 2045fc82aaf9cce682a138f20a45fc2614292ce9 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 12:57:08 -0500 Subject: [PATCH 03/23] Add erd part table parsing test. --- tests/schema_simple.py | 20 ++++++++++++++++++++ tests/test_erd.py | 11 ++++++++--- tests/test_relational_operand.py | 9 +++++---- 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/schema_simple.py b/tests/schema_simple.py index c7aebaa45..c4ec45e00 100644 --- a/tests/schema_simple.py +++ b/tests/schema_simple.py @@ -7,6 +7,7 @@ from . import PREFIX, CONN_INFO import numpy as np +from datetime import date, timedelta schema = dj.Schema(PREFIX + '_relational', locals(), connection=dj.conn(**CONN_INFO)) @@ -195,3 +196,22 @@ class ReservedWord(dj.Manual): int : int select : varchar(25) """ + + +@schema +class OutfitLaunch(dj.Lookup): + definition = """ + # Monthly released designer outfits + release_id: int + --- + day: date + """ + contents = [(0, date.today() - timedelta(days=15))] + + class OutfitPiece(dj.Part, dj.Lookup): + definition = """ + # Outfit piece associated with outfit + -> OutfitLaunch + piece: varchar(20) + """ + contents = [(0, 'jeans'), (0, 'sneakers'), (0, 'polo')] diff --git a/tests/test_erd.py b/tests/test_erd.py index 0939ca254..6c4ae24b7 100644 --- a/tests/test_erd.py +++ b/tests/test_erd.py @@ -1,6 +1,6 @@ from nose.tools import assert_false, assert_true import datajoint as dj -from .schema_simple import A, B, D, E, L, schema +from .schema_simple import A, B, D, E, L, schema, OutfitLaunch from . import schema_advanced namespace = locals() @@ -64,5 +64,10 @@ def test_make_image(): img = erd.make_image() assert_true(img.ndim == 3 and img.shape[2] in (3, 4)) - - + @staticmethod + def test_part_table_parsing(): + # https://github.com/datajoint/datajoint-python/issues/882 + erd = dj.Di(schema) + graph = erd._make_graph() + assert 'OutfitLaunch' in graph.nodes() + assert 'OutfitLaunch.OutfitPiece' in graph.nodes() diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index 00521f53c..c463b2899 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -7,7 +7,8 @@ from nose.tools import assert_equal, assert_false, assert_true, raises, assert_set_equal, assert_list_equal import datajoint as dj -from .schema_simple import A, B, D, E, F, L, DataA, DataB, TTestUpdate, IJ, JI, ReservedWord +from .schema_simple import (A, B, D, E, F, L, DataA, DataB, TTestUpdate, IJ, JI, + ReservedWord, OutfitLaunch) from .schema import Experiment, TTest3, Trial, Ephys, Child, Parent @@ -464,8 +465,8 @@ def test_permissive_restriction_basic(): def test_complex_date_restriction(): # https://github.com/datajoint/datajoint-python/issues/892 """Test a complex date restriction""" - date1 = datetime.date.today() - datetime.timedelta(days=15) - F.insert([dict(id=100, date=date1)]) - q = F & 'date between curdate() - interval 30 day and curdate()' + q = OutfitLaunch & 'day between curdate() - interval 30 day and curdate()' + assert len(q) == 1 + q = OutfitLaunch & '`day` between curdate() - interval 30 day and curdate()' assert len(q) == 1 q.delete() From c21f7225d6878163ea40a6f3c4197a148cd66f35 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 13:18:05 -0500 Subject: [PATCH 04/23] Fix style in list_tables test. --- tests/__init__.py | 2 +- tests/test_schema.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 1a48a3a91..8efad423c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -20,7 +20,7 @@ __author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman' # turn on verbose logging -logging.basicConfig(level=logging.DEBUG) +# logging.basicConfig(level=logging.DEBUG) __all__ = ['__author__', 'PREFIX', 'CONN_INFO'] diff --git a/tests/test_schema.py b/tests/test_schema.py index f7e19356d..e5bfc04a7 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -119,10 +119,12 @@ class Unit(dj.Part): test_schema.drop() + def test_list_tables(): - assert(['#a', '#argmax_test', '#data_a', '#data_b', '#i_j', '#j_i', '#l',\ - '#t_test_update', '__b', '__b__c', '__d', '__e', '__e__f', 'f',\ - 'reserved_word'] == schema_simple.list_tables()) + print(schema_simple.list_tables()) + assert(['#a', '#argmax_test', '#data_a', '#data_b', '#i_j', '#j_i', '#l', + '#t_test_update', '__b', '__b__c', '__d', '__e', '__e__f', 'f', + 'reserved_word'] == schema_simple.list_tables()) def test_schema_save(): From 7802afb812a478ae453b9a327a27dbf562029c20 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 13:28:11 -0500 Subject: [PATCH 05/23] Add debug statements. --- datajoint/diagram.py | 2 ++ tests/test_erd.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/datajoint/diagram.py b/datajoint/diagram.py index dd48e7b17..1302e6d42 100644 --- a/datajoint/diagram.py +++ b/datajoint/diagram.py @@ -219,6 +219,7 @@ def _make_graph(self): """ Make the self.graph - a graph object ready for drawing """ + import json # mark "distinguished" tables, i.e. those that introduce new primary key attributes for name in self.nodes_to_show: foreign_attributes = set( @@ -234,6 +235,7 @@ def _make_graph(self): nx.set_node_attributes(graph, name='node_type', values={n: _get_tier(n) for n in graph}) # relabel nodes to class names mapping = {node: lookup_class_name(node, self.context) or node for node in graph.nodes()} + print(f'mapping: {json.dumps(mapping, indent=4)}') new_names = [mapping.values()] if len(new_names) > len(set(new_names)): raise DataJointError('Some classes have identical names. The Diagram cannot be plotted.') diff --git a/tests/test_erd.py b/tests/test_erd.py index 6c4ae24b7..dea7fcc73 100644 --- a/tests/test_erd.py +++ b/tests/test_erd.py @@ -69,5 +69,7 @@ def test_part_table_parsing(): # https://github.com/datajoint/datajoint-python/issues/882 erd = dj.Di(schema) graph = erd._make_graph() + # print(f'nodes: {erd.nodes_to_show}') + # print(f'graph nodes: {graph.nodes()}') assert 'OutfitLaunch' in graph.nodes() assert 'OutfitLaunch.OutfitPiece' in graph.nodes() From 2b49facc476a0b45c3ef1b7b8b8aaad633fecc98 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 13:54:50 -0500 Subject: [PATCH 06/23] Add debug of parts. --- datajoint/diagram.py | 1 + datajoint/table.py | 1 + 2 files changed, 2 insertions(+) diff --git a/datajoint/diagram.py b/datajoint/diagram.py index 1302e6d42..7d194c6da 100644 --- a/datajoint/diagram.py +++ b/datajoint/diagram.py @@ -234,6 +234,7 @@ def _make_graph(self): graph = nx.DiGraph(nx.DiGraph(self).subgraph(nodes)) nx.set_node_attributes(graph, name='node_type', values={n: _get_tier(n) for n in graph}) # relabel nodes to class names + # print(f'context: {self.context}') mapping = {node: lookup_class_name(node, self.context) or node for node in graph.nodes()} print(f'mapping: {json.dumps(mapping, indent=4)}') new_names = [mapping.values()] diff --git a/datajoint/table.py b/datajoint/table.py index d79c07a75..9e9bd676b 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -721,6 +721,7 @@ def lookup_class_name(name, context, depth=3): return '.'.join([node['context_name'], member_name]).lstrip('.') try: # look for part tables parts = member._ordered_class_members + print(f'parts: {parts}') except AttributeError: pass # not a UserTable -- cannot have part tables. else: From 0008645b341b13ca24b223747300dc51cc2b7775 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 14:27:39 -0500 Subject: [PATCH 07/23] Remove usage of _ordered_class_members, ordered_dir since upgrading to py36(https://www.python.org/dev/peps/pep-0520/ --- datajoint/schemas.py | 18 +----------------- datajoint/table.py | 2 +- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/datajoint/schemas.py b/datajoint/schemas.py index 2c050a915..84a8f4513 100644 --- a/datajoint/schemas.py +++ b/datajoint/schemas.py @@ -19,22 +19,6 @@ logger = logging.getLogger(__name__) -def ordered_dir(class_): - """ - List (most) attributes of the class including inherited ones, similar to `dir` build-in function, - but respects order of attribute declaration as much as possible. - This becomes unnecessary in Python 3.6+ as dicts became ordered. - :param class_: class to list members for - :return: a list of attributes declared in class_ and its superclasses - """ - attr_list = list() - for c in reversed(class_.mro()): - attr_list.extend(e for e in ( - c._ordered_class_members if hasattr(c, '_ordered_class_members') else c.__dict__) - if e not in attr_list) - return attr_list - - class Schema: """ A schema object is a decorator for UserTable classes that binds them to their database. @@ -158,7 +142,7 @@ def _decorate_master(self, cls, context): """ self._decorate_table(cls, context=dict(context, self=cls, **{cls.__name__: cls})) # Process part tables - for part in ordered_dir(cls): + for part in dir(cls.__dict__): if part[0].isupper(): part = getattr(cls, part) if inspect.isclass(part) and issubclass(part, Part): diff --git a/datajoint/table.py b/datajoint/table.py index 9e9bd676b..5a9969dd7 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -720,7 +720,7 @@ def lookup_class_name(name, context, depth=3): if member.full_table_name == name: # found it! return '.'.join([node['context_name'], member_name]).lstrip('.') try: # look for part tables - parts = member._ordered_class_members + parts = member.__dict__ print(f'parts: {parts}') except AttributeError: pass # not a UserTable -- cannot have part tables. From 65b4c46c79403a7c4c4fa026be16cfa727637ef7 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 14:40:09 -0500 Subject: [PATCH 08/23] Clean up. --- datajoint/diagram.py | 26 +++++++++++++++----------- datajoint/schemas.py | 16 +++++++++++++++- datajoint/table.py | 1 - tests/__init__.py | 2 +- tests/test_erd.py | 2 -- tests/test_schema.py | 5 +++-- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/datajoint/diagram.py b/datajoint/diagram.py index 7d194c6da..9c6df6775 100644 --- a/datajoint/diagram.py +++ b/datajoint/diagram.py @@ -219,27 +219,31 @@ def _make_graph(self): """ Make the self.graph - a graph object ready for drawing """ - import json - # mark "distinguished" tables, i.e. those that introduce new primary key attributes + # mark "distinguished" tables i.e. ones introduce new primary key attributes for name in self.nodes_to_show: foreign_attributes = set( - attr for p in self.in_edges(name, data=True) for attr in p[2]['attr_map'] if p[2]['primary']) + attr for p in self.in_edges(name, data=True) + for attr in p[2]['attr_map'] if p[2]['primary']) self.nodes[name]['distinguished'] = ( - 'primary_key' in self.nodes[name] and foreign_attributes < self.nodes[name]['primary_key']) + 'primary_key' in self.nodes[name] and + foreign_attributes < self.nodes[name]['primary_key']) # include aliased nodes that are sandwiched between two displayed nodes - gaps = set(nx.algorithms.boundary.node_boundary(self, self.nodes_to_show)).intersection( - nx.algorithms.boundary.node_boundary(nx.DiGraph(self).reverse(), self.nodes_to_show)) + gaps = set(nx.algorithms.boundary.node_boundary( + self, self.nodes_to_show)).intersection( + nx.algorithms.boundary.node_boundary(nx.DiGraph(self).reverse(), + self.nodes_to_show)) nodes = self.nodes_to_show.union(a for a in gaps if a.isdigit) # construct subgraph and rename nodes to class names graph = nx.DiGraph(nx.DiGraph(self).subgraph(nodes)) - nx.set_node_attributes(graph, name='node_type', values={n: _get_tier(n) for n in graph}) + nx.set_node_attributes(graph, name='node_type', values={n: _get_tier(n) + for n in graph}) # relabel nodes to class names - # print(f'context: {self.context}') - mapping = {node: lookup_class_name(node, self.context) or node for node in graph.nodes()} - print(f'mapping: {json.dumps(mapping, indent=4)}') + mapping = {node: lookup_class_name(node, self.context) or node + for node in graph.nodes()} new_names = [mapping.values()] if len(new_names) > len(set(new_names)): - raise DataJointError('Some classes have identical names. The Diagram cannot be plotted.') + raise DataJointError( + 'Some classes have identical names. The Diagram cannot be plotted.') nx.relabel_nodes(graph, mapping, copy=False) return graph diff --git a/datajoint/schemas.py b/datajoint/schemas.py index 84a8f4513..865b18500 100644 --- a/datajoint/schemas.py +++ b/datajoint/schemas.py @@ -19,6 +19,20 @@ logger = logging.getLogger(__name__) +def ordered_dir(class_): + """ + List (most) attributes of the class including inherited ones, similar to `dir` build-in function, + but respects order of attribute declaration as much as possible. + This becomes unnecessary in Python 3.6+ as dicts became ordered. + :param class_: class to list members for + :return: a list of attributes declared in class_ and its superclasses + """ + attr_list = list() + for c in reversed(class_.mro()): + attr_list.extend(e for e in c.__dict__ if e not in attr_list) + return attr_list + + class Schema: """ A schema object is a decorator for UserTable classes that binds them to their database. @@ -142,7 +156,7 @@ def _decorate_master(self, cls, context): """ self._decorate_table(cls, context=dict(context, self=cls, **{cls.__name__: cls})) # Process part tables - for part in dir(cls.__dict__): + for part in ordered_dir(cls): if part[0].isupper(): part = getattr(cls, part) if inspect.isclass(part) and issubclass(part, Part): diff --git a/datajoint/table.py b/datajoint/table.py index 5a9969dd7..043c9fa57 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -721,7 +721,6 @@ def lookup_class_name(name, context, depth=3): return '.'.join([node['context_name'], member_name]).lstrip('.') try: # look for part tables parts = member.__dict__ - print(f'parts: {parts}') except AttributeError: pass # not a UserTable -- cannot have part tables. else: diff --git a/tests/__init__.py b/tests/__init__.py index 8efad423c..1a48a3a91 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -20,7 +20,7 @@ __author__ = 'Edgar Walker, Fabian Sinz, Dimitri Yatsenko, Raphael Guzman' # turn on verbose logging -# logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.DEBUG) __all__ = ['__author__', 'PREFIX', 'CONN_INFO'] diff --git a/tests/test_erd.py b/tests/test_erd.py index dea7fcc73..6c4ae24b7 100644 --- a/tests/test_erd.py +++ b/tests/test_erd.py @@ -69,7 +69,5 @@ def test_part_table_parsing(): # https://github.com/datajoint/datajoint-python/issues/882 erd = dj.Di(schema) graph = erd._make_graph() - # print(f'nodes: {erd.nodes_to_show}') - # print(f'graph nodes: {graph.nodes()}') assert 'OutfitLaunch' in graph.nodes() assert 'OutfitLaunch.OutfitPiece' in graph.nodes() diff --git a/tests/test_schema.py b/tests/test_schema.py index e5bfc04a7..ba7c35920 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -123,8 +123,9 @@ class Unit(dj.Part): def test_list_tables(): print(schema_simple.list_tables()) assert(['#a', '#argmax_test', '#data_a', '#data_b', '#i_j', '#j_i', '#l', - '#t_test_update', '__b', '__b__c', '__d', '__e', '__e__f', 'f', - 'reserved_word'] == schema_simple.list_tables()) + '#outfit_launch', '#outfit_launch__outfit_piece', '#t_test_update', '__b', + '__b__c', '__d', '__e', '__e__f', 'f', 'reserved_word' + ] == schema_simple.list_tables()) def test_schema_save(): From 84c5c9d668a7f033728e429b6c49ca094ccdfc0d Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 14:45:38 -0500 Subject: [PATCH 09/23] Clean up2. --- datajoint/table.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/datajoint/table.py b/datajoint/table.py index 043c9fa57..9a322aafb 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -724,10 +724,14 @@ def lookup_class_name(name, context, depth=3): except AttributeError: pass # not a UserTable -- cannot have part tables. else: - for part in (getattr(member, p) for p in parts if p[0].isupper() and hasattr(member, p)): - if inspect.isclass(part) and issubclass(part, Table) and part.full_table_name == name: - return '.'.join([node['context_name'], member_name, part.__name__]).lstrip('.') - elif node['depth'] > 0 and inspect.ismodule(member) and member.__name__ != 'datajoint': + for part in (getattr(member, p) for p in parts + if p[0].isupper() and hasattr(member, p)): + if inspect.isclass(part) and issubclass(part, Table) and \ + part.full_table_name == name: + return '.'.join([node['context_name'], + member_name, part.__name__]).lstrip('.') + elif node['depth'] > 0 and inspect.ismodule(member) and \ + member.__name__ != 'datajoint': try: nodes.append( dict(context=dict(inspect.getmembers(member)), From 4c2613caf44b6451900cbcb637516c419685bb5c Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 15:26:17 -0500 Subject: [PATCH 10/23] Add topological sort to list_tables and additional reserved MySQL keywords. --- datajoint/condition.py | 3 ++- datajoint/schemas.py | 6 +++--- tests/test_schema.py | 9 ++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/datajoint/condition.py b/datajoint/condition.py index b1e130530..5459eb70a 100644 --- a/datajoint/condition.py +++ b/datajoint/condition.py @@ -207,6 +207,7 @@ def extract_column_names(sql_expression): remaining_tokens = set(re.findall(r"\b[a-z][a-z_0-9]*\b", s)) # update result removing reserved words result.update(remaining_tokens - {"is", "in", "between", "like", "and", "or", "null", - "not", "interval", "day" + "not", "interval", "second", "minute", "hour", "day", + "month", "week", "year" }) return result diff --git a/datajoint/schemas.py b/datajoint/schemas.py index 865b18500..d108caef2 100644 --- a/datajoint/schemas.py +++ b/datajoint/schemas.py @@ -372,9 +372,9 @@ def list_tables(self): as ~logs and ~job :return: A list of table names from the database schema. """ - return [table_name for (table_name,) in self.connection.query(""" - SELECT table_name FROM information_schema.tables - WHERE table_schema = %s and table_name NOT LIKE '~%%'""", args=(self.database,))] + return [t for d, t in (full_t.replace('`', '').split('.') + for full_t in Diagram(self).topological_sort()) + if d == self.database] class VirtualModule(types.ModuleType): diff --git a/tests/test_schema.py b/tests/test_schema.py index ba7c35920..bc025960e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -121,11 +121,10 @@ class Unit(dj.Part): def test_list_tables(): - print(schema_simple.list_tables()) - assert(['#a', '#argmax_test', '#data_a', '#data_b', '#i_j', '#j_i', '#l', - '#outfit_launch', '#outfit_launch__outfit_piece', '#t_test_update', '__b', - '__b__c', '__d', '__e', '__e__f', 'f', 'reserved_word' - ] == schema_simple.list_tables()) + assert(set(['reserved_word', '#l', '#a', '__d', '__b', '__b__c', '__e', '__e__f', + '#outfit_launch', '#outfit_launch__outfit_piece', '#i_j', '#j_i', + '#t_test_update', '#data_a', '#data_b', 'f', '#argmax_test' + ]) == set(schema_simple.list_tables())) def test_schema_save(): From 202ecd34391df404a6cd7efff8ebeba136a3b415 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 18:17:07 -0500 Subject: [PATCH 11/23] Add test for uppercase schema. --- tests/test_schema.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_schema.py b/tests/test_schema.py index bc025960e..0af9e7d01 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -121,6 +121,7 @@ class Unit(dj.Part): def test_list_tables(): + # https://github.com/datajoint/datajoint-python/issues/838 assert(set(['reserved_word', '#l', '#a', '__d', '__b', '__b__c', '__e', '__e__f', '#outfit_launch', '#outfit_launch__outfit_piece', '#i_j', '#j_i', '#t_test_update', '#data_a', '#data_b', 'f', '#argmax_test' @@ -130,3 +131,28 @@ def test_list_tables(): def test_schema_save(): assert_true("class Experiment(dj.Imported)" in schema.schema.code) assert_true("class Experiment(dj.Imported)" in schema_empty.schema.code) + + +def test_uppercase_schema(): + # https://github.com/datajoint/datajoint-python/issues/564 + schema1 = dj.schema('Upper_Schema') + + @schema1 + class Subject(dj.Manual): + definition = """ + name: varchar(32) + """ + + Upper_Schema = dj.VirtualModule('Upper_Schema', 'Upper_Schema') + + schema2 = dj.schema('schema_b') + + @schema2 + class Recording(dj.Manual): + definition = """ + -> Upper_Schema.Subject + id: smallint + """ + + schema2.drop() + schema1.drop() From 33e95dbef14fba0074096a02d2734acff0625b84 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 18:25:24 -0500 Subject: [PATCH 12/23] Update test for uppercased schema. --- tests/test_schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index 0af9e7d01..f59cdd121 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -135,7 +135,7 @@ def test_schema_save(): def test_uppercase_schema(): # https://github.com/datajoint/datajoint-python/issues/564 - schema1 = dj.schema('Upper_Schema') + schema1 = dj.Schema('Schema_A') @schema1 class Subject(dj.Manual): @@ -143,14 +143,14 @@ class Subject(dj.Manual): name: varchar(32) """ - Upper_Schema = dj.VirtualModule('Upper_Schema', 'Upper_Schema') + Schema_A = dj.VirtualModule('Schema_A', 'Schema_A') - schema2 = dj.schema('schema_b') + schema2 = dj.Schema('schema_b') @schema2 class Recording(dj.Manual): definition = """ - -> Upper_Schema.Subject + -> Schema_A.Subject id: smallint """ From 2547748b8109bae83bb88d7d190395f1771da0c0 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 18:33:52 -0500 Subject: [PATCH 13/23] Use root user for uppercase schema test due to permissions. --- tests/test_schema.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_schema.py b/tests/test_schema.py index f59cdd121..42d4e0c0e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -3,7 +3,7 @@ from inspect import getmembers from . import schema from . import schema_empty -from . import PREFIX, CONN_INFO +from . import PREFIX, CONN_INFO, CONN_INFO_ROOT from .schema_simple import schema as schema_simple @@ -135,6 +135,7 @@ def test_schema_save(): def test_uppercase_schema(): # https://github.com/datajoint/datajoint-python/issues/564 + dj.conn(**CONN_INFO_ROOT, reset=True) schema1 = dj.Schema('Schema_A') @schema1 From f77399125b737c7e934582f6f62629dab26f49c6 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 19:29:44 -0500 Subject: [PATCH 14/23] Allow None to be used in dict restrictions. --- datajoint/condition.py | 40 +++++++++++++++++++++----------- tests/test_relational_operand.py | 10 ++++++++ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/datajoint/condition.py b/datajoint/condition.py index 5459eb70a..fed138cf1 100644 --- a/datajoint/condition.py +++ b/datajoint/condition.py @@ -21,8 +21,9 @@ def __init__(self, operand): class AndList(list): """ - A list of conditions to by applied to a query expression by logical conjunction: the conditions are AND-ed. - All other collections (lists, sets, other entity sets, etc) are applied by logical disjunction (OR). + A list of conditions to by applied to a query expression by logical conjunction: the + conditions are AND-ed. All other collections (lists, sets, other entity sets, etc) are + applied by logical disjunction (OR). Example: expr2 = expr & dj.AndList((cond1, cond2, cond3)) @@ -49,6 +50,7 @@ def assert_join_compatibility(expr1, expr2): the matching attributes in the two expressions must be in the primary key of one or the other expression. Raises an exception if not compatible. + :param expr1: A QueryExpression object :param expr2: A QueryExpression object """ @@ -56,7 +58,8 @@ def assert_join_compatibility(expr1, expr2): for rel in (expr1, expr2): if not isinstance(rel, (U, QueryExpression)): - raise DataJointError('Object %r is not a QueryExpression and cannot be joined.' % rel) + raise DataJointError( + 'Object %r is not a QueryExpression and cannot be joined.' % rel) if not isinstance(expr1, U) and not isinstance(expr2, U): # dj.U is always compatible try: raise DataJointError( @@ -70,9 +73,11 @@ def assert_join_compatibility(expr1, expr2): def make_condition(query_expression, condition, columns): """ Translate the input condition into the equivalent SQL condition (a string) + :param query_expression: a dj.QueryExpression object to apply condition :param condition: any valid restriction object. - :param columns: a set passed by reference to collect all column names used in the condition. + :param columns: a set passed by reference to collect all column names used in the + condition. :return: an SQL condition string or a boolean value. """ from .expression import QueryExpression, Aggregation, U @@ -102,12 +107,13 @@ def prep_value(k, v): # restrict by string if isinstance(condition, str): columns.update(extract_column_names(condition)) - return template % condition.strip().replace("%", "%%") # escape % in strings, see issue #376 + return template % condition.strip().replace("%", "%%") # escape %, see issue #376 # restrict by AndList if isinstance(condition, AndList): # omit all conditions that evaluate to True - items = [item for item in (make_condition(query_expression, cond, columns) for cond in condition) + items = [item for item in (make_condition(query_expression, cond, columns) + for cond in condition) if item is not True] if any(item is False for item in items): return negate # if any item is False, the whole thing is False @@ -123,18 +129,21 @@ def prep_value(k, v): if isinstance(condition, bool): return negate != condition - # restrict by a mapping such as a dict -- convert to an AndList of string equality conditions + # restrict by a mapping/dict -- convert to an AndList of string equality conditions if isinstance(condition, collections.abc.Mapping): common_attributes = set(condition).intersection(query_expression.heading.names) if not common_attributes: return not negate # no matching attributes -> evaluates to True columns.update(common_attributes) return template % ('(' + ') AND ('.join( - '`%s`=%s' % (k, prep_value(k, condition[k])) for k in common_attributes) + ')') + '`%s`%s' % (k, ' IS NULL' if condition[k] is None + else f'={prep_value(k, condition[k])}') + for k in common_attributes) + ')') # restrict by a numpy record -- convert to an AndList of string equality conditions if isinstance(condition, numpy.void): - common_attributes = set(condition.dtype.fields).intersection(query_expression.heading.names) + common_attributes = set(condition.dtype.fields).intersection( + query_expression.heading.names) if not common_attributes: return not negate # no matching attributes -> evaluate to True columns.update(common_attributes) @@ -154,7 +163,8 @@ def prep_value(k, v): if isinstance(condition, QueryExpression): if check_compatibility: assert_join_compatibility(query_expression, condition) - common_attributes = [q for q in condition.heading.names if q in query_expression.heading.names] + common_attributes = [q for q in condition.heading.names + if q in query_expression.heading.names] columns.update(common_attributes) if isinstance(condition, Aggregation): condition = condition.make_subquery() @@ -176,15 +186,17 @@ def prep_value(k, v): except TypeError: raise DataJointError('Invalid restriction type %r' % condition) else: - or_list = [item for item in or_list if item is not False] # ignore all False conditions - if any(item is True for item in or_list): # if any item is True, the whole thing is True + or_list = [item for item in or_list if item is not False] # ignore False conditions + if any(item is True for item in or_list): # if any item is True, entirely True return not negate - return template % ('(%s)' % ' OR '.join(or_list)) if or_list else negate # an empty or list is False + return template % ('(%s)' % ' OR '.join(or_list)) if or_list else negate def extract_column_names(sql_expression): """ - extract all presumed column names from an sql expression such as the WHERE clause, for example. + extract all presumed column names from an sql expression such as the WHERE clause, + for example. + :param sql_expression: a string containing an SQL expression :return: set of extracted column names This may be MySQL-specific for now. diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index c463b2899..daf6e14ab 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -470,3 +470,13 @@ def test_complex_date_restriction(): q = OutfitLaunch & '`day` between curdate() - interval 30 day and curdate()' assert len(q) == 1 q.delete() + + @staticmethod + def test_null_dict_restriction(): + # https://github.com/datajoint/datajoint-python/issues/824 + """Test a restriction for null using dict""" + F.insert([dict(id=5)]) + q = F & 'date is NULL' + assert len(q) == 1 + q = F & dict(date=None) + assert len(q) == 1 From 8c0b8a38aa4ec1bf3baaae526fda433514154dd0 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Thu, 25 Mar 2021 19:40:54 -0500 Subject: [PATCH 15/23] Restrict test to just the created id. --- tests/test_relational_operand.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index daf6e14ab..efcb0be0b 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -476,7 +476,7 @@ def test_null_dict_restriction(): # https://github.com/datajoint/datajoint-python/issues/824 """Test a restriction for null using dict""" F.insert([dict(id=5)]) - q = F & 'date is NULL' + q = F & dj.AndList([dict(id=5), 'date is NULL']) assert len(q) == 1 - q = F & dict(date=None) + q = F & dict(id=5, date=None) assert len(q) == 1 From 0c98e86718b836369d2c9ae7219bc3df9b770883 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Fri, 26 Mar 2021 09:09:30 -0500 Subject: [PATCH 16/23] Update release log and bump version. --- CHANGELOG.md | 20 ++++++++++++++------ datajoint/version.py | 2 +- docs-parts/intro/Releases_lang1.rst | 21 +++++++++++++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6625d1d5b..11e19342f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,21 @@ ## Release notes +### 0.13.1 -- TBD +* Add `None` as an alias for `NULL` in `dict` restrictions (#824) PR #893 +* Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 +* Bugfix - Diagram part tables do not show proper class name (#882) PR #893 +* Bugfix - Error in complex restrictions (#892) PR #893 + ### 0.13.0 -- Mar 24, 2021 -* Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484). PR #754 -* Re-implement cascading deletes for better performance. PR #839. -* Add table method `.update1` to update a row in the table with new values PR #763 -* Python datatypes are now enabled by default in blobs (#761). PR #785 +* Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484, #558). PR #754 +* Re-implement cascading deletes for better performance. PR #839 +* Add support for deferred schema activation to allow for greater modularity. (#834) PR #839 +* Add query caching mechanism for offline development (#550) PR #839 +* Add table method `.update1` to update a row in the table with new values (#867) PR #763, #889 +* Python datatypes are now enabled by default in blobs (#761). PR #859 * Added permissive join and restriction operators `@` and `^` (#785) PR #754 * Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735 -* Add `dj.key_hash` alias to `dj.hash.key_hash` +* Add `dj.key_hash` alias to `dj.hash.key_hash` (#804) PR #862 * Default enable_python_native_blobs to True * Bugfix - Regression error on joins with same attribute name (#857) PR #878 * Bugfix - Error when `fetch1('KEY')` when `dj.config['fetch_format']='frame'` set (#876) PR #880, #878 @@ -15,7 +23,7 @@ * Add deprecation warning for `_update`. PR #889 * Add `purge_query_cache` utility. PR #889 * Add tests for query caching and permissive join and restriction. PR #889 -* Drop support for Python 3.5 +* Drop support for Python 3.5 (#829) PR #861 ### 0.12.9 -- Mar 12, 2021 * Fix bug with fetch1 with `dj.config['fetch_format']="frame"`. (#876) PR #880 diff --git a/datajoint/version.py b/datajoint/version.py index a7571b6c4..403e38347 100644 --- a/datajoint/version.py +++ b/datajoint/version.py @@ -1,3 +1,3 @@ -__version__ = "0.13.0" +__version__ = "0.13.1" assert len(__version__) <= 10 # The log table limits version to the 10 characters diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index 3dc72f2ab..9c661e2b8 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,12 +1,21 @@ +0.13.1 -- TBD +---------------------- +* Add `None` as an alias for `NULL` in `dict` restrictions (#824) PR #893 +* Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 +* Bugfix - Diagram part tables do not show proper class name (#882) PR #893 +* Bugfix - Error in complex restrictions (#892) PR #893 + 0.13.0 -- Mar 24, 2021 ---------------------- -* Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484). PR #754 -* Re-implement cascading deletes for better performance. PR #839. -* Add table method `.update1` to update a row in the table with new values PR #763 -* Python datatypes are now enabled by default in blobs (#761). PR #785 +* Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484, #558). PR #754 +* Re-implement cascading deletes for better performance. PR #839 +* Add support for deferred schema activation to allow for greater modularity. (#834) PR #839 +* Add query caching mechanism for offline development (#550) PR #839 +* Add table method `.update1` to update a row in the table with new values (#867) PR #763, #889 +* Python datatypes are now enabled by default in blobs (#761). PR #859 * Added permissive join and restriction operators `@` and `^` (#785) PR #754 * Support DataJoint datatype and connection plugins (#715, #729) PR 730, #735 -* Add `dj.key_hash` alias to `dj.hash.key_hash` +* Add `dj.key_hash` alias to `dj.hash.key_hash` (#804) PR #862 * Default enable_python_native_blobs to True * Bugfix - Regression error on joins with same attribute name (#857) PR #878 * Bugfix - Error when `fetch1('KEY')` when `dj.config['fetch_format']='frame'` set (#876) PR #880, #878 @@ -14,7 +23,7 @@ * Add deprecation warning for `_update`. PR #889 * Add `purge_query_cache` utility. PR #889 * Add tests for query caching and permissive join and restriction. PR #889 -* Drop support for Python 3.5 +* Drop support for Python 3.5 (#829) PR #861 0.12.9 -- Mar 12, 2021 ---------------------- From ed16dc8d3597cbb422da9bb45cfd8e363348bf44 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Wed, 7 Apr 2021 11:01:42 -0500 Subject: [PATCH 17/23] Incorporate feedback and add test for #898. --- datajoint/diagram.py | 3 +- tests/test_relational_operand.py | 79 +++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/datajoint/diagram.py b/datajoint/diagram.py index 9c6df6775..c4f823035 100644 --- a/datajoint/diagram.py +++ b/datajoint/diagram.py @@ -219,7 +219,8 @@ def _make_graph(self): """ Make the self.graph - a graph object ready for drawing """ - # mark "distinguished" tables i.e. ones introduce new primary key attributes + # mark "distinguished" tables, i.e. those that introduce new primary key + # attributes for name in self.nodes_to_show: foreign_attributes = set( attr for p in self.in_edges(name, data=True) diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index efcb0be0b..f4e66a6c2 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -4,7 +4,8 @@ import datetime import numpy as np -from nose.tools import assert_equal, assert_false, assert_true, raises, assert_set_equal, assert_list_equal +from nose.tools import (assert_equal, assert_false, assert_true, raises, assert_set_equal, + assert_list_equal) import datajoint as dj from .schema_simple import (A, B, D, E, F, L, DataA, DataB, TTestUpdate, IJ, JI, @@ -159,6 +160,76 @@ def test_issue_376(): def test_issue_463(): assert_equal(((A & B) * B).fetch().size, len(A * B)) + @staticmethod + def test_issue_898(): + # https://github.com/datajoint/datajoint-python/issues/898 + schema = dj.schema('djtest_raphael') + + @schema + class Subject(dj.Lookup): + definition = """ + subject_id: varchar(32) + --- + dob : date + sex : enum('M', 'F', 'U') + """ + contents = [ + ('mouse1', '2020-09-01', 'M'), + ('mouse2', '2020-03-19', 'F'), + ('mouse3', '2020-08-23', 'F') + ] + + @schema + class Session(dj.Lookup): + definition = """ + -> Subject + session_start_time: datetime + --- + session_dir='' : varchar(32) + """ + contents = [ + ('mouse1', '2020-12-01 12:32:34', ''), + ('mouse1', '2020-12-02 12:32:34', ''), + ('mouse1', '2020-12-03 12:32:34', ''), + ('mouse1', '2020-12-04 12:32:34', '') + ] + + @schema + class SessionStatus(dj.Lookup): + definition = """ + -> Session + --- + status: enum('in_training', 'trained_1a', 'trained_1b', 'ready4ephys') + """ + contents = [ + ('mouse1', '2020-12-01 12:32:34', 'in_training'), + ('mouse1', '2020-12-02 12:32:34', 'trained_1a'), + ('mouse1', '2020-12-03 12:32:34', 'trained_1b'), + ('mouse1', '2020-12-04 12:32:34', 'ready4ephys'), + ] + + @schema + class SessionDate(dj.Lookup): + definition = """ + -> Subject + session_date: date + """ + contents = [ + ('mouse1', '2020-12-01'), + ('mouse1', '2020-12-02'), + ('mouse1', '2020-12-03'), + ('mouse1', '2020-12-04') + ] + + subjects = Subject.aggr( + SessionStatus & 'status="trained_1a" or status="trained_1b"', + date_trained='min(date(session_start_time))') + + print(f'subjects: {subjects}') + print(f'SessionDate: {SessionDate()}') + print(f'join: {SessionDate * subjects}') + print(f'join query: {(SessionDate * subjects).make_sql()}') + @staticmethod def test_project(): x = A().proj(a='id_a') # rename @@ -467,6 +538,12 @@ def test_complex_date_restriction(): """Test a complex date restriction""" q = OutfitLaunch & 'day between curdate() - interval 30 day and curdate()' assert len(q) == 1 + q = OutfitLaunch & 'day between curdate() - interval 4 week and curdate()' + assert len(q) == 1 + q = OutfitLaunch & 'day between curdate() - interval 1 month and curdate()' + assert len(q) == 1 + q = OutfitLaunch & 'day between curdate() - interval 1 year and curdate()' + assert len(q) == 1 q = OutfitLaunch & '`day` between curdate() - interval 30 day and curdate()' assert len(q) == 1 q.delete() From f71fe382713d49bf1b713d01e90a5c1d986d71ba Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Fri, 9 Apr 2021 08:57:21 -0500 Subject: [PATCH 18/23] Fix join with aggregations. --- datajoint/connection.py | 4 +- datajoint/expression.py | 26 +++++---- tests/schema.py | 60 +++++++++++++++++++++ tests/test_relational_operand.py | 90 +++++++------------------------- 4 files changed, 97 insertions(+), 83 deletions(-) diff --git a/datajoint/connection.py b/datajoint/connection.py index 9db3dcb77..c82ca5b3e 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -203,7 +203,7 @@ def connect(self): self._conn = client.connect( init_command=self.init_fun, sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO," - "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION", + "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY", charset=config['connection.charset'], **{k: v for k, v in self.conn_info.items() if k not in ['ssl_input', 'host_input']}) @@ -211,7 +211,7 @@ def connect(self): self._conn = client.connect( init_command=self.init_fun, sql_mode="NO_ZERO_DATE,NO_ZERO_IN_DATE,ERROR_FOR_DIVISION_BY_ZERO," - "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION", + "STRICT_ALL_TABLES,NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY", charset=config['connection.charset'], **{k: v for k, v in self.conn_info.items() if not(k in ['ssl_input', 'host_input'] or diff --git a/datajoint/expression.py b/datajoint/expression.py index 6d07784f6..507af14f3 100644 --- a/datajoint/expression.py +++ b/datajoint/expression.py @@ -84,14 +84,15 @@ def restriction_attributes(self): def primary_key(self): return self.heading.primary_key - _subquery_alias_count = count() # count for alias names used in from_clause + _subquery_alias_count = count() # count for alias names used in the FROM clause def from_clause(self): - support = ('(' + src.make_sql() + ') as `_s%x`' % next( - self._subquery_alias_count) if isinstance(src, QueryExpression) else src for src in self.support) + support = ('(' + src.make_sql() + ') as `$%x`' % next( + self._subquery_alias_count) if isinstance(src, QueryExpression) + else src for src in self.support) clause = next(support) for s, left in zip(support, self._left): - clause += 'NATURAL{left} JOIN {clause}'.format( + clause += ' NATURAL{left} JOIN {clause}'.format( left=" LEFT" if left else "", clause=s) return clause @@ -264,8 +265,10 @@ def join(self, other, semantic_check=True, left=False): (set(self.original_heading.names) & set(other.original_heading.names)) - join_attributes) # need subquery if any of the join attributes are derived - need_subquery1 = need_subquery1 or any(n in self.heading.new_attributes for n in join_attributes) - need_subquery2 = need_subquery2 or any(n in other.heading.new_attributes for n in join_attributes) + need_subquery1 = (need_subquery1 or isinstance(self, Aggregation) or + any(n in self.heading.new_attributes for n in join_attributes)) + need_subquery2 = (need_subquery2 or isinstance(other, Aggregation) or + any(n in other.heading.new_attributes for n in join_attributes)) if need_subquery1: self = self.make_subquery() if need_subquery2: @@ -721,8 +724,9 @@ def __and__(self, other): def join(self, other, left=False): """ - Joining U with a query expression has the effect of promoting the attributes of U to the primary key of - the other query expression. + Joining U with a query expression has the effect of promoting the attributes of U to + the primary key of the other query expression. + :param other: the other query expression to join with. :param left: ignored. dj.U always acts as if left=False :return: a copy of the other query expression with the primary key extended. @@ -733,12 +737,14 @@ def join(self, other, left=False): raise DataJointError('Set U can only be joined with a QueryExpression.') try: raise DataJointError( - 'Attribute `%s` not found' % next(k for k in self.primary_key if k not in other.heading.names)) + 'Attribute `%s` not found' % next(k for k in self.primary_key + if k not in other.heading.names)) except StopIteration: pass # all ok result = copy.copy(other) result._heading = result.heading.set_primary_key( - other.primary_key + [k for k in self.primary_key if k not in other.primary_key]) + other.primary_key + [k for k in self.primary_key + if k not in other.primary_key]) return result def __mul__(self, other): diff --git a/tests/schema.py b/tests/schema.py index 1fd187637..a0b336a1c 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -379,3 +379,63 @@ class ComplexChild(dj.Lookup): definition = '\n'.join(['-> ComplexParent'] + ['child_id_{}: int'.format(i+1) for i in range(1)]) contents = [tuple(i for i in range(9))] + + +@schema +class SubjectA(dj.Lookup): + definition = """ + subject_id: varchar(32) + --- + dob : date + sex : enum('M', 'F', 'U') + """ + contents = [ + ('mouse1', '2020-09-01', 'M'), + ('mouse2', '2020-03-19', 'F'), + ('mouse3', '2020-08-23', 'F') + ] + + +@schema +class SessionA(dj.Lookup): + definition = """ + -> SubjectA + session_start_time: datetime + --- + session_dir='' : varchar(32) + """ + contents = [ + ('mouse1', '2020-12-01 12:32:34', ''), + ('mouse1', '2020-12-02 12:32:34', ''), + ('mouse1', '2020-12-03 12:32:34', ''), + ('mouse1', '2020-12-04 12:32:34', '') + ] + + +@schema +class SessionStatusA(dj.Lookup): + definition = """ + -> SessionA + --- + status: enum('in_training', 'trained_1a', 'trained_1b', 'ready4ephys') + """ + contents = [ + ('mouse1', '2020-12-01 12:32:34', 'in_training'), + ('mouse1', '2020-12-02 12:32:34', 'trained_1a'), + ('mouse1', '2020-12-03 12:32:34', 'trained_1b'), + ('mouse1', '2020-12-04 12:32:34', 'ready4ephys'), + ] + + +@schema +class SessionDateA(dj.Lookup): + definition = """ + -> SubjectA + session_date: date + """ + contents = [ + ('mouse1', '2020-12-01'), + ('mouse1', '2020-12-02'), + ('mouse1', '2020-12-03'), + ('mouse1', '2020-12-04') + ] diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index f4e66a6c2..a14d43bae 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -10,7 +10,8 @@ import datajoint as dj from .schema_simple import (A, B, D, E, F, L, DataA, DataB, TTestUpdate, IJ, JI, ReservedWord, OutfitLaunch) -from .schema import Experiment, TTest3, Trial, Ephys, Child, Parent +from .schema import (Experiment, TTest3, Trial, Ephys, Child, Parent, SubjectA, SessionA, + SessionStatusA, SessionDateA) def setup(): @@ -160,76 +161,6 @@ def test_issue_376(): def test_issue_463(): assert_equal(((A & B) * B).fetch().size, len(A * B)) - @staticmethod - def test_issue_898(): - # https://github.com/datajoint/datajoint-python/issues/898 - schema = dj.schema('djtest_raphael') - - @schema - class Subject(dj.Lookup): - definition = """ - subject_id: varchar(32) - --- - dob : date - sex : enum('M', 'F', 'U') - """ - contents = [ - ('mouse1', '2020-09-01', 'M'), - ('mouse2', '2020-03-19', 'F'), - ('mouse3', '2020-08-23', 'F') - ] - - @schema - class Session(dj.Lookup): - definition = """ - -> Subject - session_start_time: datetime - --- - session_dir='' : varchar(32) - """ - contents = [ - ('mouse1', '2020-12-01 12:32:34', ''), - ('mouse1', '2020-12-02 12:32:34', ''), - ('mouse1', '2020-12-03 12:32:34', ''), - ('mouse1', '2020-12-04 12:32:34', '') - ] - - @schema - class SessionStatus(dj.Lookup): - definition = """ - -> Session - --- - status: enum('in_training', 'trained_1a', 'trained_1b', 'ready4ephys') - """ - contents = [ - ('mouse1', '2020-12-01 12:32:34', 'in_training'), - ('mouse1', '2020-12-02 12:32:34', 'trained_1a'), - ('mouse1', '2020-12-03 12:32:34', 'trained_1b'), - ('mouse1', '2020-12-04 12:32:34', 'ready4ephys'), - ] - - @schema - class SessionDate(dj.Lookup): - definition = """ - -> Subject - session_date: date - """ - contents = [ - ('mouse1', '2020-12-01'), - ('mouse1', '2020-12-02'), - ('mouse1', '2020-12-03'), - ('mouse1', '2020-12-04') - ] - - subjects = Subject.aggr( - SessionStatus & 'status="trained_1a" or status="trained_1b"', - date_trained='min(date(session_start_time))') - - print(f'subjects: {subjects}') - print(f'SessionDate: {SessionDate()}') - print(f'join: {SessionDate * subjects}') - print(f'join query: {(SessionDate * subjects).make_sql()}') - @staticmethod def test_project(): x = A().proj(a='id_a') # rename @@ -557,3 +488,20 @@ def test_null_dict_restriction(): assert len(q) == 1 q = F & dict(id=5, date=None) assert len(q) == 1 + + @staticmethod + def test_joins_with_aggregation(): + # https://github.com/datajoint/datajoint-python/issues/898 + # https://github.com/datajoint/datajoint-python/issues/899 + subjects = SubjectA.aggr( + SessionStatusA & 'status="trained_1a" or status="trained_1b"', + date_trained='min(date(session_start_time))') + assert len(SessionDateA * subjects) == 4 + assert len(subjects * SessionDateA) == 4 + + subj_query = SubjectA.aggr( + SessionA * SessionStatusA & 'status="trained_1a" or status="trained_1b"', + date_trained='min(date(session_start_time))') + session_dates = ((SessionDateA * (subj_query & 'date_trained<"2020-12-21"')) & + 'session_date<"date_trained"') + assert len(session_dates) == 4 From c7e933181ae73e40058d3b5a7dc5a09447f50984 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Fri, 9 Apr 2021 09:04:53 -0500 Subject: [PATCH 19/23] Update changelog. --- CHANGELOG.md | 1 + docs-parts/intro/Releases_lang1.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11e19342f..b8ee50d29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 * Bugfix - Diagram part tables do not show proper class name (#882) PR #893 * Bugfix - Error in complex restrictions (#892) PR #893 +* Bugfix - WHERE and GROUP BY clases are dropped on joins with aggregation (#898, #899) PR #893 ### 0.13.0 -- Mar 24, 2021 * Re-implement query transpilation into SQL, fixing issues (#386, #449, #450, #484, #558). PR #754 diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index 9c661e2b8..8d9360d17 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -4,6 +4,7 @@ * Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 * Bugfix - Diagram part tables do not show proper class name (#882) PR #893 * Bugfix - Error in complex restrictions (#892) PR #893 +* Bugfix - WHERE and GROUP BY clases are dropped on joins with aggregation (#898, #899) PR #893 0.13.0 -- Mar 24, 2021 ---------------------- From efeb90bd10ec2ebf0b588a052ca3195387c401bf Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Fri, 9 Apr 2021 10:26:24 -0500 Subject: [PATCH 20/23] Update test. --- tests/test_relational_operand.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_relational_operand.py b/tests/test_relational_operand.py index a14d43bae..108bf895b 100644 --- a/tests/test_relational_operand.py +++ b/tests/test_relational_operand.py @@ -503,5 +503,5 @@ def test_joins_with_aggregation(): SessionA * SessionStatusA & 'status="trained_1a" or status="trained_1b"', date_trained='min(date(session_start_time))') session_dates = ((SessionDateA * (subj_query & 'date_trained<"2020-12-21"')) & - 'session_date<"date_trained"') - assert len(session_dates) == 4 + 'session_date Date: Mon, 12 Apr 2021 08:14:48 -0500 Subject: [PATCH 21/23] Drop tests for MySQL 5.6 since it has reached EOL. --- .github/workflows/development.yaml | 2 +- tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index 9785580b3..c4c2b3475 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -15,7 +15,7 @@ jobs: strategy: matrix: py_ver: ["3.8"] - mysql_ver: ["8.0", "5.7", "5.6"] + mysql_ver: ["8.0", "5.7"] include: - py_ver: "3.7" mysql_ver: "5.7" diff --git a/tests/__init__.py b/tests/__init__.py index 1a48a3a91..6b802e332 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -101,7 +101,7 @@ def setup_package(): conn_root.query( "GRANT SELECT ON `djtest%%`.* TO 'djssl'@'%%';") else: - # grant permissions. For mysql5.6/5.7 this also automatically creates user + # grant permissions. For MySQL 5.7 this also automatically creates user # if not exists conn_root.query(""" GRANT ALL PRIVILEGES ON `djtest%%`.* TO 'datajoint'@'%%' From 4fdebb573584302a57f2ec932e1b447ace176861 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 12 Apr 2021 08:26:35 -0500 Subject: [PATCH 22/23] Update changelog. --- CHANGELOG.md | 1 + docs-parts/intro/Releases_lang1.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8ee50d29..4265e1043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ### 0.13.1 -- TBD * Add `None` as an alias for `NULL` in `dict` restrictions (#824) PR #893 +* Drop support for MySQL 5.6 since it has reached EOL PR #893 * Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 * Bugfix - Diagram part tables do not show proper class name (#882) PR #893 * Bugfix - Error in complex restrictions (#892) PR #893 diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index 8d9360d17..9be833315 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,6 +1,7 @@ 0.13.1 -- TBD ---------------------- * Add `None` as an alias for `NULL` in `dict` restrictions (#824) PR #893 +* Drop support for MySQL 5.6 since it has reached EOL PR #893 * Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 * Bugfix - Diagram part tables do not show proper class name (#882) PR #893 * Bugfix - Error in complex restrictions (#892) PR #893 From 80db98d546d89c0abfbcd6c9157d75381432fab6 Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Mon, 12 Apr 2021 08:31:27 -0500 Subject: [PATCH 23/23] Adjust wording. --- CHANGELOG.md | 2 +- docs-parts/intro/Releases_lang1.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4265e1043..d59027904 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## Release notes ### 0.13.1 -- TBD -* Add `None` as an alias for `NULL` in `dict` restrictions (#824) PR #893 +* Add `None` as an alias for `IS NULL` comparison in `dict` restrictions (#824) PR #893 * Drop support for MySQL 5.6 since it has reached EOL PR #893 * Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 * Bugfix - Diagram part tables do not show proper class name (#882) PR #893 diff --git a/docs-parts/intro/Releases_lang1.rst b/docs-parts/intro/Releases_lang1.rst index 9be833315..ca6decaff 100644 --- a/docs-parts/intro/Releases_lang1.rst +++ b/docs-parts/intro/Releases_lang1.rst @@ -1,6 +1,6 @@ 0.13.1 -- TBD ---------------------- -* Add `None` as an alias for `NULL` in `dict` restrictions (#824) PR #893 +* Add `None` as an alias for `IS NULL` comparison in `dict` restrictions (#824) PR #893 * Drop support for MySQL 5.6 since it has reached EOL PR #893 * Bugfix - `schema.list_tables()` is not topologically sorted (#838) PR #893 * Bugfix - Diagram part tables do not show proper class name (#882) PR #893