diff --git a/gcloud/datastore/dataset.py b/gcloud/datastore/dataset.py index dddb37544ca2..6451d160019f 100644 --- a/gcloud/datastore/dataset.py +++ b/gcloud/datastore/dataset.py @@ -16,7 +16,6 @@ from gcloud.datastore import helpers from gcloud.datastore.entity import Entity -from gcloud.datastore.query import Query from gcloud.datastore.transaction import Transaction @@ -75,19 +74,6 @@ def id(self): return self._id - def query(self, *args, **kwargs): - """Create a query bound to this dataset. - - :param args: positional arguments, passed through to the Query - - :param kw: keyword arguments, passed through to the Query - - :rtype: :class:`gcloud.datastore.query.Query` - :returns: a new Query instance, bound to this dataset. - """ - kwargs['dataset'] = self - return Query(*args, **kwargs) - def entity(self, kind, exclude_from_indexes=()): """Create an entity bound to this dataset. diff --git a/gcloud/datastore/query.py b/gcloud/datastore/query.py index 569db3ce1d37..3604db1eb04a 100644 --- a/gcloud/datastore/query.py +++ b/gcloud/datastore/query.py @@ -19,6 +19,7 @@ from gcloud.datastore import _implicit_environ from gcloud.datastore import datastore_v1_pb2 as datastore_pb from gcloud.datastore import helpers +from gcloud.datastore.dataset import Dataset from gcloud.datastore.key import Key @@ -28,40 +29,33 @@ class Query(_implicit_environ._DatastoreBase): This class serves as an abstraction for creating a query over data stored in the Cloud Datastore. - Each :class:`Query` object is immutable, and a clone is returned - whenever any part of the query is modified:: + :type kind: string. + :param kind: The kind to query. - >>> query = Query('MyKind') - >>> limited_query = query.limit(10) - >>> query.limit() == 10 - False - >>> limited_query.limit() == 10 - True + :type dataset: :class:`gcloud.datastore.dataset.Dataset`. + :param dataset: The dataset to query. - You typically won't construct a :class:`Query` by initializing it - like ``Query('MyKind', dataset=...)`` but instead use the helper - :func:`gcloud.datastore.dataset.Dataset.query` method which - generates a query that can be executed without any additional work:: + :type namespace: string or None. + :param namespace: The namespace to which to restrict results. - >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id') - >>> query = dataset.query('MyKind') + :type ancestor: :class:`gcloud.datastore.key.Key` or None. + :param ancestor: key of the ancestor to which this query's results are + restricted. - :type kind: string - :param kind: The kind to query. + :type filters: sequence of (property_name, operator, value) tuples. + :param filters: property filters applied by this query. - :type dataset: :class:`gcloud.datastore.dataset.Dataset` - :param dataset: The dataset to query. + :type projection: sequence of string. + :param projection: fields returned as part of query results. + + :type order: sequence of string. + :param order: field names used to order query results. Prepend '-' + to a field name to sort it in descending order. - :type namespace: string or None - :param dataset: The namespace to which to restrict results. + :type group_by: sequence_of_string. + :param group_by: field names used to group query results. """ - _NOT_FINISHED = datastore_pb.QueryResultBatch.NOT_FINISHED - _FINISHED = ( - datastore_pb.QueryResultBatch.NO_MORE_RESULTS, - datastore_pb.QueryResultBatch.MORE_RESULTS_AFTER_LIMIT, - ) OPERATORS = { '<=': datastore_pb.PropertyFilter.LESS_THAN_OR_EQUAL, '>=': datastore_pb.PropertyFilter.GREATER_THAN_OR_EQUAL, @@ -71,26 +65,47 @@ class Query(_implicit_environ._DatastoreBase): } """Mapping of operator strings and their protobuf equivalents.""" - def __init__(self, kind=None, dataset=None, namespace=None): + def __init__(self, + kind=None, + dataset=None, + namespace=None, + ancestor=None, + filters=(), + projection=(), + order=(), + group_by=()): super(Query, self).__init__(dataset=dataset) + self._kind = kind self._namespace = namespace - self._pb = datastore_pb.Query() - self._offset = 0 + self._ancestor = ancestor + self._filters = list(filters) + self._projection = list(projection) + self._order = list(order) + self._group_by = list(group_by) - if kind: - self._pb.kind.add().name = kind + @property + def dataset(self): + """Get the dataset for this Query. - def _clone(self): - """Create a new Query, copying self. + The dataset against which the Query will be run. - :rtype: :class:`gcloud.datastore.query.Query` - :returns: a copy of 'self'. + :rtype: :class:`gcloud.datastore.dataset.Dataset` or None, + :returns: the current dataset. """ - clone = self.__class__(dataset=self._dataset, - namespace=self._namespace) - clone._pb.CopyFrom(self._pb) - return clone + return self._dataset + @dataset.setter + def dataset(self, value): + """Set the dataset for the query + + :type value: class:`gcloud.datastore.dataset.Dataset` + :param value: the new dataset + """ + if not isinstance(value, Dataset): + raise ValueError("Dataset must be a Dataset") + self._dataset = value + + @property def namespace(self): """This query's namespace @@ -99,37 +114,90 @@ def namespace(self): """ return self._namespace - def to_protobuf(self): - """Convert :class:`Query` instance to :class:`.datastore_v1_pb2.Query`. + @namespace.setter + def namespace(self, value): + """Update the query's namespace. - :rtype: :class:`gcloud.datastore.datastore_v1_pb2.Query` - :returns: A Query protobuf that can be sent to the protobuf API. + :type value: string """ - return self._pb + if not isinstance(value, str): + raise ValueError("Namespace must be a string") + self._namespace = value - def filter(self, property_name, operator, value): - """Filter the query based on a property name, operator and a value. + @property + def kind(self): + """Get the Kind of the Query. + + :rtype: string or :class:`Query` + """ + return self._kind + + @kind.setter + def kind(self, value): + """Update the Kind of the Query. - This will return a clone of the current :class:`Query` - filtered by the expression and value provided. + :type value: string + :param value: updated kind for the query. + + .. note:: + + The protobuf specification allows for ``kind`` to be repeated, + but the current implementation returns an error if more than + one value is passed. If the back-end changes in the future to + allow multiple values, this method will be updated to allow passing + either a string or a sequence of strings. + """ + if not isinstance(value, str): + raise TypeError("Kind must be a string") + self._kind = value + + @property + def ancestor(self): + """The ancestor key for the query. + + :rtype: Key or None + """ + return self._ancestor + + @ancestor.setter + def ancestor(self, value): + """Set the ancestor for the query + + :type value: Key + :param value: the new ancestor key + """ + if not isinstance(value, Key): + raise TypeError("Ancestor must be a Key") + self._ancestor = value + + @ancestor.deleter + def ancestor(self): + """Remove the ancestor for the query. + """ + self._ancestor = None + + @property + def filters(self): + """Filters set on the query. + + :rtype: sequence of (property_name, operator, value) tuples. + """ + return self._filters[:] + + def add_filter(self, property_name, operator, value): + """Filter the query based on a property name, operator and a value. Expressions take the form of:: - .filter('', '', ) + .add_filter('', '', ) where property is a property stored on the entity in the datastore and operator is one of ``OPERATORS`` (ie, ``=``, ``<``, ``<=``, ``>``, ``>=``):: >>> query = Query('Person') - >>> filtered_query = query.filter('name', '=', 'James') - >>> filtered_query = query.filter('age', '>', 50) - - Because each call to ``.filter()`` returns a cloned ``Query`` object - we are able to string these together:: - - >>> query = Query('Person').filter( - ... 'name', '=', 'James').filter('age', '>', 50) + >>> query.add_filter('name', '=', 'James') + >>> query.add_filter('age', '>', 50) :type property_name: string :param property_name: A property name. @@ -140,219 +208,162 @@ def filter(self, property_name, operator, value): :type value: integer, string, boolean, float, None, datetime :param value: The value to filter on. - :rtype: :class:`Query` - :returns: A Query filtered by the expression and value provided. :raises: `ValueError` if `operation` is not one of the specified - values. + values, or if a filter names '__key__' but passes invalid + operator (``==`` is required) or value (a key is required). """ - clone = self._clone() - - pb_op_enum = self.OPERATORS.get(operator) - if pb_op_enum is None: + if self.OPERATORS.get(operator) is None: error_message = 'Invalid expression: "%s"' % (operator,) choices_message = 'Please use one of: =, <, <=, >, >=.' raise ValueError(error_message, choices_message) - # Build a composite filter AND'd together. - composite_filter = clone._pb.filter.composite_filter - composite_filter.operator = datastore_pb.CompositeFilter.AND - - # Add the specific filter - property_filter = composite_filter.filter.add().property_filter - property_filter.property.name = property_name - property_filter.operator = pb_op_enum - - # Set the value to filter on based on the type. if property_name == '__key__': if not isinstance(value, Key): - raise TypeError('__key__ query requires a Key instance.') - key_pb = value.to_protobuf() - property_filter.value.key_value.CopyFrom( - helpers._prepare_key_for_request(key_pb)) - else: - helpers._set_protobuf_value(property_filter.value, value) - return clone - - def ancestor(self, ancestor): - """Filter the query based on an ancestor. - - This will return a clone of the current :class:`Query` filtered - by the ancestor provided. For example:: + raise ValueError('Invalid key: "%s"' % value) + if operator != '=': + raise ValueError('Invalid operator for key: "%s"' % operator) - >>> parent_key = Key('Person', '1') - >>> query = dataset.query('Person') - >>> filtered_query = query.ancestor(parent_key) + self._filters.append((property_name, operator, value)) - Each call to ``.ancestor()`` returns a cloned :class:`Query`, - however a query may only have one ancestor at a time. - - :type ancestor: :class:`gcloud.datastore.key.Key` - :param ancestor: A Key to an entity + @property + def projection(self): + """Fields names returned by the query. - :rtype: :class:`Query` - :returns: A Query filtered by the ancestor provided. + :rtype: sequence of string + :returns: names of fields in query results. """ + return self._projection[:] - clone = self._clone() + @projection.setter + def projection(self, projection): + """Set the fields returned the query. - # If an ancestor filter already exists, remove it. - for i, filter in enumerate(clone._pb.filter.composite_filter.filter): - property_filter = filter.property_filter - if (property_filter.operator == - datastore_pb.PropertyFilter.HAS_ANCESTOR): - del clone._pb.filter.composite_filter.filter[i] + :type projection: string or sequence of strings + :param projection: Each value is a string giving the name of a + property to be included in the projection query. + """ + if isinstance(projection, str): + projection = [projection] + self._projection[:] = projection - # If we just deleted the last item, make sure to clear out the - # filter property all together. - if not clone._pb.filter.composite_filter.filter: - clone._pb.ClearField('filter') + @property + def order(self): + """Names of fields used to sort query results. - # If the ancestor is None, just return (we already removed the filter). - if not ancestor: - return clone + :rtype: sequence of string + """ + return self._order[:] - # If we don't have a Key value by now, something is wrong. - if not isinstance(ancestor, Key): - raise TypeError('Expected Key, got %s.' % type(ancestor)) + @order.setter + def order(self, value): + """Set the fields used to sort query results. - # Get the composite filter and add a new property filter. - composite_filter = clone._pb.filter.composite_filter - composite_filter.operator = datastore_pb.CompositeFilter.AND + Sort fields will be applied in the order specified. - # Filter on __key__ HAS_ANCESTOR == ancestor. - ancestor_filter = composite_filter.filter.add().property_filter - ancestor_filter.property.name = '__key__' - ancestor_filter.operator = datastore_pb.PropertyFilter.HAS_ANCESTOR - ancestor_pb = helpers._prepare_key_for_request(ancestor.to_protobuf()) - ancestor_filter.value.key_value.CopyFrom(ancestor_pb) + :type value: string or sequence of strings + :param value: Each value is a string giving the name of the + property on which to sort, optionally preceded by a + hyphen (-) to specify descending order. + Omitting the hyphen implies ascending order. + """ + if isinstance(value, str): + value = [value] + self._order[:] = value - return clone + @property + def group_by(self): + """Names of fields used to group query results. - def kind(self, kind=None): - """Get or set the Kind of the Query. + :rtype: sequence of string + """ + return self._group_by[:] - :type kind: string - :param kind: Optional. The entity kinds for which to query. + @group_by.setter + def group_by(self, value): + """Set fields used to group query results. - :rtype: string or :class:`Query` - :returns: If `kind` is None, returns the kind. If a kind is provided, - returns a clone of the :class:`Query` with that kind set. - :raises: `ValueError` from the getter if multiple kinds are set on - the query. + :type value: string or sequence of strings + :param value: Each value is a string giving the name of a + property to use to group results together. """ - if kind is not None: - kinds = [kind] - clone = self._clone() - clone._pb.ClearField('kind') - for new_kind in kinds: - clone._pb.kind.add().name = new_kind - return clone - else: - # In the proto definition for Query, `kind` is repeated. - kind_names = [kind_expr.name for kind_expr in self._pb.kind] - num_kinds = len(kind_names) - if num_kinds == 1: - return kind_names[0] - elif num_kinds > 1: - raise ValueError('Only a single kind can be set.') + if isinstance(value, str): + value = [value] + self._group_by[:] = value - def limit(self, limit=None): - """Get or set the limit of the Query. + def fetch(self, limit=0, offset=0, start_cursor=None, end_cursor=None): + """Execute the Query; return an iterator for the matching entities. - This is the maximum number of rows (Entities) to return for this - Query. + For example:: - This is a hybrid getter / setter, used as:: + >>> from gcloud import datastore + >>> dataset = datastore.get_dataset('dataset-id') + >>> query = dataset.query('Person').filter('name', '=', 'Sally') + >>> list(query.fetch()) + [, , ...] + >>> list(query.fetch(1)) + [] - >>> query = Query('Person') - >>> query = query.limit(100) # Set the limit to 100 rows. - >>> query.limit() # Get the limit for this query. - 100 - - :rtype: integer, None, or :class:`Query` - :returns: If no arguments, returns the current limit. - If a limit is provided, returns a clone of the :class:`Query` - with that limit set. - """ - if limit: - clone = self._clone() - clone._pb.limit = limit - return clone - else: - return self._pb.limit + :type limit: integer + :param limit: An optional limit passed through to the iterator. - def dataset(self, dataset=None): - """Get or set the :class:`.datastore.dataset.Dataset` for this Query. + :type limit: offset + :param limit: An optional offset passed through to the iterator. - This is the dataset against which the Query will be run. + :type start_cursor: offset + :param start_cursor: An optional cursor passed through to the iterator. - This is a hybrid getter / setter, used as:: + :type end_cursor: offset + :param end_cursor: An optional cursor passed through to the iterator. - >>> query = Query('Person') - >>> query = query.dataset(my_dataset) # Set the dataset. - >>> query.dataset() # Get the current dataset. - - - :rtype: :class:`gcloud.datastore.dataset.Dataset`, None, - or :class:`Query` - :returns: If no arguments, returns the current dataset. - If a dataset is provided, returns a clone of the - :class:`Query` with that dataset set. + :rtype: :class:`Iterator` """ - if dataset: - clone = self._clone() - clone._dataset = dataset - return clone - else: - return self._dataset + return Iterator(self, limit, offset, start_cursor, end_cursor) - def fetch_page(self, limit=None): - """Executes the Query and returns matching entities, and paging info. - In addition to the fetched entities, it also returns a cursor to allow - paging through a results set and a boolean `more_results` indicating - if there are any more. +class Iterator(object): + """Represent the state of a given execution of a Query. + """ + _NOT_FINISHED = datastore_pb.QueryResultBatch.NOT_FINISHED - This makes an API call to the Cloud Datastore, sends the Query - as a protobuf, parses the responses to Entity protobufs, and - then converts them to :class:`gcloud.datastore.entity.Entity` - objects. + _FINISHED = ( + datastore_pb.QueryResultBatch.NO_MORE_RESULTS, + datastore_pb.QueryResultBatch.MORE_RESULTS_AFTER_LIMIT, + ) - For example:: + def __init__(self, query, limit=0, offset=0, + start_cursor=None, end_cursor=None): + self._query = query + self._limit = limit + self._offset = offset + self._start_cursor = start_cursor + self._end_cursor = end_cursor + self._page = self._more_results = None - >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id') - >>> query = dataset.query('Person').filter('name', '=', 'Sally') - >>> query.fetch_page() - [, , ...], 'cursorbase64', True - >>> query.fetch_page(1) - [], 'cursorbase64', True - >>> query.limit() - None + def next_page(self): + """Fetch a single "page" of query results. - :type limit: integer - :param limit: An optional limit to apply temporarily to this query. - That is, the Query itself won't be altered, - but the limit will be applied to the query - before it is executed. - - :rtype: tuple of mixed types - :returns: The first entry is a :class:`gcloud.datastore.entity.Entity` - list matching this query's criteria. The second is a base64 - encoded cursor for paging and the third is a boolean - indicating if there are more results. - :raises: `ValueError` if more_results is not one of the enums - NOT_FINISHED, MORE_RESULTS_AFTER_LIMIT, NO_MORE_RESULTS. + Low-level API for fine control: the more convenient API is + to iterate on us. + + :rtype: tuple, (entities, more_results, cursor) """ - clone = self + pb = _pb_from_query(self._query) + + start_cursor = self._start_cursor + if start_cursor is not None: + pb.start_cursor = base64.b64decode(start_cursor) - if limit: - clone = self.limit(limit) + end_cursor = self._end_cursor + if end_cursor is not None: + pb.end_cursor = base64.b64decode(end_cursor) - query_results = self.dataset().connection().run_query( - query_pb=clone.to_protobuf(), - dataset_id=self.dataset().id(), - namespace=self._namespace, + pb.limit = self._limit + pb.offset = self._offset + + query_results = self._query.dataset.connection().run_query( + query_pb=pb, + dataset_id=self._query.dataset.id(), + namespace=self._query.namespace, ) # NOTE: `query_results` contains an extra value that we don't use, # namely `skipped_results`. @@ -365,216 +376,98 @@ def fetch_page(self, limit=None): # for discussion. entity_pbs, cursor_as_bytes, more_results_enum = query_results[:3] - entities = [helpers.entity_from_protobuf(entity, - dataset=self.dataset()) - for entity in entity_pbs] - - cursor = base64.b64encode(cursor_as_bytes) + self._start_cursor = base64.b64encode(cursor_as_bytes) + self._end_cursor = None if more_results_enum == self._NOT_FINISHED: - more_results = True + self._more_results = True elif more_results_enum in self._FINISHED: - more_results = False + self._more_results = False else: raise ValueError('Unexpected value returned for `more_results`.') - return entities, cursor, more_results + dataset = self._query.dataset + self._page = [ + helpers.entity_from_protobuf(entity, dataset=dataset) + for entity in entity_pbs] + return self._page, self._more_results, self._start_cursor - def fetch(self, limit=None): - """Executes the Query and returns matching entities + def __iter__(self): + """Generator yielding all results matching our query. - This calls `fetch_page()` but does not use the paging information. - - For example:: - - >>> from gcloud import datastore - >>> dataset = datastore.get_dataset('dataset-id') - >>> query = dataset.query('Person').filter('name', '=', 'Sally') - >>> query.fetch() - [, , ...] - >>> query.fetch(1) - [] - >>> query.limit() - None - - :type limit: integer - :param limit: An optional limit to apply temporarily to this query. - That is, the Query itself won't be altered, - but the limit will be applied to the query - before it is executed. - - :rtype: list of :class:`gcloud.datastore.entity.Entity`'s - :returns: The list of entities matching this query's criteria. + :rtype: sequence of :class:`gcloud.datastore.entity.Entity` """ - entities, _, _ = self.fetch_page(limit=limit) - return entities - - @property - def start_cursor(self): - """Property to encode start cursor bytes as base64.""" - if not self._pb.HasField('start_cursor'): - return None - - start_as_bytes = self._pb.start_cursor - return base64.b64encode(start_as_bytes) - - @property - def end_cursor(self): - """Property to encode end cursor bytes as base64.""" - if not self._pb.HasField('end_cursor'): - return None - - end_as_bytes = self._pb.end_cursor - return base64.b64encode(end_as_bytes) + self.next_page() + while True: + for entity in self._page: + yield entity + if not self._more_results: + break + self.next_page() - def with_cursor(self, start_cursor, end_cursor=None): - """Specifies the starting / ending positions in a query's result set. - :type start_cursor: bytes - :param start_cursor: Base64-encoded cursor string specifying where to - start reading query results. +def _pb_from_query(query): + """Convert a Query instance to the corresponding protobuf. - :type end_cursor: bytes - :param end_cursor: Base64-encoded cursor string specifying where to - stop reading query results. + :type query: :class:`Query` + :param query: the source query - :rtype: :class:`Query` - :returns: If neither cursor is passed, returns self; else, returns a - clone of the :class:`Query`, with cursors updated. - """ - clone = self - if start_cursor or end_cursor: - clone = self._clone() - if start_cursor: - clone._pb.start_cursor = base64.b64decode(start_cursor) - if end_cursor: - clone._pb.end_cursor = base64.b64decode(end_cursor) - return clone - - def order(self, *properties): - """Adds a sort order to the query. - - Sort fields will be applied in the order specified. - - :type properties: sequence of strings - :param properties: Each value is a string giving the name of the - property on which to sort, optionally preceded by a - hyphen (-) to specify descending order. - Omitting the hyphen implies ascending order. - - :rtype: :class:`Query` - :returns: A new Query instance, ordered as specified. - """ - clone = self._clone() - - for prop in properties: - property_order = clone._pb.order.add() - - if prop.startswith('-'): - property_order.property.name = prop[1:] - property_order.direction = property_order.DESCENDING - else: - property_order.property.name = prop - property_order.direction = property_order.ASCENDING - - return clone - - def projection(self, projection=None): - """Adds a projection to the query. - - This is a hybrid getter / setter, used as:: - - >>> query = Query('Person') - >>> query.projection() # Get the projection for this query. - [] - >>> query = query.projection(['name']) - >>> query.projection() # Get the projection for this query. - ['name'] - - :type projection: sequence of strings - :param projection: Each value is a string giving the name of a - property to be included in the projection query. - - :rtype: :class:`Query` or `list` of strings. - :returns: If no arguments, returns the current projection. - If a projection is provided, returns a clone of the - :class:`Query` with that projection set. - """ - if projection is None: - return [prop_expr.property.name - for prop_expr in self._pb.projection] - - clone = self._clone() - - # Reset projection values to empty. - clone._pb.ClearField('projection') + :rtype: :class:`gcloud.datastore.datastore_v1_pb2.Query` + :returns: a protobuf that can be sent to the protobuf API. N.b. that + it does not contain "in-flight" fields for ongoing query + executions (cursors, offset, limit). + """ + pb = datastore_pb.Query() - # Add each name to list of projections. - for projection_name in projection: - clone._pb.projection.add().property.name = projection_name - return clone + for projection_name in query.projection: + pb.projection.add().property.name = projection_name - def offset(self, offset=None): - """Adds offset to the query to allow pagination. + if query.kind: + pb.kind.add().name = query.kind - NOTE: Paging with cursors should be preferred to using an offset. + composite_filter = pb.filter.composite_filter + composite_filter.operator = datastore_pb.CompositeFilter.AND - This is a hybrid getter / setter, used as:: + if query.ancestor: + ancestor_pb = helpers._prepare_key_for_request( + query.ancestor.to_protobuf()) - >>> query = Query('Person') - >>> query.offset() # Get the offset for this query. - 0 - >>> query = query.offset(10) - >>> query.offset() # Get the offset for this query. - 10 - - :type offset: non-negative integer. - :param offset: Value representing where to start a query for - a given kind. - - :rtype: :class:`Query` or `int`. - :returns: If no arguments, returns the current offset. - If an offset is provided, returns a clone of the - :class:`Query` with that offset set. - """ - if offset is None: - return self._offset + # Filter on __key__ HAS_ANCESTOR == ancestor. + ancestor_filter = composite_filter.filter.add().property_filter + ancestor_filter.property.name = '__key__' + ancestor_filter.operator = datastore_pb.PropertyFilter.HAS_ANCESTOR + ancestor_filter.value.key_value.CopyFrom(ancestor_pb) - clone = self._clone() - clone._offset = offset - clone._pb.offset = offset - return clone + for property_name, operator, value in query.filters: + pb_op_enum = query.OPERATORS.get(operator) - def group_by(self, group_by=None): - """Adds a group_by to the query. + # Add the specific filter + property_filter = composite_filter.filter.add().property_filter + property_filter.property.name = property_name + property_filter.operator = pb_op_enum - This is a hybrid getter / setter, used as:: + # Set the value to filter on based on the type. + if property_name == '__key__': + key_pb = value.to_protobuf() + property_filter.value.key_value.CopyFrom( + helpers._prepare_key_for_request(key_pb)) + else: + helpers._set_protobuf_value(property_filter.value, value) - >>> query = Query('Person') - >>> query.group_by() # Get the group_by for this query. - [] - >>> query = query.group_by(['name']) - >>> query.group_by() # Get the group_by for this query. - ['name'] - - :type group_by: sequence of strings - :param group_by: Each value is a string giving the name of a - property to use to group results together. + if not composite_filter.filter: + pb.ClearField('filter') - :rtype: :class:`Query` or `list` of strings. - :returns: If no arguments, returns the current group_by. - If a list of group by properties is provided, returns a clone - of the :class:`Query` with that list of values set. - """ - if group_by is None: - return [prop_ref.name for prop_ref in self._pb.group_by] + for prop in query.order: + property_order = pb.order.add() - clone = self._clone() + if prop.startswith('-'): + property_order.property.name = prop[1:] + property_order.direction = property_order.DESCENDING + else: + property_order.property.name = prop + property_order.direction = property_order.ASCENDING - # Reset group_by values to empty. - clone._pb.ClearField('group_by') + for group_by_name in query.group_by: + pb.group_by.add().name = group_by_name - # Add each name to list of group_bys. - for group_by_name in group_by: - clone._pb.group_by.add().name = group_by_name - return clone + return pb diff --git a/gcloud/datastore/test_connection.py b/gcloud/datastore/test_connection.py index 2e305476b63a..a61b29ddbb8f 100644 --- a/gcloud/datastore/test_connection.py +++ b/gcloud/datastore/test_connection.py @@ -29,6 +29,12 @@ def _make_key_pb(self, dataset_id, id=1234): path_args += (id,) return Key(*path_args, dataset_id=dataset_id).to_protobuf() + def _make_query_pb(self, kind): + from gcloud.datastore.connection import datastore_pb + pb = datastore_pb.Query() + pb.kind.add().name = kind + return pb + def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) @@ -511,12 +517,11 @@ def test_lookup_multiple_keys_w_deferred_from_backend_but_not_passed(self): def test_run_query_w_eventual_no_transaction(self): from gcloud.datastore.connection import datastore_pb - from gcloud.datastore.query import Query DATASET_ID = 'DATASET' KIND = 'Nonesuch' CURSOR = b'\x00' - q_pb = Query(KIND, DATASET_ID).to_protobuf() + q_pb = self._make_query_pb(KIND) rsp_pb = datastore_pb.RunQueryResponse() rsp_pb.batch.end_cursor = CURSOR no_more = datastore_pb.QueryResultBatch.NO_MORE_RESULTS @@ -551,13 +556,12 @@ def test_run_query_w_eventual_no_transaction(self): def test_run_query_wo_eventual_w_transaction(self): from gcloud.datastore.connection import datastore_pb - from gcloud.datastore.query import Query DATASET_ID = 'DATASET' KIND = 'Nonesuch' CURSOR = b'\x00' TRANSACTION = 'TRANSACTION' - q_pb = Query(KIND, DATASET_ID).to_protobuf() + q_pb = self._make_query_pb(KIND) rsp_pb = datastore_pb.RunQueryResponse() rsp_pb.batch.end_cursor = CURSOR no_more = datastore_pb.QueryResultBatch.NO_MORE_RESULTS @@ -592,13 +596,12 @@ def test_run_query_wo_eventual_w_transaction(self): def test_run_query_w_eventual_and_transaction(self): from gcloud.datastore.connection import datastore_pb - from gcloud.datastore.query import Query DATASET_ID = 'DATASET' KIND = 'Nonesuch' CURSOR = b'\x00' TRANSACTION = 'TRANSACTION' - q_pb = Query(KIND, DATASET_ID).to_protobuf() + q_pb = self._make_query_pb(KIND) rsp_pb = datastore_pb.RunQueryResponse() rsp_pb.batch.end_cursor = CURSOR no_more = datastore_pb.QueryResultBatch.NO_MORE_RESULTS @@ -611,12 +614,11 @@ def test_run_query_w_eventual_and_transaction(self): def test_run_query_wo_namespace_empty_result(self): from gcloud.datastore.connection import datastore_pb - from gcloud.datastore.query import Query DATASET_ID = 'DATASET' KIND = 'Nonesuch' CURSOR = b'\x00' - q_pb = Query(KIND, DATASET_ID).to_protobuf() + q_pb = self._make_query_pb(KIND) rsp_pb = datastore_pb.RunQueryResponse() rsp_pb.batch.end_cursor = CURSOR no_more = datastore_pb.QueryResultBatch.NO_MORE_RESULTS @@ -647,12 +649,11 @@ def test_run_query_wo_namespace_empty_result(self): def test_run_query_w_namespace_nonempty_result(self): from gcloud.datastore.connection import datastore_pb - from gcloud.datastore.query import Query DATASET_ID = 'DATASET' KIND = 'Kind' entity = datastore_pb.Entity() - q_pb = Query(KIND, DATASET_ID).to_protobuf() + q_pb = self._make_query_pb(KIND) rsp_pb = datastore_pb.RunQueryResponse() rsp_pb.batch.entity_result.add(entity=entity) rsp_pb.batch.entity_result_type = 1 # FULL diff --git a/gcloud/datastore/test_dataset.py b/gcloud/datastore/test_dataset.py index cec7b70496bf..783ef0df684d 100644 --- a/gcloud/datastore/test_dataset.py +++ b/gcloud/datastore/test_dataset.py @@ -40,14 +40,6 @@ def test_ctor_explicit(self): self.assertEqual(dataset.id(), DATASET_ID) self.assertTrue(dataset.connection() is CONNECTION) - def test_query_factory(self): - from gcloud.datastore.query import Query - DATASET_ID = 'DATASET' - dataset = self._makeOne(DATASET_ID) - query = dataset.query() - self.assertIsInstance(query, Query) - self.assertTrue(query.dataset() is dataset) - def test_entity_factory_defaults(self): from gcloud.datastore.entity import Entity DATASET_ID = 'DATASET' diff --git a/gcloud/datastore/test_query.py b/gcloud/datastore/test_query.py index 1e964b0dcdbc..20cd4d210811 100644 --- a/gcloud/datastore/test_query.py +++ b/gcloud/datastore/test_query.py @@ -30,255 +30,94 @@ def _getTargetClass(self): from gcloud.datastore.query import Query return Query - def _makeOne(self, kind=None, dataset=None, namespace=None): - return self._getTargetClass()(kind, dataset, namespace) + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) def test_ctor_defaults(self): query = self._getTargetClass()() - self.assertEqual(query.dataset(), None) - self.assertEqual(query.kind(), None) - self.assertEqual(query.limit(), 0) - self.assertEqual(query.namespace(), None) + self.assertEqual(query.dataset, None) + self.assertEqual(query.kind, None) + self.assertEqual(query.namespace, None) + self.assertEqual(query.ancestor, None) + self.assertEqual(query.filters, []) + self.assertEqual(query.projection, []) + self.assertEqual(query.order, []) + self.assertEqual(query.group_by, []) def test_ctor_explicit(self): from gcloud.datastore.dataset import Dataset - + from gcloud.datastore.key import Key _DATASET = 'DATASET' _KIND = 'KIND' _NAMESPACE = 'NAMESPACE' dataset = Dataset(_DATASET) - query = self._makeOne(_KIND, dataset, _NAMESPACE) - self.assertTrue(query.dataset() is dataset) - self.assertEqual(query.kind(), _KIND) - self.assertEqual(query.namespace(), _NAMESPACE) + ancestor = Key('ANCESTOR', 123, dataset_id=_DATASET) + FILTERS = [('foo', '=', 'Qux'), ('bar', '<', 17)] + PROJECTION = ['foo', 'bar', 'baz'] + ORDER = ['foo', 'bar'] + GROUP_BY = ['foo'] + query = self._makeOne( + kind=_KIND, + dataset=dataset, + namespace=_NAMESPACE, + ancestor=ancestor, + filters=FILTERS, + projection=PROJECTION, + order=ORDER, + group_by=GROUP_BY, + ) + self.assertTrue(query.dataset is dataset) + self.assertEqual(query.kind, _KIND) + self.assertEqual(query.namespace, _NAMESPACE) + self.assertEqual(query.ancestor.path, ancestor.path) + self.assertEqual(query.filters, FILTERS) + self.assertEqual(query.projection, PROJECTION) + self.assertEqual(query.order, ORDER) + self.assertEqual(query.group_by, GROUP_BY) + + def test_dataset_setter_w_non_dataset(self): + query = self._makeOne() - def test__clone(self): - from gcloud.datastore.dataset import Dataset + def _assign(val): + query.dataset = val + self.assertRaises(ValueError, _assign, object()) + + def test_dataset_setter(self): + from gcloud.datastore.dataset import Dataset _DATASET = 'DATASET' _KIND = 'KIND' - _NAMESPACE = 'NAMESPACE' dataset = Dataset(_DATASET) - query = self._makeOne(_KIND, dataset, _NAMESPACE) - clone = query._clone() - self.assertFalse(clone is query) - self.assertTrue(isinstance(clone, self._getTargetClass())) - self.assertTrue(clone.dataset() is dataset) - self.assertEqual(clone.namespace(), _NAMESPACE) - self.assertEqual(clone.kind(), _KIND) - - def test_to_protobuf_empty(self): - query = self._makeOne() - q_pb = query.to_protobuf() - self.assertEqual(list(q_pb.kind), []) - self.assertEqual(list(q_pb.filter.composite_filter.filter), []) - - def test_to_protobuf_w_kind(self): - _KIND = 'KIND' query = self._makeOne(_KIND) - q_pb = query.to_protobuf() - kq_pb, = list(q_pb.kind) - self.assertEqual(kq_pb.name, _KIND) - - def test_filter_w_unknown_operator(self): - query = self._makeOne() - self.assertRaises(ValueError, query.filter, 'firstname', '~~', 'John') - - def test_filter_w_known_operator(self): - from gcloud.datastore import datastore_v1_pb2 as datastore_pb + query.dataset = dataset + self.assertTrue(query.dataset is dataset) + self.assertEqual(query.kind, _KIND) + def test_namespace_setter_w_non_string(self): query = self._makeOne() - after = query.filter('firstname', '=', u'John') - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.filter.composite_filter.operator, - datastore_pb.CompositeFilter.AND) - f_pb, = list(q_pb.filter.composite_filter.filter) - p_pb = f_pb.property_filter - self.assertEqual(p_pb.property.name, 'firstname') - self.assertEqual(p_pb.value.string_value, u'John') - self.assertEqual(p_pb.operator, datastore_pb.PropertyFilter.EQUAL) - - def test_filter_w_all_operators(self): - from gcloud.datastore import datastore_v1_pb2 as datastore_pb - query = self._makeOne() - query = query.filter('leq_prop', '<=', u'val1') - query = query.filter('geq_prop', '>=', u'val2') - query = query.filter('lt_prop', '<', u'val3') - query = query.filter('gt_prop', '>', u'val4') - query = query.filter('eq_prop', '=', u'val5') - - query_pb = query.to_protobuf() - pb_values = [ - ('leq_prop', 'val1', - datastore_pb.PropertyFilter.LESS_THAN_OR_EQUAL), - ('geq_prop', 'val2', - datastore_pb.PropertyFilter.GREATER_THAN_OR_EQUAL), - ('lt_prop', 'val3', datastore_pb.PropertyFilter.LESS_THAN), - ('gt_prop', 'val4', datastore_pb.PropertyFilter.GREATER_THAN), - ('eq_prop', 'val5', datastore_pb.PropertyFilter.EQUAL), - ] - query_filter = query_pb.filter.composite_filter.filter - for filter_pb, pb_value in zip(query_filter, pb_values): - name, val, filter_enum = pb_value - prop_filter = filter_pb.property_filter - self.assertEqual(prop_filter.property.name, name) - self.assertEqual(prop_filter.value.string_value, val) - self.assertEqual(prop_filter.operator, filter_enum) - - def test_filter_w_known_operator_and_entity(self): - import operator - from gcloud.datastore.entity import Entity - query = self._makeOne() - other = Entity() - other['firstname'] = u'John' - other['lastname'] = u'Smith' - after = query.filter('other', '=', other) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.filter.composite_filter.operator, 1) # AND - f_pb, = list(q_pb.filter.composite_filter.filter) - p_pb = f_pb.property_filter - self.assertEqual(p_pb.property.name, 'other') - other_pb = p_pb.value.entity_value - props = sorted(other_pb.property, key=operator.attrgetter('name')) - self.assertEqual(len(props), 2) - self.assertEqual(props[0].name, 'firstname') - self.assertEqual(props[0].value.string_value, u'John') - self.assertEqual(props[1].name, 'lastname') - self.assertEqual(props[1].value.string_value, u'Smith') - - def test_filter_w_whitespace_property_name(self): - from gcloud.datastore import datastore_v1_pb2 as datastore_pb + def _assign(val): + query.namespace = val - query = self._makeOne() - PROPERTY_NAME = ' property with lots of space ' - after = query.filter(PROPERTY_NAME, '=', u'John') - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.filter.composite_filter.operator, - datastore_pb.CompositeFilter.AND) - f_pb, = list(q_pb.filter.composite_filter.filter) - p_pb = f_pb.property_filter - self.assertEqual(p_pb.property.name, PROPERTY_NAME) - self.assertEqual(p_pb.value.string_value, u'John') - self.assertEqual(p_pb.operator, datastore_pb.PropertyFilter.EQUAL) - - def test_filter___key__valid_key(self): - from gcloud.datastore.key import Key - from gcloud.datastore import test_connection + self.assertRaises(ValueError, _assign, object()) - query = self._makeOne() - key = Key('Foo', dataset_id='DATASET') - new_query = query.filter('__key__', '=', key) - - query_pb = new_query._pb - all_filters = query_pb.filter.composite_filter.filter - self.assertEqual(len(all_filters), 1) - - prop_filter = all_filters[0].property_filter - value_fields = prop_filter.value._fields - self.assertEqual(len(value_fields), 1) - field_name, field_value = value_fields.popitem() - self.assertEqual(field_name.name, 'key_value') - - test_connection._compare_key_pb_after_request( - self, key.to_protobuf(), field_value) - - def test_filter___key__invalid_value(self): - query = self._makeOne() - self.assertRaises(TypeError, query.filter, '__key__', '=', None) - - def test_ancestor_w_non_key(self): - query = self._makeOne() - self.assertRaises(TypeError, query.ancestor, object()) - self.assertRaises(TypeError, query.ancestor, ['KIND', 'NAME']) - - def test_ancestor_wo_existing_ancestor_query_w_key_and_propfilter(self): - from gcloud.datastore.key import Key - from gcloud.datastore import test_connection - - _NAME = u'NAME' - key = Key('KIND', 123, dataset_id='DATASET') - query = self._makeOne().filter('name', '=', _NAME) - after = query.ancestor(key) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.filter.composite_filter.operator, 1) # AND - n_pb, f_pb, = list(q_pb.filter.composite_filter.filter) - p_pb = n_pb.property_filter - self.assertEqual(p_pb.property.name, 'name') - self.assertEqual(p_pb.value.string_value, _NAME) - p_pb = f_pb.property_filter - self.assertEqual(p_pb.property.name, '__key__') - test_connection._compare_key_pb_after_request( - self, key.to_protobuf(), p_pb.value.key_value) - - def test_ancestor_wo_existing_ancestor_query_w_key(self): - from gcloud.datastore.key import Key - from gcloud.datastore import test_connection - - key = Key('KIND', 123, dataset_id='DATASET') - query = self._makeOne() - after = query.ancestor(key) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.filter.composite_filter.operator, 1) # AND - f_pb, = list(q_pb.filter.composite_filter.filter) - p_pb = f_pb.property_filter - self.assertEqual(p_pb.property.name, '__key__') - test_connection._compare_key_pb_after_request( - self, key.to_protobuf(), p_pb.value.key_value) - - def test_ancestor_clears_existing_ancestor_query_w_only(self): - from gcloud.datastore import _implicit_environ + def test_namespace_setter(self): from gcloud.datastore.dataset import Dataset - from gcloud.datastore.key import Key + _DATASET = 'DATASET' + _NAMESPACE = 'NAMESPACE' + dataset = Dataset(_DATASET) + query = self._makeOne(dataset=dataset) + query.namespace = _NAMESPACE + self.assertTrue(query.dataset is dataset) + self.assertEqual(query.namespace, _NAMESPACE) - _KIND = 'KIND' - _ID = 123 + def test_kind_setter_w_non_string(self): query = self._makeOne() - # All keys will have dataset attached. - _implicit_environ.DATASET = Dataset('DATASET') + def _assign(val): + query.kind = val - key = Key(_KIND, _ID) - between = query.ancestor(key) - after = between.ancestor(None) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - q_pb = after.to_protobuf() - self.assertEqual(list(q_pb.filter.composite_filter.filter), []) - - def test_ancestor_clears_existing_ancestor_query_w_others(self): - from gcloud.datastore import _implicit_environ - from gcloud.datastore.dataset import Dataset - from gcloud.datastore.key import Key - - _KIND = 'KIND' - _ID = 123 - _NAME = u'NAME' - query = self._makeOne().filter('name', '=', _NAME) - - # All keys will have dataset attached. - _implicit_environ.DATASET = Dataset('DATASET') - - key = Key(_KIND, _ID) - between = query.ancestor(key) - after = between.ancestor(None) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - q_pb = after.to_protobuf() - n_pb, = list(q_pb.filter.composite_filter.filter) - p_pb = n_pb.property_filter - self.assertEqual(p_pb.property.name, 'name') - self.assertEqual(p_pb.value.string_value, _NAME) + self.assertRaises(TypeError, _assign, object()) def test_kind_setter_wo_existing(self): from gcloud.datastore.dataset import Dataset @@ -286,11 +125,9 @@ def test_kind_setter_wo_existing(self): _KIND = 'KIND' dataset = Dataset(_DATASET) query = self._makeOne(dataset=dataset) - after = query.kind(_KIND) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - self.assertTrue(after.dataset() is dataset) - self.assertEqual(after.kind(), _KIND) + query.kind = _KIND + self.assertTrue(query.dataset is dataset) + self.assertEqual(query.kind, _KIND) def test_kind_setter_w_existing(self): from gcloud.datastore.dataset import Dataset @@ -299,306 +136,486 @@ def test_kind_setter_w_existing(self): _KIND_AFTER = 'KIND_AFTER' dataset = Dataset(_DATASET) query = self._makeOne(_KIND_BEFORE, dataset) - self.assertEqual(query.kind(), _KIND_BEFORE) - after = query.kind(_KIND_AFTER) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - self.assertTrue(after.dataset() is dataset) - self.assertEqual(after.kind(), _KIND_AFTER) - - def test_kind_getter_unset(self): - query = self._makeOne() - self.assertEqual(query.kind(), None) + self.assertEqual(query.kind, _KIND_BEFORE) + query.kind = _KIND_AFTER + self.assertTrue(query.dataset is dataset) + self.assertEqual(query.kind, _KIND_AFTER) - def test_kind_getter_bad_pb(self): + def test_ancestor_setter_w_non_key(self): query = self._makeOne() - query._pb.kind.add().name = 'foo' - query._pb.kind.add().name = 'bar' - self.assertRaises(ValueError, query.kind) - def test_limit_setter_wo_existing(self): - from gcloud.datastore.dataset import Dataset - _DATASET = 'DATASET' - _KIND = 'KIND' - _LIMIT = 42 - dataset = Dataset(_DATASET) - query = self._makeOne(_KIND, dataset) - after = query.limit(_LIMIT) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - self.assertTrue(after.dataset() is dataset) - self.assertEqual(after.limit(), _LIMIT) - self.assertEqual(after.kind(), _KIND) + def _assign(val): + query.ancestor = val - def test_dataset_setter(self): - from gcloud.datastore.dataset import Dataset - _DATASET = 'DATASET' - _KIND = 'KIND' - dataset = Dataset(_DATASET) - query = self._makeOne(_KIND) - after = query.dataset(dataset) - self.assertFalse(after is query) - self.assertTrue(isinstance(after, self._getTargetClass())) - self.assertTrue(after.dataset() is dataset) - self.assertEqual(query.kind(), _KIND) - - def _fetch_page_helper(self, cursor=b'\x00', limit=None, - more_results=False, _more_pb=None, - use_fetch=False): - import base64 - from gcloud.datastore.datastore_v1_pb2 import Entity - _CURSOR_FOR_USER = (None if cursor is None - else base64.b64encode(cursor)) - _MORE_RESULTS = more_results - _DATASET = 'DATASET' - _KIND = 'KIND' - _ID = 123 - _NAMESPACE = 'NAMESPACE' - entity_pb = Entity() - entity_pb.key.partition_id.dataset_id = _DATASET - path_element = entity_pb.key.path_element.add() - path_element.kind = _KIND - path_element.id = _ID - prop = entity_pb.property.add() - prop.name = 'foo' - prop.value.string_value = u'Foo' - if _more_pb is None: - connection = _Connection(entity_pb) - else: - connection = _Connection(entity_pb, _more=_more_pb) - connection._cursor = cursor - dataset = _Dataset(_DATASET, connection) - - query = self._makeOne(_KIND, dataset, _NAMESPACE) - if use_fetch: - entities = query.fetch(limit) - else: - entities, cursor, more_results = query.fetch_page(limit) - self.assertEqual(cursor, _CURSOR_FOR_USER) - self.assertEqual(more_results, _MORE_RESULTS) + self.assertRaises(TypeError, _assign, object()) + self.assertRaises(TypeError, _assign, ['KIND', 'NAME']) - self.assertEqual(len(entities), 1) - self.assertEqual(entities[0].key().path, - [{'kind': _KIND, 'id': _ID}]) - limited_query = query - if limit is not None: - limited_query = query.limit(limit) - expected_called_with = { - 'dataset_id': _DATASET, - 'query_pb': limited_query.to_protobuf(), - 'namespace': _NAMESPACE, - } - self.assertEqual(connection._called_with, expected_called_with) + def test_ancestor_setter_w_key(self): + from gcloud.datastore.key import Key + _NAME = u'NAME' + key = Key('KIND', 123, dataset_id='DATASET') + query = self._makeOne() + query.add_filter('name', '=', _NAME) + query.ancestor = key + self.assertEqual(query.ancestor.path, key.path) - def test_fetch_page_default_limit(self): - self._fetch_page_helper() + def test_ancestor_deleter_w_key(self): + from gcloud.datastore.key import Key + key = Key('KIND', 123, dataset_id='DATASET') + query = self._makeOne(ancestor=key) + del query.ancestor + self.assertTrue(query.ancestor is None) - def test_fetch_defaults(self): - self._fetch_page_helper(use_fetch=True) + def test_add_filter_setter_w_unknown_operator(self): + query = self._makeOne() + self.assertRaises(ValueError, query.add_filter, + 'firstname', '~~', 'John') - def test_fetch_page_explicit_limit(self): - self._fetch_page_helper(cursor='CURSOR', limit=13) + def test_add_filter_w_known_operator(self): + query = self._makeOne() + query.add_filter('firstname', '=', u'John') + self.assertEqual(query.filters, [('firstname', '=', u'John')]) - def test_fetch_page_no_more_results(self): - from gcloud.datastore import datastore_v1_pb2 as datastore_pb - no_more = datastore_pb.QueryResultBatch.NO_MORE_RESULTS - self._fetch_page_helper(cursor='CURSOR', limit=13, _more_pb=no_more) + def test_add_filter_w_all_operators(self): + query = self._makeOne() + query.add_filter('leq_prop', '<=', u'val1') + query.add_filter('geq_prop', '>=', u'val2') + query.add_filter('lt_prop', '<', u'val3') + query.add_filter('gt_prop', '>', u'val4') + query.add_filter('eq_prop', '=', u'val5') + self.assertEqual(len(query.filters), 5) + self.assertEqual(query.filters[0], ('leq_prop', '<=', u'val1')) + self.assertEqual(query.filters[1], ('geq_prop', '>=', u'val2')) + self.assertEqual(query.filters[2], ('lt_prop', '<', u'val3')) + self.assertEqual(query.filters[3], ('gt_prop', '>', u'val4')) + self.assertEqual(query.filters[4], ('eq_prop', '=', u'val5')) + + def test_add_filter_w_known_operator_and_entity(self): + from gcloud.datastore.entity import Entity + query = self._makeOne() + other = Entity() + other['firstname'] = u'John' + other['lastname'] = u'Smith' + query.add_filter('other', '=', other) + self.assertEqual(query.filters, [('other', '=', other)]) - def test_fetch_page_not_finished(self): - from gcloud.datastore import datastore_v1_pb2 as datastore_pb - not_finished = datastore_pb.QueryResultBatch.NOT_FINISHED - self._fetch_page_helper(_more_pb=not_finished, more_results=True) + def test_add_filter_w_whitespace_property_name(self): + query = self._makeOne() + PROPERTY_NAME = ' property with lots of space ' + query.add_filter(PROPERTY_NAME, '=', u'John') + self.assertEqual(query.filters, [(PROPERTY_NAME, '=', u'John')]) - def test_fetch_page_more_results_invalid(self): - self.assertRaises(ValueError, self._fetch_page_helper, - _more_pb=object()) + def test_add_filter___key__valid_key(self): + from gcloud.datastore.key import Key + query = self._makeOne() + key = Key('Foo', dataset_id='DATASET') + query.add_filter('__key__', '=', key) + self.assertEqual(query.filters, [('__key__', '=', key)]) - def test_with_cursor_neither(self): - _DATASET = 'DATASET' - _KIND = 'KIND' - connection = _Connection() - dataset = _Dataset(_DATASET, connection) - query = self._makeOne(_KIND, dataset) - self.assertTrue(query.with_cursor(None) is query) - - def test_with_cursor_w_start(self): - import base64 - _CURSOR = 'CURSOR' - _CURSOR_B64 = base64.b64encode(_CURSOR) - _DATASET = 'DATASET' - _KIND = 'KIND' - connection = _Connection() - dataset = _Dataset(_DATASET, connection) - query = self._makeOne(_KIND, dataset) - after = query.with_cursor(_CURSOR_B64) - self.assertFalse(after is query) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.start_cursor, _CURSOR) - self.assertEqual(after.start_cursor, _CURSOR_B64) - self.assertEqual(q_pb.end_cursor, '') - self.assertEqual(after.end_cursor, None) - - def test_with_cursor_w_end(self): - import base64 - _CURSOR = 'CURSOR' - _CURSOR_B64 = base64.b64encode(_CURSOR) - _DATASET = 'DATASET' - _KIND = 'KIND' - connection = _Connection() - dataset = _Dataset(_DATASET, connection) - query = self._makeOne(_KIND, dataset) - after = query.with_cursor(None, _CURSOR_B64) - self.assertFalse(after is query) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.start_cursor, '') - self.assertEqual(after.start_cursor, None) - self.assertEqual(q_pb.end_cursor, _CURSOR) - self.assertEqual(after.end_cursor, _CURSOR_B64) - - def test_with_cursor_w_both(self): - import base64 - _START = 'START' - _START_B64 = base64.b64encode(_START) - _END = 'CURSOR' - _END_B64 = base64.b64encode(_END) - _DATASET = 'DATASET' - _KIND = 'KIND' - connection = _Connection() - dataset = _Dataset(_DATASET, connection) - query = self._makeOne(_KIND, dataset) - after = query.with_cursor(_START_B64, _END_B64) - self.assertFalse(after is query) - q_pb = after.to_protobuf() - self.assertEqual(q_pb.start_cursor, _START) - self.assertEqual(after.start_cursor, _START_B64) - self.assertEqual(q_pb.end_cursor, _END) - self.assertEqual(after.end_cursor, _END_B64) - - def test_order_empty(self): - _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.order() - self.assertFalse(after is before) - self.assertTrue(isinstance(after, self._getTargetClass())) - self.assertEqual(before.to_protobuf(), after.to_protobuf()) + def test_filter___key__invalid_operator(self): + from gcloud.datastore.key import Key + key = Key('Foo', dataset_id='DATASET') + query = self._makeOne() + self.assertRaises(ValueError, query.add_filter, '__key__', '<', key) - def test_order_single_asc(self): - _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.order('field') - after_pb = after.to_protobuf() - order_pb = list(after_pb.order) - self.assertEqual(len(order_pb), 1) - prop_pb = order_pb[0] - self.assertEqual(prop_pb.property.name, 'field') - self.assertEqual(prop_pb.direction, prop_pb.ASCENDING) - - def test_order_single_desc(self): - _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.order('-field') - after_pb = after.to_protobuf() - order_pb = list(after_pb.order) - self.assertEqual(len(order_pb), 1) - prop_pb = order_pb[0] - self.assertEqual(prop_pb.property.name, 'field') - self.assertEqual(prop_pb.direction, prop_pb.DESCENDING) - - def test_order_multiple(self): - _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.order('foo', '-bar') - after_pb = after.to_protobuf() - order_pb = list(after_pb.order) - self.assertEqual(len(order_pb), 2) - prop_pb = order_pb[0] - self.assertEqual(prop_pb.property.name, 'foo') - self.assertEqual(prop_pb.direction, prop_pb.ASCENDING) - prop_pb = order_pb[1] - self.assertEqual(prop_pb.property.name, 'bar') - self.assertEqual(prop_pb.direction, prop_pb.DESCENDING) - - def test_projection_empty(self): + def test_filter___key__invalid_value(self): + query = self._makeOne() + self.assertRaises(ValueError, query.add_filter, '__key__', '=', None) + + def test_projection_setter_empty(self): _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.projection([]) - self.assertFalse(after is before) - self.assertTrue(isinstance(after, self._getTargetClass())) - self.assertEqual(before.to_protobuf(), after.to_protobuf()) + query = self._makeOne(_KIND) + query.projection = [] + self.assertEqual(query.projection, []) - def test_projection_non_empty(self): + def test_projection_setter_string(self): _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.projection(['field1', 'field2']) - projection_pb = list(after.to_protobuf().projection) - self.assertEqual(len(projection_pb), 2) - prop_pb1 = projection_pb[0] - self.assertEqual(prop_pb1.property.name, 'field1') - prop_pb2 = projection_pb[1] - self.assertEqual(prop_pb2.property.name, 'field2') - - def test_get_projection_non_empty(self): + query = self._makeOne(_KIND) + query.projection = 'field1' + self.assertEqual(query.projection, ['field1']) + + def test_projection_setter_non_empty(self): _KIND = 'KIND' - _PROJECTION = ['field1', 'field2'] - after = self._makeOne(_KIND).projection(_PROJECTION) - self.assertEqual(after.projection(), _PROJECTION) + query = self._makeOne(_KIND) + query.projection = ['field1', 'field2'] + self.assertEqual(query.projection, ['field1', 'field2']) - def test_projection_multiple_calls(self): + def test_projection_setter_multiple_calls(self): _KIND = 'KIND' _PROJECTION1 = ['field1', 'field2'] _PROJECTION2 = ['field3'] - before = self._makeOne(_KIND).projection(_PROJECTION1) - self.assertEqual(before.projection(), _PROJECTION1) - after = before.projection(_PROJECTION2) - self.assertEqual(after.projection(), _PROJECTION2) + query = self._makeOne(_KIND) + query.projection = _PROJECTION1 + self.assertEqual(query.projection, _PROJECTION1) + query.projection = _PROJECTION2 + self.assertEqual(query.projection, _PROJECTION2) - def test_set_offset(self): + def test_order_setter_empty(self): _KIND = 'KIND' - _OFFSET = 42 - before = self._makeOne(_KIND) - after = before.offset(_OFFSET) - offset_pb = after.to_protobuf().offset - self.assertEqual(offset_pb, _OFFSET) + query = self._makeOne(_KIND, order=['foo', '-bar']) + query.order = [] + self.assertEqual(query.order, []) - def test_get_offset(self): + def test_order_setter_string(self): _KIND = 'KIND' - _OFFSET = 10 - after = self._makeOne(_KIND).offset(_OFFSET) - self.assertEqual(after.offset(), _OFFSET) + query = self._makeOne(_KIND) + query.order = 'field' + self.assertEqual(query.order, ['field']) - def test_group_by_empty(self): + def test_order_setter_single_item_list_desc(self): _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.group_by([]) - self.assertFalse(after is before) - self.assertTrue(isinstance(after, self._getTargetClass())) - self.assertEqual(before.to_protobuf(), after.to_protobuf()) + query = self._makeOne(_KIND) + query.order = ['-field'] + self.assertEqual(query.order, ['-field']) - def test_group_by_non_empty(self): + def test_order_setter_multiple(self): _KIND = 'KIND' - before = self._makeOne(_KIND) - after = before.group_by(['field1', 'field2']) - group_by_pb = list(after.to_protobuf().group_by) - self.assertEqual(len(group_by_pb), 2) - prop_pb1 = group_by_pb[0] - self.assertEqual(prop_pb1.name, 'field1') - prop_pb2 = group_by_pb[1] - self.assertEqual(prop_pb2.name, 'field2') - - def test_get_group_by_non_empty(self): + query = self._makeOne(_KIND) + query.order = ['foo', '-bar'] + self.assertEqual(query.order, ['foo', '-bar']) + + def test_group_by_setter_empty(self): + _KIND = 'KIND' + query = self._makeOne(_KIND, group_by=['foo', 'bar']) + query.group_by = [] + self.assertEqual(query.group_by, []) + + def test_group_by_setter_string(self): _KIND = 'KIND' - _GROUP_BY = ['field1', 'field2'] - after = self._makeOne(_KIND).group_by(_GROUP_BY) - self.assertEqual(after.group_by(), _GROUP_BY) + query = self._makeOne(_KIND) + query.group_by = 'field1' + self.assertEqual(query.group_by, ['field1']) + + def test_group_by_setter_non_empty(self): + _KIND = 'KIND' + query = self._makeOne(_KIND) + query.group_by = ['field1', 'field2'] + self.assertEqual(query.group_by, ['field1', 'field2']) def test_group_by_multiple_calls(self): _KIND = 'KIND' _GROUP_BY1 = ['field1', 'field2'] _GROUP_BY2 = ['field3'] - before = self._makeOne(_KIND).group_by(_GROUP_BY1) - self.assertEqual(before.group_by(), _GROUP_BY1) - after = before.group_by(_GROUP_BY2) - self.assertEqual(after.group_by(), _GROUP_BY2) + query = self._makeOne(_KIND) + query.group_by = _GROUP_BY1 + self.assertEqual(query.group_by, _GROUP_BY1) + query.group_by = _GROUP_BY2 + self.assertEqual(query.group_by, _GROUP_BY2) + + def test_fetch_defaults(self): + _KIND = 'KIND' + query = self._makeOne(_KIND) + iterator = query.fetch() + self.assertTrue(iterator._query is query) + self.assertEqual(iterator._limit, 0) + self.assertEqual(iterator._offset, 0) + + def test_fetch_explicit(self): + _KIND = 'KIND' + query = self._makeOne(_KIND) + iterator = query.fetch(limit=7, offset=8) + self.assertTrue(iterator._query is query) + self.assertEqual(iterator._limit, 7) + self.assertEqual(iterator._offset, 8) + + +class Test__pb_from_query(unittest2.TestCase): + + def _callFUT(self, query): + from gcloud.datastore.query import _pb_from_query + return _pb_from_query(query) + + def test_empty(self): + from gcloud.datastore import datastore_v1_pb2 as datastore_pb + pb = self._callFUT(_Query()) + self.assertEqual(list(pb.projection), []) + self.assertEqual(list(pb.kind), []) + self.assertEqual(list(pb.order), []) + self.assertEqual(list(pb.group_by), []) + self.assertEqual(pb.filter.property_filter.property.name, '') + cfilter = pb.filter.composite_filter + self.assertEqual(cfilter.operator, datastore_pb.CompositeFilter.AND) + self.assertEqual(list(cfilter.filter), []) + self.assertEqual(pb.start_cursor, b'') + self.assertEqual(pb.end_cursor, b'') + self.assertEqual(pb.limit, 0) + self.assertEqual(pb.offset, 0) + + def test_projection(self): + pb = self._callFUT(_Query(projection=['a', 'b', 'c'])) + self.assertEqual([item.property.name for item in pb.projection], + ['a', 'b', 'c']) + + def test_kind(self): + pb = self._callFUT(_Query(kind='KIND')) + self.assertEqual([item.name for item in pb.kind], ['KIND']) + + def test_ancestor(self): + from gcloud.datastore import datastore_v1_pb2 as datastore_pb + from gcloud.datastore.key import Key + from gcloud.datastore.helpers import _prepare_key_for_request + ancestor = Key('Ancestor', 123, dataset_id='DATASET') + pb = self._callFUT(_Query(ancestor=ancestor)) + cfilter = pb.filter.composite_filter + self.assertEqual(cfilter.operator, datastore_pb.CompositeFilter.AND) + self.assertEqual(len(cfilter.filter), 1) + pfilter = cfilter.filter[0].property_filter + self.assertEqual(pfilter.property.name, '__key__') + ancestor_pb = _prepare_key_for_request(ancestor.to_protobuf()) + self.assertEqual(pfilter.value.key_value, ancestor_pb) + + def test_filter(self): + from gcloud.datastore import datastore_v1_pb2 as datastore_pb + query = _Query(filters=[('name', '=', u'John')]) + query.OPERATORS = { + '=': datastore_pb.PropertyFilter.EQUAL, + } + pb = self._callFUT(query) + cfilter = pb.filter.composite_filter + self.assertEqual(cfilter.operator, datastore_pb.CompositeFilter.AND) + self.assertEqual(len(cfilter.filter), 1) + pfilter = cfilter.filter[0].property_filter + self.assertEqual(pfilter.property.name, 'name') + self.assertEqual(pfilter.value.string_value, u'John') + + def test_filter_key(self): + from gcloud.datastore import datastore_v1_pb2 as datastore_pb + from gcloud.datastore.key import Key + from gcloud.datastore.helpers import _prepare_key_for_request + key = Key('Kind', 123, dataset_id='DATASET') + query = _Query(filters=[('__key__', '=', key)]) + query.OPERATORS = { + '=': datastore_pb.PropertyFilter.EQUAL, + } + pb = self._callFUT(query) + cfilter = pb.filter.composite_filter + self.assertEqual(cfilter.operator, datastore_pb.CompositeFilter.AND) + self.assertEqual(len(cfilter.filter), 1) + pfilter = cfilter.filter[0].property_filter + self.assertEqual(pfilter.property.name, '__key__') + key_pb = _prepare_key_for_request(key.to_protobuf()) + self.assertEqual(pfilter.value.key_value, key_pb) + + def test_order(self): + from gcloud.datastore import datastore_v1_pb2 as datastore_pb + pb = self._callFUT(_Query(order=['a', '-b', 'c'])) + self.assertEqual([item.property.name for item in pb.order], + ['a', 'b', 'c']) + self.assertEqual([item.direction for item in pb.order], + [datastore_pb.PropertyOrder.ASCENDING, + datastore_pb.PropertyOrder.DESCENDING, + datastore_pb.PropertyOrder.ASCENDING]) + + def test_group_by(self): + pb = self._callFUT(_Query(group_by=['a', 'b', 'c'])) + self.assertEqual([item.name for item in pb.group_by], + ['a', 'b', 'c']) + + +class TestIterator(unittest2.TestCase): + _DATASET = 'DATASET' + _NAMESPACE = 'NAMESPACE' + _KIND = 'KIND' + _ID = 123 + _START = b'\x00' + _END = b'\xFF' + + def setUp(self): + from gcloud.datastore import _implicit_environ + self._replaced_dataset = _implicit_environ.DATASET + _implicit_environ.DATASET = None + + def tearDown(self): + from gcloud.datastore import _implicit_environ + _implicit_environ.DATASET = self._replaced_dataset + + def _getTargetClass(self): + from gcloud.datastore.query import Iterator + return Iterator + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def _makeDataset(self): + connection = _Connection() + dataset = _Dataset(self._DATASET, connection) + return dataset, connection + + def _addQueryResults(self, dataset, cursor=_END, more=False): + from gcloud.datastore import datastore_v1_pb2 as datastore_pb + MORE = datastore_pb.QueryResultBatch.NOT_FINISHED + NO_MORE = datastore_pb.QueryResultBatch.MORE_RESULTS_AFTER_LIMIT + _ID = 123 + entity_pb = datastore_pb.Entity() + entity_pb.key.partition_id.dataset_id = dataset.id() + path_element = entity_pb.key.path_element.add() + path_element.kind = self._KIND + path_element.id = _ID + prop = entity_pb.property.add() + prop.name = 'foo' + prop.value.string_value = u'Foo' + dataset.connection()._results.append( + ([entity_pb], cursor, MORE if more else NO_MORE)) + + def test_ctor_defaults(self): + query = object() + iterator = self._makeOne(query) + self.assertTrue(iterator._query is query) + self.assertEqual(iterator._limit, 0) + self.assertEqual(iterator._offset, 0) + + def test_ctor_explicit(self): + query = object() + iterator = self._makeOne(query, 13, 29) + self.assertTrue(iterator._query is query) + self.assertEqual(iterator._limit, 13) + self.assertEqual(iterator._offset, 29) + + def test_next_page_no_cursors_no_more(self): + from base64 import b64encode + from gcloud.datastore.query import _pb_from_query + self._KIND = 'KIND' + dataset, connection = self._makeDataset() + query = _Query(self._KIND, dataset, self._NAMESPACE) + self._addQueryResults(dataset) + iterator = self._makeOne(query) + entities, more_results, cursor = iterator.next_page() + + self.assertEqual(cursor, b64encode(self._END)) + self.assertFalse(more_results) + self.assertFalse(iterator._more_results) + self.assertEqual(len(entities), 1) + self.assertEqual(entities[0].key().path, + [{'kind': self._KIND, 'id': self._ID}]) + self.assertEqual(entities[0]['foo'], u'Foo') + qpb = _pb_from_query(query) + qpb.limit = qpb.offset = 0 + EXPECTED = { + 'dataset_id': self._DATASET, + 'query_pb': qpb, + 'namespace': self._NAMESPACE, + } + self.assertEqual(connection._called_with, [EXPECTED]) + + def test_next_page_w_cursors_w_more(self): + from base64 import b64decode + from base64 import b64encode + from gcloud.datastore.query import _pb_from_query + dataset, connection = self._makeDataset() + query = _Query(self._KIND, dataset, self._NAMESPACE) + self._addQueryResults(dataset, cursor=self._END, more=True) + iterator = self._makeOne(query) + iterator._start_cursor = self._START + iterator._end_cursor = self._END + entities, more_results, cursor = iterator.next_page() + + self.assertEqual(cursor, b64encode(self._END)) + self.assertTrue(more_results) + self.assertTrue(iterator._more_results) + self.assertEqual(iterator._end_cursor, None) + self.assertEqual(b64decode(iterator._start_cursor), self._END) + self.assertEqual(len(entities), 1) + self.assertEqual(entities[0].key().path, + [{'kind': self._KIND, 'id': self._ID}]) + self.assertEqual(entities[0]['foo'], u'Foo') + qpb = _pb_from_query(query) + qpb.limit = qpb.offset = 0 + qpb.start_cursor = b64decode(self._START) + qpb.end_cursor = b64decode(self._END) + EXPECTED = { + 'dataset_id': self._DATASET, + 'query_pb': qpb, + 'namespace': self._NAMESPACE, + } + self.assertEqual(connection._called_with, [EXPECTED]) + + def test_next_page_w_cursors_w_bogus_more(self): + dataset, connection = self._makeDataset() + query = _Query(self._KIND, dataset, self._NAMESPACE) + self._addQueryResults(dataset, cursor=self._END, more=True) + epb, cursor, _ = connection._results.pop() + connection._results.append((epb, cursor, 4)) # invalid enum + iterator = self._makeOne(query) + self.assertRaises(ValueError, iterator.next_page) + + def test___iter___no_more(self): + from gcloud.datastore.query import _pb_from_query + self._KIND = 'KIND' + dataset, connection = self._makeDataset() + query = _Query(self._KIND, dataset, self._NAMESPACE) + self._addQueryResults(dataset) + iterator = self._makeOne(query) + entities = list(iterator) + + self.assertFalse(iterator._more_results) + self.assertEqual(len(entities), 1) + self.assertEqual(entities[0].key().path, + [{'kind': self._KIND, 'id': self._ID}]) + self.assertEqual(entities[0]['foo'], u'Foo') + qpb = _pb_from_query(query) + qpb.limit = qpb.offset = 0 + EXPECTED = { + 'dataset_id': self._DATASET, + 'query_pb': qpb, + 'namespace': self._NAMESPACE, + } + self.assertEqual(connection._called_with, [EXPECTED]) + + def test___iter___w_more(self): + from gcloud.datastore.query import _pb_from_query + dataset, connection = self._makeDataset() + query = _Query(self._KIND, dataset, self._NAMESPACE) + self._addQueryResults(dataset, cursor=self._END, more=True) + self._addQueryResults(dataset) + iterator = self._makeOne(query) + entities = list(iterator) + + self.assertFalse(iterator._more_results) + self.assertEqual(len(entities), 2) + for entity in entities: + self.assertEqual( + entity.key().path, + [{'kind': self._KIND, 'id': self._ID}]) + self.assertEqual(entities[1]['foo'], u'Foo') + qpb1 = _pb_from_query(query) + qpb1.limit = qpb1.offset = 0 + qpb2 = _pb_from_query(query) + qpb2.limit = qpb2.offset = 0 + qpb2.start_cursor = self._END + EXPECTED1 = { + 'dataset_id': self._DATASET, + 'query_pb': qpb1, + 'namespace': self._NAMESPACE, + } + EXPECTED2 = { + 'dataset_id': self._DATASET, + 'query_pb': qpb2, + 'namespace': self._NAMESPACE, + } + self.assertEqual(len(connection._called_with), 2) + self.assertEqual(connection._called_with[0], EXPECTED1) + self.assertEqual(connection._called_with[1], EXPECTED2) + + +class _Query(object): + + def __init__(self, + kind=None, + dataset=None, + namespace=None, + ancestor=None, + filters=(), + projection=(), + order=(), + group_by=()): + self.kind = kind + self.dataset = dataset + self.namespace = namespace + self.ancestor = ancestor + self.filters = filters + self.projection = projection + self.order = order + self.group_by = group_by class _Dataset(object): @@ -620,13 +637,11 @@ class _Connection(object): _cursor = b'\x00' _skipped = 0 - def __init__(self, *result, **kwargs): - from gcloud.datastore import datastore_v1_pb2 as datastore_pb - - self._result = list(result) - more_default = datastore_pb.QueryResultBatch.MORE_RESULTS_AFTER_LIMIT - self._more = kwargs.get('_more', more_default) + def __init__(self): + self._results = [] + self._called_with = [] def run_query(self, **kw): - self._called_with = kw - return self._result, self._cursor, self._more, self._skipped + self._called_with.append(kw) + result, self._results = self._results[0], self._results[1:] + return result diff --git a/regression/clear_datastore.py b/regression/clear_datastore.py index ecc75ada54b4..7dc72fa5df27 100644 --- a/regression/clear_datastore.py +++ b/regression/clear_datastore.py @@ -17,6 +17,7 @@ # This assumes the command is being run via tox hence the # repository root is the current directory. +from gcloud.datastore.query import Query from regression import regression_utils from six.moves import input @@ -34,12 +35,11 @@ def fetch_keys(dataset, kind, fetch_max=FETCH_MAX, query=None, cursor=None): if query is None: - query = dataset.query(kind=kind).limit( - fetch_max).projection(['__key__']) - # Make new query with start cursor. Will be ignored if None. - query = query.with_cursor(cursor) + query = Query(kind=kind, dataset=dataset, projection=['__key__']) - entities, cursor, _ = query.fetch_page() + iterator = query.fetch(limit=fetch_max, start_cursor=cursor) + + entities, _, cursor = iterator.next_page() return query, entities, cursor diff --git a/regression/datastore.py b/regression/datastore.py index 87b23576c4e6..2bd4c20b8bf1 100644 --- a/regression/datastore.py +++ b/regression/datastore.py @@ -18,6 +18,10 @@ import unittest2 from gcloud import datastore +from gcloud.datastore.entity import Entity +from gcloud.datastore.key import Key +from gcloud.datastore.query import Query +from gcloud.datastore.transaction import Transaction # This assumes the command is being run via tox hence the # repository root is the current directory. from regression import populate_datastore @@ -33,7 +37,7 @@ def setUp(self): self.case_entities_to_delete = [] def tearDown(self): - with datastore.transaction.Transaction(): + with Transaction(): for entity in self.case_entities_to_delete: entity.delete() @@ -41,7 +45,7 @@ def tearDown(self): class TestDatastoreAllocateIDs(TestDatastore): def test_allocate_ids(self): - incomplete_key = datastore.key.Key('Kind') + incomplete_key = Key('Kind') num_ids = 10 allocated_keys = datastore.allocate_ids(incomplete_key, num_ids) self.assertEqual(len(allocated_keys), num_ids) @@ -68,7 +72,7 @@ def _get_post(self, name=None, key_id=None, post_content=None): 'rating': 5.0, } # Create an entity with the given content. - entity = datastore.entity.Entity(kind='Post') + entity = Entity(kind='Post') entity.update(post_content) # Update the entity key. @@ -114,7 +118,7 @@ def test_post_with_generated_id(self): self._generic_test_post() def test_save_multiple(self): - with datastore.transaction.Transaction(): + with Transaction(): entity1 = self._get_post() entity1.save() # Register entity to be deleted. @@ -139,25 +143,26 @@ def test_save_multiple(self): self.assertEqual(len(matches), 2) def test_empty_kind(self): - posts = datastore.query.Query(kind='Post').limit(2).fetch() + query = Query(kind='Post') + posts = list(query.fetch(limit=2)) self.assertEqual(posts, []) class TestDatastoreSaveKeys(TestDatastore): def test_save_key_self_reference(self): - key = datastore.key.Key('Person', 'name') - entity = datastore.entity.Entity(kind=None).key(key) + key = Key('Person', 'name') + entity = Entity(kind=None).key(key) entity['fullName'] = u'Full name' entity['linkedTo'] = key # Self reference. entity.save() self.case_entities_to_delete.append(entity) - query = datastore.query.Query(kind='Person').filter( - 'linkedTo', '=', key).limit(2) + query = Query(kind='Person') + query.add_filter('linkedTo', '=', key) - stored_persons = query.fetch() + stored_persons = list(query.fetch(limit=2)) self.assertEqual(len(stored_persons), 1) stored_person = stored_persons[0] @@ -172,42 +177,43 @@ class TestDatastoreQuery(TestDatastore): def setUpClass(cls): super(TestDatastoreQuery, cls).setUpClass() cls.CHARACTERS = populate_datastore.CHARACTERS - cls.ANCESTOR_KEY = datastore.key.Key(*populate_datastore.ANCESTOR) + cls.ANCESTOR_KEY = Key(*populate_datastore.ANCESTOR) def _base_query(self): - return datastore.query.Query(kind='Character').ancestor( - self.ANCESTOR_KEY) + return Query(kind='Character', ancestor=self.ANCESTOR_KEY) def test_limit_queries(self): limit = 5 - query = self._base_query().limit(limit) + query = self._base_query() # Fetch characters. - character_entities, cursor, _ = query.fetch_page() + iterator = query.fetch(limit=limit) + character_entities, _, cursor = iterator.next_page() self.assertEqual(len(character_entities), limit) # Check cursor after fetch. self.assertTrue(cursor is not None) - # Fetch next batch of characters. - new_query = self._base_query().with_cursor(cursor) - new_character_entities = new_query.fetch() + # Fetch remaining of characters. + new_character_entities = list(iterator) characters_remaining = len(self.CHARACTERS) - limit self.assertEqual(len(new_character_entities), characters_remaining) def test_query_simple_filter(self): - query = self._base_query().filter('appearances', '>=', 20) + query = self._base_query() + query.add_filter('appearances', '>=', 20) expected_matches = 6 # We expect 6, but allow the query to get 1 extra. - entities = query.fetch(limit=expected_matches + 1) + entities = list(query.fetch(limit=expected_matches + 1)) self.assertEqual(len(entities), expected_matches) def test_query_multiple_filters(self): - query = self._base_query().filter( - 'appearances', '>=', 26).filter('family', '=', 'Stark') + query = self._base_query() + query.add_filter('appearances', '>=', 26) + query.add_filter('family', '=', 'Stark') expected_matches = 4 # We expect 4, but allow the query to get 1 extra. - entities = query.fetch(limit=expected_matches + 1) + entities = list(query.fetch(limit=expected_matches + 1)) self.assertEqual(len(entities), expected_matches) def test_ancestor_query(self): @@ -215,23 +221,25 @@ def test_ancestor_query(self): expected_matches = 8 # We expect 8, but allow the query to get 1 extra. - entities = filtered_query.fetch(limit=expected_matches + 1) + entities = list(filtered_query.fetch(limit=expected_matches + 1)) self.assertEqual(len(entities), expected_matches) def test_query___key___filter(self): - rickard_key = datastore.key.Key(*populate_datastore.RICKARD) + rickard_key = Key(*populate_datastore.RICKARD) - query = self._base_query().filter('__key__', '=', rickard_key) + query = self._base_query() + query.add_filter('__key__', '=', rickard_key) expected_matches = 1 # We expect 1, but allow the query to get 1 extra. - entities = query.fetch(limit=expected_matches + 1) + entities = list(query.fetch(limit=expected_matches + 1)) self.assertEqual(len(entities), expected_matches) def test_ordered_query(self): - query = self._base_query().order('appearances') + query = self._base_query() + query.order = 'appearances' expected_matches = 8 # We expect 8, but allow the query to get 1 extra. - entities = query.fetch(limit=expected_matches + 1) + entities = list(query.fetch(limit=expected_matches + 1)) self.assertEqual(len(entities), expected_matches) # Actually check the ordered data returned. @@ -239,14 +247,15 @@ def test_ordered_query(self): self.assertEqual(entities[7]['name'], self.CHARACTERS[3]['name']) def test_projection_query(self): - filtered_query = self._base_query().projection(['name', 'family']) + filtered_query = self._base_query() + filtered_query.projection = ['name', 'family'] # NOTE: There are 9 responses because of Catelyn. She has both # Stark and Tully as her families, hence occurs twice in # the results. expected_matches = 9 # We expect 9, but allow the query to get 1 extra. - entities = filtered_query.fetch(limit=expected_matches + 1) + entities = list(filtered_query.fetch(limit=expected_matches + 1)) self.assertEqual(len(entities), expected_matches) arya_entity = entities[0] @@ -278,54 +287,57 @@ def test_projection_query(self): self.assertEqual(sansa_dict, {'name': 'Sansa', 'family': 'Stark'}) def test_query_paginate_with_offset(self): - query = self._base_query() + page_query = self._base_query() + page_query.order = 'appearances' offset = 2 limit = 3 - page_query = query.offset(offset).limit(limit).order('appearances') + iterator = page_query.fetch(limit=limit, offset=offset) # Fetch characters. - entities, cursor, _ = page_query.fetch_page() + entities, _, cursor = iterator.next_page() self.assertEqual(len(entities), limit) self.assertEqual(entities[0]['name'], 'Robb') self.assertEqual(entities[1]['name'], 'Bran') self.assertEqual(entities[2]['name'], 'Catelyn') - # Use cursor to begin next query. - next_query = page_query.with_cursor(cursor).offset(0) - self.assertEqual(next_query.limit(), limit) # Fetch next set of characters. - entities = next_query.fetch() + new_iterator = page_query.fetch(limit=limit, offset=0, + start_cursor=cursor) + entities = list(new_iterator) self.assertEqual(len(entities), limit) self.assertEqual(entities[0]['name'], 'Sansa') self.assertEqual(entities[1]['name'], 'Jon Snow') self.assertEqual(entities[2]['name'], 'Arya') def test_query_paginate_with_start_cursor(self): - query = self._base_query() + page_query = self._base_query() + page_query.order = 'appearances' + limit = 3 offset = 2 - limit = 2 - page_query = query.offset(offset).limit(limit).order('appearances') + iterator = page_query.fetch(limit=limit, offset=offset) # Fetch characters. - entities, cursor, _ = page_query.fetch_page() + entities, _, cursor = iterator.next_page() self.assertEqual(len(entities), limit) # Use cursor to create a fresh query. fresh_query = self._base_query() - fresh_query = fresh_query.order('appearances').with_cursor(cursor) + fresh_query.order = 'appearances' - new_entities = fresh_query.fetch() + new_entities = list(fresh_query.fetch(start_cursor=cursor, + limit=limit)) characters_remaining = len(self.CHARACTERS) - limit - offset self.assertEqual(len(new_entities), characters_remaining) - self.assertEqual(new_entities[0]['name'], 'Catelyn') - self.assertEqual(new_entities[3]['name'], 'Arya') + self.assertEqual(new_entities[0]['name'], 'Sansa') + self.assertEqual(new_entities[2]['name'], 'Arya') def test_query_group_by(self): - query = self._base_query().group_by(['alive']) + query = self._base_query() + query.group_by = ['alive'] expected_matches = 2 # We expect 2, but allow the query to get 1 extra. - entities = query.fetch(limit=expected_matches + 1) + entities = list(query.fetch(limit=expected_matches + 1)) self.assertEqual(len(entities), expected_matches) self.assertEqual(entities[0]['name'], 'Catelyn') @@ -335,11 +347,11 @@ def test_query_group_by(self): class TestDatastoreTransaction(TestDatastore): def test_transaction(self): - key = datastore.key.Key('Company', 'Google') - entity = datastore.entity.Entity(kind=None).key(key) + key = Key('Company', 'Google') + entity = Entity(kind=None).key(key) entity['url'] = u'www.google.com' - with datastore.transaction.Transaction(): + with Transaction(): retrieved_entity = datastore.get_entity(key) if retrieved_entity is None: entity.save()