From 58ae075463518e477185816094eb83f42ce5b77c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 Aug 2015 11:52:56 -0400 Subject: [PATCH 01/12] Add public API entties from 'bigquery.table'. --- gcloud/bigquery/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gcloud/bigquery/__init__.py b/gcloud/bigquery/__init__.py index 5b8efe9b4499..30207de37489 100644 --- a/gcloud/bigquery/__init__.py +++ b/gcloud/bigquery/__init__.py @@ -24,3 +24,5 @@ from gcloud.bigquery.client import Client from gcloud.bigquery.connection import SCOPE from gcloud.bigquery.dataset import Dataset +from gcloud.bigquery.table import SchemaField +from gcloud.bigquery.table import Table From f46de66cdd5dbab75f64a09c2b353ce424a02716 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 Aug 2015 12:44:53 -0400 Subject: [PATCH 02/12] Add 'Dataset.from_api_repr' factory. --- gcloud/bigquery/dataset.py | 22 +++++++++++++ gcloud/bigquery/test_dataset.py | 55 +++++++++++++++++++++++++++++---- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index c725b8200462..f310031c44ec 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -190,6 +190,28 @@ def location(self, value): raise ValueError("Pass a string, or None") self._properties['location'] = value + @classmethod + def from_api_repr(cls, resource, client): + """Factory: construct a dataset given its API representation + + :type resource: dict + :param resource: dataset resource representation returned from the API + + :type client: :class:`gcloud.pubsub.client.Client` + :param client: Client which holds credentials and project + configuration for the dataset. + + :rtype: :class:`gcloud.pubsub.dataset.Topic` + :returns: Topic parsed from ``resource``. + :raises: :class:`ValueError` if ``client`` is not ``None`` and the + project from the resource does not agree with the project + from the client. + """ + name = resource['datasetReference']['datasetId'] + dataset = cls(name, client=client) + dataset._properties = resource.copy() + return dataset + def _require_client(self, client): """Check client or verify over-ride. diff --git a/gcloud/bigquery/test_dataset.py b/gcloud/bigquery/test_dataset.py index 1ae3699c2e57..7f814cdae4cb 100644 --- a/gcloud/bigquery/test_dataset.py +++ b/gcloud/bigquery/test_dataset.py @@ -26,7 +26,7 @@ def _getTargetClass(self): def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) - def _makeResource(self): + def _setUpConstants(self): import datetime import pytz self.WHEN_TS = 1437767599.006 @@ -35,11 +35,14 @@ def _makeResource(self): self.ETAG = 'ETAG' self.DS_ID = '%s:%s' % (self.PROJECT, self.DS_NAME) self.RESOURCE_URL = 'http://example.com/path/to/resource' + + def _makeResource(self): + self._setUpConstants() return { 'creationTime': self.WHEN_TS * 1000, 'datasetReference': {'projectId': self.PROJECT, 'datasetId': self.DS_NAME}, - 'etag': 'ETAG', + 'etag': self.ETAG, 'id': self.DS_ID, 'lastModifiedTime': self.WHEN_TS * 1000, 'location': 'US', @@ -47,11 +50,23 @@ def _makeResource(self): } def _verifyResourceProperties(self, dataset, resource): - self.assertEqual(dataset.created, self.WHEN) self.assertEqual(dataset.dataset_id, self.DS_ID) - self.assertEqual(dataset.etag, self.ETAG) - self.assertEqual(dataset.modified, self.WHEN) - self.assertEqual(dataset.self_link, self.RESOURCE_URL) + if 'creationTime' in resource: + self.assertEqual(dataset.created, self.WHEN) + else: + self.assertEqual(dataset.created, None) + if 'etag' in resource: + self.assertEqual(dataset.etag, self.ETAG) + else: + self.assertEqual(dataset.etag, None) + if 'lastModifiedTime' in resource: + self.assertEqual(dataset.modified, self.WHEN) + else: + self.assertEqual(dataset.modified, None) + if 'selfLink' in resource: + self.assertEqual(dataset.self_link, self.RESOURCE_URL) + else: + self.assertEqual(dataset.self_link, None) self.assertEqual(dataset.default_table_expiration_ms, resource.get('defaultTableExpirationMs')) @@ -128,6 +143,34 @@ def test_location_setter(self): dataset.location = 'LOCATION' self.assertEqual(dataset.location, 'LOCATION') + def test_from_api_repr_bare(self): + self._setUpConstants() + CLIENT = _Client(self.PROJECT) + RESOURCE = { + 'id': '%s:%s' % (self.PROJECT, self.DS_NAME), + 'datasetReference': { + 'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + } + } + klass = self._getTargetClass() + dataset = klass.from_api_repr(RESOURCE, client=CLIENT) + self.assertTrue(dataset._client is CLIENT) + self._verifyResourceProperties(dataset, RESOURCE) + + def test_from_api_repr_w_properties(self): + import datetime + import pytz + self.WHEN_TS = 1437767599.006 + self.WHEN = datetime.datetime.utcfromtimestamp(self.WHEN_TS).replace( + tzinfo=pytz.UTC) + CLIENT = _Client(self.PROJECT) + RESOURCE = self._makeResource() + klass = self._getTargetClass() + dataset = klass.from_api_repr(RESOURCE, client=CLIENT) + self.assertTrue(dataset._client is CLIENT) + self._verifyResourceProperties(dataset, RESOURCE) + def test_create_w_bound_client(self): PATH = 'projects/%s/datasets' % self.PROJECT RESOURCE = self._makeResource() From e427864671b87cd46afc2328e522332184be668e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 Aug 2015 12:45:26 -0400 Subject: [PATCH 03/12] Add 'bigquery.client.Client.list_datasets' API method. --- gcloud/bigquery/client.py | 43 +++++++++++++++ gcloud/bigquery/dataset.py | 2 +- gcloud/bigquery/test_client.py | 94 +++++++++++++++++++++++++++++++++ gcloud/bigquery/test_dataset.py | 5 -- 4 files changed, 138 insertions(+), 6 deletions(-) diff --git a/gcloud/bigquery/client.py b/gcloud/bigquery/client.py index 0079e8c5142b..e8f9effd0ace 100644 --- a/gcloud/bigquery/client.py +++ b/gcloud/bigquery/client.py @@ -43,6 +43,49 @@ class Client(JSONClient): _connection_class = Connection + def list_datasets(self, include_all=False, max_results=None, + page_token=None): + """List datasets for the project associated with this client. + + See: + https://cloud.google.com/pubsub/reference/rest/v1beta2/projects/datasets/list + + :type include_all: boolean + :param include_all: Should results include hidden datasets? + + :type max_results: int + :param max_results: maximum number of datasets to return, If not + passed, defaults to a value set by the API. + + :type page_token: string + :param page_token: opaque marker for the next "page" of datasets. If + not passed, the API will return the first page of + datasets. + + :rtype: tuple, (list, str) + :returns: list of :class:`gcloud.pubsub.dataset.Dataset`, plus a + "next page token" string: if not None, indicates that + more datasets can be retrieved with another call (pass that + value as ``page_token``). + """ + params = {} + + if include_all: + params['all'] = True + + if max_results is not None: + params['maxResults'] = max_results + + if page_token is not None: + params['pageToken'] = page_token + + path = '/projects/%s/datasets' % (self.project,) + resp = self.connection.api_request(method='GET', path=path, + query_params=params) + datasets = [Dataset.from_api_repr(resource, self) + for resource in resp['datasets']] + return datasets, resp.get('nextPageToken') + def dataset(self, name): """Construct a dataset bound to this client. diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index f310031c44ec..65858234606f 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -209,7 +209,7 @@ def from_api_repr(cls, resource, client): """ name = resource['datasetReference']['datasetId'] dataset = cls(name, client=client) - dataset._properties = resource.copy() + dataset._set_properties(resource) return dataset def _require_client(self, client): diff --git a/gcloud/bigquery/test_client.py b/gcloud/bigquery/test_client.py index b9d547fd359a..554314fa9515 100644 --- a/gcloud/bigquery/test_client.py +++ b/gcloud/bigquery/test_client.py @@ -46,6 +46,88 @@ def test_dataset(self): self.assertEqual(dataset.name, DATASET) self.assertTrue(dataset._client is client) + def test_list_datasets_defaults(self): + from gcloud.bigquery.dataset import Dataset + PROJECT = 'PROJECT' + DATASET_1 = 'dataset_one' + DATASET_2 = 'dataset_two' + PATH = 'projects/%s/datasets' % PROJECT + TOKEN = 'TOKEN' + DATA = { + 'nextPageToken': TOKEN, + 'datasets': [ + {'kind': 'bigquery#dataset', + 'id': '%s:%s' % (PROJECT, DATASET_1), + 'datasetReference': {'datasetId': DATASET_1, + 'projectId': PROJECT}, + 'friendlyName': None}, + {'kind': 'bigquery#dataset', + 'id': '%s:%s' % (PROJECT, DATASET_2), + 'datasetReference': {'datasetId': DATASET_2, + 'projectId': PROJECT}, + 'friendlyName': 'Two'}, + ] + } + creds = _Credentials() + client = self._makeOne(PROJECT, creds) + conn = client.connection = _Connection(DATA) + + datasets, token = client.list_datasets() + + self.assertEqual(len(datasets), len(DATA['datasets'])) + for found, expected in zip(datasets, DATA['datasets']): + self.assertTrue(isinstance(found, Dataset)) + self.assertEqual(found.dataset_id, expected['id']) + self.assertEqual(found.friendly_name, expected['friendlyName']) + self.assertEqual(token, TOKEN) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + + def test_list_datasets_explicit(self): + from gcloud.bigquery.dataset import Dataset + PROJECT = 'PROJECT' + DATASET_1 = 'dataset_one' + DATASET_2 = 'dataset_two' + PATH = 'projects/%s/datasets' % PROJECT + TOKEN = 'TOKEN' + DATA = { + 'datasets': [ + {'kind': 'bigquery#dataset', + 'id': '%s:%s' % (PROJECT, DATASET_1), + 'datasetReference': {'datasetId': DATASET_1, + 'projectId': PROJECT}, + 'friendlyName': None}, + {'kind': 'bigquery#dataset', + 'id': '%s:%s' % (PROJECT, DATASET_2), + 'datasetReference': {'datasetId': DATASET_2, + 'projectId': PROJECT}, + 'friendlyName': 'Two'}, + ] + } + creds = _Credentials() + client = self._makeOne(PROJECT, creds) + conn = client.connection = _Connection(DATA) + + datasets, token = client.list_datasets( + include_all=True, max_results=3, page_token=TOKEN) + + self.assertEqual(len(datasets), len(DATA['datasets'])) + for found, expected in zip(datasets, DATA['datasets']): + self.assertTrue(isinstance(found, Dataset)) + self.assertEqual(found.dataset_id, expected['id']) + self.assertEqual(found.friendly_name, expected['friendlyName']) + self.assertEqual(token, None) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], + {'all': True, 'maxResults': 3, 'pageToken': TOKEN}) + class _Credentials(object): @@ -58,3 +140,15 @@ def create_scoped_required(): def create_scoped(self, scope): self._scopes = scope return self + + +class _Connection(object): + + def __init__(self, *responses): + self._responses = responses + self._requested = [] + + def api_request(self, **kw): + self._requested.append(kw) + response, self._responses = self._responses[0], self._responses[1:] + return response diff --git a/gcloud/bigquery/test_dataset.py b/gcloud/bigquery/test_dataset.py index 7f814cdae4cb..5f0000ae55b0 100644 --- a/gcloud/bigquery/test_dataset.py +++ b/gcloud/bigquery/test_dataset.py @@ -159,11 +159,6 @@ def test_from_api_repr_bare(self): self._verifyResourceProperties(dataset, RESOURCE) def test_from_api_repr_w_properties(self): - import datetime - import pytz - self.WHEN_TS = 1437767599.006 - self.WHEN = datetime.datetime.utcfromtimestamp(self.WHEN_TS).replace( - tzinfo=pytz.UTC) CLIENT = _Client(self.PROJECT) RESOURCE = self._makeResource() klass = self._getTargetClass() From f3089d5210e1696987193502843d29229b29a5c2 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 Aug 2015 12:48:16 -0400 Subject: [PATCH 04/12] Avoid all-uppercase for non-constant variable name. --- gcloud/bigquery/dataset.py | 9 ++---- gcloud/bigquery/test_dataset.py | 52 ++++++++++++++++----------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 65858234606f..05f5409fca24 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -197,15 +197,12 @@ def from_api_repr(cls, resource, client): :type resource: dict :param resource: dataset resource representation returned from the API - :type client: :class:`gcloud.pubsub.client.Client` + :type client: :class:`gcloud.bigquery.client.Client` :param client: Client which holds credentials and project configuration for the dataset. - :rtype: :class:`gcloud.pubsub.dataset.Topic` - :returns: Topic parsed from ``resource``. - :raises: :class:`ValueError` if ``client`` is not ``None`` and the - project from the resource does not agree with the project - from the client. + :rtype: :class:`gcloud.bigquery.dataset.Dataset` + :returns: Dataset parsed from ``resource``. """ name = resource['datasetReference']['datasetId'] dataset = cls(name, client=client) diff --git a/gcloud/bigquery/test_dataset.py b/gcloud/bigquery/test_dataset.py index 5f0000ae55b0..f2b6189fda2c 100644 --- a/gcloud/bigquery/test_dataset.py +++ b/gcloud/bigquery/test_dataset.py @@ -145,7 +145,7 @@ def test_location_setter(self): def test_from_api_repr_bare(self): self._setUpConstants() - CLIENT = _Client(self.PROJECT) + client = _Client(self.PROJECT) RESOURCE = { 'id': '%s:%s' % (self.PROJECT, self.DS_NAME), 'datasetReference': { @@ -154,24 +154,24 @@ def test_from_api_repr_bare(self): } } klass = self._getTargetClass() - dataset = klass.from_api_repr(RESOURCE, client=CLIENT) - self.assertTrue(dataset._client is CLIENT) + dataset = klass.from_api_repr(RESOURCE, client=client) + self.assertTrue(dataset._client is client) self._verifyResourceProperties(dataset, RESOURCE) def test_from_api_repr_w_properties(self): - CLIENT = _Client(self.PROJECT) + client = _Client(self.PROJECT) RESOURCE = self._makeResource() klass = self._getTargetClass() - dataset = klass.from_api_repr(RESOURCE, client=CLIENT) - self.assertTrue(dataset._client is CLIENT) + dataset = klass.from_api_repr(RESOURCE, client=client) + self.assertTrue(dataset._client is client) self._verifyResourceProperties(dataset, RESOURCE) def test_create_w_bound_client(self): PATH = 'projects/%s/datasets' % self.PROJECT RESOURCE = self._makeResource() conn = _Connection(RESOURCE) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) dataset.create() @@ -226,8 +226,8 @@ def test_create_w_missing_output_properties(self): del RESOURCE['lastModifiedTime'] self.WHEN = None conn = _Connection(RESOURCE) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) dataset.create() @@ -245,8 +245,8 @@ def test_create_w_missing_output_properties(self): def test_exists_miss_w_bound_client(self): PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) conn = _Connection() - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) self.assertFalse(dataset.exists()) @@ -277,8 +277,8 @@ def test_reload_w_bound_client(self): PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) RESOURCE = self._makeResource() conn = _Connection(RESOURCE) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) dataset.reload() @@ -309,8 +309,8 @@ def test_reload_w_alternate_client(self): def test_patch_w_invalid_expiration(self): RESOURCE = self._makeResource() conn = _Connection(RESOURCE) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) with self.assertRaises(ValueError): dataset.patch(default_table_expiration_ms='BOGUS') @@ -323,8 +323,8 @@ def test_patch_w_bound_client(self): RESOURCE['description'] = DESCRIPTION RESOURCE['friendlyName'] = TITLE conn = _Connection(RESOURCE) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) dataset.patch(description=DESCRIPTION, friendly_name=TITLE) @@ -376,8 +376,8 @@ def test_update_w_bound_client(self): RESOURCE['description'] = DESCRIPTION RESOURCE['friendlyName'] = TITLE conn = _Connection(RESOURCE) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) dataset.description = DESCRIPTION dataset.friendly_name = TITLE @@ -430,8 +430,8 @@ def test_update_w_alternate_client(self): def test_delete_w_bound_client(self): PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) conn = _Connection({}) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) dataset.delete() @@ -459,8 +459,8 @@ def test_delete_w_alternate_client(self): def test_table_wo_schema(self): from gcloud.bigquery.table import Table conn = _Connection({}) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) table = dataset.table('table_name') self.assertTrue(isinstance(table, Table)) self.assertEqual(table.name, 'table_name') @@ -471,8 +471,8 @@ def test_table_w_schema(self): from gcloud.bigquery.table import SchemaField from gcloud.bigquery.table import Table conn = _Connection({}) - CLIENT = _Client(project=self.PROJECT, connection=conn) - dataset = self._makeOne(self.DS_NAME, client=CLIENT) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') age = SchemaField('age', 'INTEGER', mode='REQUIRED') table = dataset.table('table_name', schema=[full_name, age]) From 853d1f96436087ee4e2f796079dd4f53a5de933c Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 Aug 2015 13:44:05 -0400 Subject: [PATCH 05/12] Add 'Table.from_api_repr' factory. --- gcloud/bigquery/table.py | 20 ++++++++- gcloud/bigquery/test_table.py | 76 +++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index 67ac13e09a78..be5e26545f20 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -298,6 +298,24 @@ def view_query(self): """Delete SQL query defining the table as a view.""" self._properties.pop('view', None) + @classmethod + def from_api_repr(cls, resource, dataset): + """Factory: construct a table given its API representation + + :type resource: dict + :param resource: table resource representation returned from the API + + :type dataset: :class:`gcloud.bigquery.dataset.Dataset` + :param dataset: The dataset containing the table. + + :rtype: :class:`gcloud.bigquery.table.Table` + :returns: Table parsed from ``resource``. + """ + table_name = resource['tableReference']['tableId'] + table = cls(table_name, dataset=dataset) + table._set_properties(resource) + return table + def _require_client(self, client): """Check client or verify over-ride. @@ -344,7 +362,7 @@ def _set_properties(self, api_response): """ self._properties.clear() cleaned = api_response.copy() - schema = cleaned.pop('schema', {}) + schema = cleaned.pop('schema', {'fields': ()}) self.schema = self._parse_schema_resource(schema) if 'creationTime' in cleaned: cleaned['creationTime'] = float(cleaned['creationTime']) diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 5c678ebe545f..8060500008e7 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -74,7 +74,7 @@ def _getTargetClass(self): def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) - def _makeResource(self): + def _setUpConstants(self): import datetime import pytz self.WHEN_TS = 1437767599.006 @@ -86,6 +86,9 @@ def _makeResource(self): self.RESOURCE_URL = 'http://example.com/path/to/resource' self.NUM_BYTES = 12345 self.NUM_ROWS = 67 + + def _makeResource(self): + self._setUpConstants() return { 'creationTime': self.WHEN_TS * 1000, 'tableReference': @@ -117,29 +120,58 @@ def _verifySchema(self, schema, resource): for field, r_field in zip(schema, r_fields): self._verify_field(field, r_field) - def _verifyResourceProperties(self, table, resource): - self.assertEqual(table.created, self.WHEN) - self.assertEqual(table.etag, self.ETAG) - self.assertEqual(table.num_rows, self.NUM_ROWS) - self.assertEqual(table.num_bytes, self.NUM_BYTES) - self.assertEqual(table.self_link, self.RESOURCE_URL) + def _verifyReadonlyResourceProperties(self, table, resource): + if 'creationTime' in resource: + self.assertEqual(table.created, self.WHEN) + else: + self.assertEqual(table.created, None) + + if 'etag' in resource: + self.assertEqual(table.etag, self.ETAG) + else: + self.assertEqual(table.etag, None) + + if 'numRows' in resource: + self.assertEqual(table.num_rows, self.NUM_ROWS) + else: + self.assertEqual(table.num_rows, None) + + if 'numBytes' in resource: + self.assertEqual(table.num_bytes, self.NUM_BYTES) + else: + self.assertEqual(table.num_bytes, None) + + if 'selfLink' in resource: + self.assertEqual(table.self_link, self.RESOURCE_URL) + else: + self.assertEqual(table.self_link, None) + self.assertEqual(table.table_id, self.TABLE_ID) self.assertEqual(table.table_type, 'TABLE' if 'view' not in resource else 'VIEW') + def _verifyResourceProperties(self, table, resource): + + self._verifyReadonlyResourceProperties(table, resource) + if 'expirationTime' in resource: self.assertEqual(table.expires, self.EXP_TIME) else: self.assertEqual(table.expires, None) + self.assertEqual(table.description, resource.get('description')) self.assertEqual(table.friendly_name, resource.get('friendlyName')) self.assertEqual(table.location, resource.get('location')) + if 'view' in resource: self.assertEqual(table.view_query, resource['view']['query']) else: self.assertEqual(table.view_query, None) - self._verifySchema(table.schema, resource) + if 'schema' in resource: + self._verifySchema(table.schema, resource) + else: + self.assertEqual(table.schema, []) def test_ctor(self): client = _Client(self.PROJECT) @@ -316,6 +348,34 @@ def test_view_query_deleter(self): del table.view_query self.assertEqual(table.view_query, None) + def test_from_api_repr_bare(self): + self._setUpConstants() + client = _Client(self.PROJECT) + dataset = _Dataset(client) + RESOURCE = { + 'id': '%s:%s:%s' % (self.PROJECT, self.DS_NAME, self.TABLE_NAME), + 'tableReference': { + 'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + 'tableId': self.TABLE_NAME, + }, + 'type': 'TABLE', + } + klass = self._getTargetClass() + table = klass.from_api_repr(RESOURCE, dataset) + self.assertEqual(table.name, self.TABLE_NAME) + self.assertTrue(table._dataset is dataset) + self._verifyResourceProperties(table, RESOURCE) + + def test_from_api_repr_w_properties(self): + client = _Client(self.PROJECT) + dataset = _Dataset(client) + RESOURCE = self._makeResource() + klass = self._getTargetClass() + table = klass.from_api_repr(RESOURCE, dataset) + self.assertTrue(table._dataset._client is client) + self._verifyResourceProperties(table, RESOURCE) + def test__parse_schema_resource_defaults(self): client = _Client(self.PROJECT) dataset = _Dataset(client) From e409492842094bf541777b455ad27305b78696b8 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 7 Aug 2015 14:21:16 -0400 Subject: [PATCH 06/12] Add 'Dataset.list_tables' API method. --- gcloud/bigquery/dataset.py | 37 ++++++++++++++ gcloud/bigquery/test_dataset.py | 87 +++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 05f5409fca24..041de1a8e51d 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -376,6 +376,43 @@ def delete(self, client=None): client = self._require_client(client) client.connection.api_request(method='DELETE', path=self.path) + def list_tables(self, max_results=None, page_token=None): + """List tables for the project associated with this client. + + See: + https://cloud.google.com/pubsub/reference/rest/v1beta2/projects/tables/list + + :type max_results: int + :param max_results: maximum number of tables to return, If not + passed, defaults to a value set by the API. + + :type page_token: string + :param page_token: opaque marker for the next "page" of datasets. If + not passed, the API will return the first page of + datasets. + + :rtype: tuple, (list, str) + :returns: list of :class:`gcloud.pubsub.table.Table`, plus a + "next page token" string: if not None, indicates that + more tables can be retrieved with another call (pass that + value as ``page_token``). + """ + params = {} + + if max_results is not None: + params['maxResults'] = max_results + + if page_token is not None: + params['pageToken'] = page_token + + path = '/projects/%s/datasets/%s/tables' % (self.project, self.name) + connection = self._client.connection + resp = connection.api_request(method='GET', path=path, + query_params=params) + tables = [Table.from_api_repr(resource, self) + for resource in resp['tables']] + return tables, resp.get('nextPageToken') + def table(self, name, schema=()): """Construct a table bound to this dataset. diff --git a/gcloud/bigquery/test_dataset.py b/gcloud/bigquery/test_dataset.py index f2b6189fda2c..d9a8c1810ed6 100644 --- a/gcloud/bigquery/test_dataset.py +++ b/gcloud/bigquery/test_dataset.py @@ -456,6 +456,93 @@ def test_delete_w_alternate_client(self): self.assertEqual(req['method'], 'DELETE') self.assertEqual(req['path'], '/%s' % PATH) + def test_list_tables_defaults(self): + from gcloud.bigquery.table import Table + conn = _Connection({}) + TABLE_1 = 'table_one' + TABLE_2 = 'table_two' + PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) + TOKEN = 'TOKEN' + DATA = { + 'nextPageToken': TOKEN, + 'tables': [ + {'kind': 'bigquery#table', + 'id': '%s:%s.%s' % (self.PROJECT, self.DS_NAME, TABLE_1), + 'tableReference': {'tableId': TABLE_1, + 'datasetId': self.DS_NAME, + 'projectId': self.PROJECT}, + 'type': 'TABLE'}, + {'kind': 'bigquery#table', + 'id': '%s:%s.%s' % (self.PROJECT, self.DS_NAME, TABLE_2), + 'tableReference': {'tableId': TABLE_2, + 'datasetId': self.DS_NAME, + 'projectId': self.PROJECT}, + 'type': 'TABLE'}, + ] + } + + conn = _Connection(DATA) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) + + tables, token = dataset.list_tables() + + self.assertEqual(len(tables), len(DATA['tables'])) + for found, expected in zip(tables, DATA['tables']): + self.assertTrue(isinstance(found, Table)) + self.assertEqual(found.table_id, expected['id']) + self.assertEqual(found.table_type, expected['type']) + self.assertEqual(token, TOKEN) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + + def test_list_tables_explicit(self): + from gcloud.bigquery.table import Table + conn = _Connection({}) + TABLE_1 = 'table_one' + TABLE_2 = 'table_two' + PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) + TOKEN = 'TOKEN' + DATA = { + 'tables': [ + {'kind': 'bigquery#dataset', + 'id': '%s:%s.%s' % (self.PROJECT, self.DS_NAME, TABLE_1), + 'tableReference': {'tableId': TABLE_1, + 'datasetId': self.DS_NAME, + 'projectId': self.PROJECT}, + 'type': 'TABLE'}, + {'kind': 'bigquery#dataset', + 'id': '%s:%s.%s' % (self.PROJECT, self.DS_NAME, TABLE_2), + 'tableReference': {'tableId': TABLE_2, + 'datasetId': self.DS_NAME, + 'projectId': self.PROJECT}, + 'type': 'TABLE'}, + ] + } + + conn = _Connection(DATA) + client = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=client) + + tables, token = dataset.list_tables(max_results=3, page_token=TOKEN) + + self.assertEqual(len(tables), len(DATA['tables'])) + for found, expected in zip(tables, DATA['tables']): + self.assertTrue(isinstance(found, Table)) + self.assertEqual(found.table_id, expected['id']) + self.assertEqual(found.table_type, expected['type']) + self.assertEqual(token, None) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], + {'maxResults': 3, 'pageToken': TOKEN}) + def test_table_wo_schema(self): from gcloud.bigquery.table import Table conn = _Connection({}) From 42b5e260d3848c25549defd54cf49c5a0cdf7fa7 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 8 Aug 2015 12:22:43 -0400 Subject: [PATCH 07/12] Fix copy-pasta in docstrings. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045/files#r36571762 https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045/files#r36571777 https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045/files#r36571768 https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045/files#r36571771 --- gcloud/bigquery/client.py | 4 ++-- gcloud/bigquery/dataset.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gcloud/bigquery/client.py b/gcloud/bigquery/client.py index e8f9effd0ace..cd840efba6e6 100644 --- a/gcloud/bigquery/client.py +++ b/gcloud/bigquery/client.py @@ -48,7 +48,7 @@ def list_datasets(self, include_all=False, max_results=None, """List datasets for the project associated with this client. See: - https://cloud.google.com/pubsub/reference/rest/v1beta2/projects/datasets/list + https://cloud.google.com/bigquery/reference/rest/v1beta2/projects/datasets/list :type include_all: boolean :param include_all: Should results include hidden datasets? @@ -63,7 +63,7 @@ def list_datasets(self, include_all=False, max_results=None, datasets. :rtype: tuple, (list, str) - :returns: list of :class:`gcloud.pubsub.dataset.Dataset`, plus a + :returns: list of :class:`gcloud.bigquery.dataset.Dataset`, plus a "next page token" string: if not None, indicates that more datasets can be retrieved with another call (pass that value as ``page_token``). diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 041de1a8e51d..55ff7bddd633 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -380,7 +380,7 @@ def list_tables(self, max_results=None, page_token=None): """List tables for the project associated with this client. See: - https://cloud.google.com/pubsub/reference/rest/v1beta2/projects/tables/list + https://cloud.google.com/bigquery/reference/rest/v1beta2/projects/tables/list :type max_results: int :param max_results: maximum number of tables to return, If not @@ -392,7 +392,7 @@ def list_tables(self, max_results=None, page_token=None): datasets. :rtype: tuple, (list, str) - :returns: list of :class:`gcloud.pubsub.table.Table`, plus a + :returns: list of :class:`gcloud.bigquery.table.Table`, plus a "next page token" string: if not None, indicates that more tables can be retrieved with another call (pass that value as ``page_token``). From 531450789ce102a903e80c236c3c65c70a47a1bc Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 8 Aug 2015 12:26:23 -0400 Subject: [PATCH 08/12] Explain boolean value w/o question. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045/files#r36571626 --- gcloud/bigquery/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud/bigquery/client.py b/gcloud/bigquery/client.py index cd840efba6e6..9dd9a1e9e3a9 100644 --- a/gcloud/bigquery/client.py +++ b/gcloud/bigquery/client.py @@ -51,7 +51,7 @@ def list_datasets(self, include_all=False, max_results=None, https://cloud.google.com/bigquery/reference/rest/v1beta2/projects/datasets/list :type include_all: boolean - :param include_all: Should results include hidden datasets? + :param include_all: True if results include hidden datasets. :type max_results: int :param max_results: maximum number of datasets to return, If not From 8b4d6f314b07ad67f5737182e717b0ede424ed97 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Sat, 8 Aug 2015 12:27:20 -0400 Subject: [PATCH 09/12] Reword :returns: for clarity. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045/files#r36571658 --- gcloud/bigquery/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gcloud/bigquery/client.py b/gcloud/bigquery/client.py index 9dd9a1e9e3a9..1a8029a5dc46 100644 --- a/gcloud/bigquery/client.py +++ b/gcloud/bigquery/client.py @@ -64,9 +64,9 @@ def list_datasets(self, include_all=False, max_results=None, :rtype: tuple, (list, str) :returns: list of :class:`gcloud.bigquery.dataset.Dataset`, plus a - "next page token" string: if not None, indicates that - more datasets can be retrieved with another call (pass that - value as ``page_token``). + "next page token" string: if the toke is not None, + indicates that more datasets can be retrieved with another + call (pass that value as ``page_token``). """ params = {} From 9a7387191348a199dcdce5a9c562d6eff6c578b2 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 10 Aug 2015 13:30:50 -0400 Subject: [PATCH 10/12] Fix API docs URLs. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045#discussion_r36657248 https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045#discussion_r36657331 --- gcloud/bigquery/client.py | 2 +- gcloud/bigquery/dataset.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gcloud/bigquery/client.py b/gcloud/bigquery/client.py index 1a8029a5dc46..aa5b0ddc8952 100644 --- a/gcloud/bigquery/client.py +++ b/gcloud/bigquery/client.py @@ -48,7 +48,7 @@ def list_datasets(self, include_all=False, max_results=None, """List datasets for the project associated with this client. See: - https://cloud.google.com/bigquery/reference/rest/v1beta2/projects/datasets/list + https://cloud.google.com/bigquery/docs/reference/v2/datasets/list :type include_all: boolean :param include_all: True if results include hidden datasets. diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 55ff7bddd633..49b06f8d5085 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -380,7 +380,7 @@ def list_tables(self, max_results=None, page_token=None): """List tables for the project associated with this client. See: - https://cloud.google.com/bigquery/reference/rest/v1beta2/projects/tables/list + https://cloud.google.com/bigquery/docs/reference/v2/tables/list :type max_results: int :param max_results: maximum number of tables to return, If not From 2cd2cd28c0fb96a1c2239c5708f3e03e4302cbee Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 10 Aug 2015 13:31:41 -0400 Subject: [PATCH 11/12] Fix typo. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045#discussion_r36657601 --- gcloud/bigquery/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gcloud/bigquery/client.py b/gcloud/bigquery/client.py index aa5b0ddc8952..891e773a1f0b 100644 --- a/gcloud/bigquery/client.py +++ b/gcloud/bigquery/client.py @@ -64,7 +64,7 @@ def list_datasets(self, include_all=False, max_results=None, :rtype: tuple, (list, str) :returns: list of :class:`gcloud.bigquery.dataset.Dataset`, plus a - "next page token" string: if the toke is not None, + "next page token" string: if the token is not None, indicates that more datasets can be retrieved with another call (pass that value as ``page_token``). """ From 34a747ccbbe4580365c2778aa3a80708c043709e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 10 Aug 2015 16:30:57 -0400 Subject: [PATCH 12/12] Pre-check for missing dataset/table name when parsing resource. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1045#issuecomment-129580676 --- gcloud/bigquery/dataset.py | 4 ++++ gcloud/bigquery/table.py | 4 ++++ gcloud/bigquery/test_dataset.py | 8 ++++++++ gcloud/bigquery/test_table.py | 9 +++++++++ 4 files changed, 25 insertions(+) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 49b06f8d5085..db5cfb70f216 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -204,6 +204,10 @@ def from_api_repr(cls, resource, client): :rtype: :class:`gcloud.bigquery.dataset.Dataset` :returns: Dataset parsed from ``resource``. """ + if ('datasetReference' not in resource or + 'datasetId' not in resource['datasetReference']): + raise KeyError('Resource lacks required identity information:' + '["datasetReference"]["datasetId"]') name = resource['datasetReference']['datasetId'] dataset = cls(name, client=client) dataset._set_properties(resource) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index be5e26545f20..ee2d40cb8a2a 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -311,6 +311,10 @@ def from_api_repr(cls, resource, dataset): :rtype: :class:`gcloud.bigquery.table.Table` :returns: Table parsed from ``resource``. """ + if ('tableReference' not in resource or + 'tableId' not in resource['tableReference']): + raise KeyError('Resource lacks required identity information:' + '["tableReference"]["tableId"]') table_name = resource['tableReference']['tableId'] table = cls(table_name, dataset=dataset) table._set_properties(resource) diff --git a/gcloud/bigquery/test_dataset.py b/gcloud/bigquery/test_dataset.py index d9a8c1810ed6..71562c762d92 100644 --- a/gcloud/bigquery/test_dataset.py +++ b/gcloud/bigquery/test_dataset.py @@ -143,6 +143,14 @@ def test_location_setter(self): dataset.location = 'LOCATION' self.assertEqual(dataset.location, 'LOCATION') + def test_from_api_repr_missing_identity(self): + self._setUpConstants() + client = _Client(self.PROJECT) + RESOURCE = {} + klass = self._getTargetClass() + with self.assertRaises(KeyError): + klass.from_api_repr(RESOURCE, client=client) + def test_from_api_repr_bare(self): self._setUpConstants() client = _Client(self.PROJECT) diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 8060500008e7..b798d66a6572 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -348,6 +348,15 @@ def test_view_query_deleter(self): del table.view_query self.assertEqual(table.view_query, None) + def test_from_api_repr_missing_identity(self): + self._setUpConstants() + client = _Client(self.PROJECT) + dataset = _Dataset(client) + RESOURCE = {} + klass = self._getTargetClass() + with self.assertRaises(KeyError): + klass.from_api_repr(RESOURCE, dataset) + def test_from_api_repr_bare(self): self._setUpConstants() client = _Client(self.PROJECT)