From 05e3da550067e79bbc0ede3aa5e39aed3d02d3a2 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 29 Jul 2015 16:17:08 -0400 Subject: [PATCH 1/7] Add API request methods to Table. --- gcloud/bigquery/table.py | 195 +++++++++++++ gcloud/bigquery/test_table.py | 503 +++++++++++++++++++++++++++++++++- 2 files changed, 696 insertions(+), 2 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index a3c63ced04ba..d4edb7a86c94 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -18,6 +18,7 @@ import six +from gcloud.exceptions import NotFound from gcloud.bigquery._helpers import _datetime_from_prop from gcloud.bigquery._helpers import _prop_from_datetime @@ -293,3 +294,197 @@ def view_query(self, value): def view_query(self): """Delete SQL query defining the table as a view.""" self._properties.pop('view', None) + + def _require_client(self, client): + """Check client or verify over-ride. + + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current dataset. + + :rtype: :class:`gcloud.bigquery.client.Client` + :returns: The client passed in or the currently bound client. + """ + if client is None: + client = self._dataset._client + return client + + def _set_properties(self, api_response): + """Update properties from resource in body of ``api_response`` + + :type api_response: httplib2.Response + :param api_response: response returned from an API call + """ + self._properties.clear() + cleaned = api_response.copy() + cleaned['creationTime'] = float(cleaned['creationTime']) + cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime']) + if 'expirationTime' in cleaned: + cleaned['expirationTime'] = float(cleaned['expirationTime']) + self._properties.update(cleaned) + + def _build_schema_resource(self, fields=None): + """Generate a resource fragment for table's schema.""" + if fields is None: + fields = self._schema + infos = [] + for field in fields: + info = {'name': field.name, + 'type': field.field_type, + 'mode': field.mode} + if field.description is not None: + info['description'] = field.description + if field.fields is not None: + info['fields'] = self._build_schema_resource(field.fields) + infos.append(info) + return infos + + def _build_resource(self): + """Generate a resource for ``create`` or ``update``.""" + resource = { + 'tableReference': { + 'projectId': self._dataset.project, + 'datasetId': self._dataset.name, + 'tableId': self.name}, + 'schema': {'fields': self._build_schema_resource()}, + } + if self.description is not None: + resource['description'] = self.description + + if self.expires is not None: + value = _prop_from_datetime(self.expires) + resource['expirationTime'] = value + + if self.friendly_name is not None: + resource['friendlyName'] = self.friendly_name + + if self.location is not None: + resource['location'] = self.location + + if self.view_query is not None: + view = resource['view'] = {} + view['query'] = self.view_query + + return resource + + def create(self, client=None): + """API call: create the dataset via a PUT request + + See: + https://cloud.google.com/bigquery/reference/rest/v2/tables/insert + + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current dataset. + """ + client = self._require_client(client) + path = '/projects/%s/datasets/%s/tables' % ( + self._dataset.project, self._dataset.name) + api_response = client.connection.api_request( + method='POST', path=path, data=self._build_resource()) + self._set_properties(api_response) + + def exists(self, client=None): + """API call: test for the existence of the table via a GET request + + See + https://cloud.google.com/bigquery/docs/reference/v2/tables/get + + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current dataset. + """ + client = self._require_client(client) + + try: + client.connection.api_request(method='GET', path=self.path, + query_params={'fields': 'id'}) + except NotFound: + return False + else: + return True + + def reload(self, client=None): + """API call: refresh table properties via a GET request + + See + https://cloud.google.com/bigquery/docs/reference/v2/tables/get + + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current dataset. + """ + client = self._require_client(client) + + api_response = client.connection.api_request( + method='GET', path=self.path) + self._set_properties(api_response) + + def patch(self, client=None, **kw): + """API call: update individual table properties via a PATCH request + + See + https://cloud.google.com/bigquery/docs/reference/v2/tables/patch + + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current dataset. + + :type kw: ``dict`` + :param kw: properties to be patched. + + :raises: ValueError for invalid value types. + """ + client = self._require_client(client) + + partial = {} + + if 'expires' in kw: + value = kw['expires'] + if not isinstance(value, datetime.datetime) and value is not None: + raise ValueError("Pass a datetime, or None") + partial['expirationTime'] = _prop_from_datetime(value) + + if 'description' in kw: + partial['description'] = kw['description'] + + if 'friendly_name' in kw: + partial['friendlyName'] = kw['friendly_name'] + + if 'location' in kw: + partial['location'] = kw['location'] + + if 'view_query' in kw: + partial['view'] = {'query': kw['view_query']} + + api_response = client.connection.api_request( + method='PATCH', path=self.path, data=partial) + self._set_properties(api_response) + + def update(self, client=None): + """API call: update table properties via a PUT request + + See + https://cloud.google.com/bigquery/docs/reference/v2/tables/update + + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current dataset. + """ + client = self._require_client(client) + api_response = client.connection.api_request( + method='PUT', path=self.path, data=self._build_resource()) + self._set_properties(api_response) + + def delete(self, client=None): + """API call: delete the table via a DELETE request + + See: + https://cloud.google.com/bigquery/reference/rest/v2/tables/delete + + :type client: :class:`gcloud.bigquery.client.Client` or ``NoneType`` + :param client: the client to use. If not passed, falls back to the + ``client`` stored on the current dataset. + """ + client = self._require_client(client) + client.connection.api_request(method='DELETE', path=self.path) diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 07498d9780e5..3edd28c773ee 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -74,6 +74,59 @@ def _getTargetClass(self): def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) + def _makeResource(self): + import datetime + import pytz + self.WHEN_TS = 1437767599.006 + self.WHEN = datetime.datetime.utcfromtimestamp(self.WHEN_TS).replace( + tzinfo=pytz.UTC) + self.ETAG = 'ETAG' + self.TABLE_ID = '%s:%s:%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + self.RESOURCE_URL = 'http://example.com/path/to/resource' + self.NUM_BYTES = 12345 + self.NUM_ROWS = 67 + return { + 'creationTime': self.WHEN_TS * 1000, + 'tableReference': + {'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + 'tableId': self.TABLE_NAME}, + 'schema': {'fields': [ + {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, + 'etag': 'ETAG', + 'id': self.TABLE_ID, + 'lastModifiedTime': self.WHEN_TS * 1000, + 'location': 'US', + 'selfLink': self.RESOURCE_URL, + 'numRows': self.NUM_ROWS, + 'numBytes': self.NUM_BYTES, + 'type': 'TABLE', + } + + def _verifyResourceProperties(self, table, resource): + self.assertEqual(table.created, self.WHEN) + self.assertEqual(table.etag, self.ETAG) + self.assertEqual(table.num_rows, self.NUM_ROWS) + self.assertEqual(table.num_bytes, self.NUM_BYTES) + self.assertEqual(table.self_link, self.RESOURCE_URL) + self.assertEqual(table.table_id, self.TABLE_ID) + self.assertEqual(table.table_type, + 'TABLE' if 'view' not in resource else 'VIEW') + + if 'expirationTime' in resource: + self.assertEqual(table.expires, self.EXP_TIME) + else: + self.assertEqual(table.expires, None) + self.assertEqual(table.description, resource.get('description')) + self.assertEqual(table.friendly_name, resource.get('friendlyName')) + self.assertEqual(table.location, resource.get('location')) + if 'view' in resource: + self.assertEqual(table.view_query, resource['view']['query']) + else: + self.assertEqual(table.view_query, None) + def test_ctor(self): client = _Client(self.PROJECT) dataset = _Dataset(client) @@ -249,6 +302,430 @@ def test_view_query_deleter(self): del table.view_query self.assertEqual(table.view_query, None) + def test__build_schema_resource_defaults(self): + from gcloud.bigquery.table import SchemaField + client = _Client(self.PROJECT) + dataset = _Dataset(client) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = self._makeOne(self.TABLE_NAME, dataset, + schema=[full_name, age]) + resource = table._build_schema_resource() + self.assertEqual(len(resource), 2) + self.assertEqual(resource[0], + {'name': 'full_name', + 'type': 'STRING', + 'mode': 'REQUIRED'}) + self.assertEqual(resource[1], + {'name': 'age', + 'type': 'INTEGER', + 'mode': 'REQUIRED'}) + + def test__build_schema_resource_w_description(self): + from gcloud.bigquery.table import SchemaField + client = _Client(self.PROJECT) + dataset = _Dataset(client) + DESCRIPTION = 'DESCRIPTION' + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED', + description=DESCRIPTION) + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = self._makeOne(self.TABLE_NAME, dataset, + schema=[full_name, age]) + resource = table._build_schema_resource() + self.assertEqual(len(resource), 2) + self.assertEqual(resource[0], + {'name': 'full_name', + 'type': 'STRING', + 'mode': 'REQUIRED', + 'description': DESCRIPTION}) + self.assertEqual(resource[1], + {'name': 'age', + 'type': 'INTEGER', + 'mode': 'REQUIRED'}) + + def test__build_schema_resource_w_subfields(self): + from gcloud.bigquery.table import SchemaField + client = _Client(self.PROJECT) + dataset = _Dataset(client) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + ph_type = SchemaField('type', 'STRING', 'REQUIRED') + ph_num = SchemaField('number', 'STRING', 'REQUIRED') + phone = SchemaField('phone', 'RECORD', mode='REPEATABLE', + fields=[ph_type, ph_num]) + table = self._makeOne(self.TABLE_NAME, dataset, + schema=[full_name, phone]) + resource = table._build_schema_resource() + self.assertEqual(len(resource), 2) + self.assertEqual(resource[0], + {'name': 'full_name', + 'type': 'STRING', + 'mode': 'REQUIRED'}) + self.assertEqual(resource[1], + {'name': 'phone', + 'type': 'RECORD', + 'mode': 'REPEATABLE', + 'fields': [{'name': 'type', + 'type': 'STRING', + 'mode': 'REQUIRED'}, + {'name': 'number', + 'type': 'STRING', + 'mode': 'REQUIRED'}]}) + + def test_create_w_bound_client(self): + from gcloud.bigquery.table import SchemaField + PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) + RESOURCE = self._makeResource() + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = self._makeOne(self.TABLE_NAME, dataset, + schema=[full_name, age]) + + table.create() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'tableReference': { + 'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + 'tableId': self.TABLE_NAME}, + 'schema': {'fields': [ + {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(table, RESOURCE) + + def test_create_w_alternate_client(self): + import datetime + import pytz + from gcloud.bigquery.table import SchemaField + from gcloud.bigquery._helpers import _millis + PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) + DESCRIPTION = 'DESCRIPTION' + TITLE = 'TITLE' + QUERY = 'select fullname, age from person_ages' + RESOURCE = self._makeResource() + RESOURCE['description'] = DESCRIPTION + RESOURCE['friendlyName'] = TITLE + self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59, + tzinfo=pytz.utc) + RESOURCE['expirationTime'] = _millis(self.EXP_TIME) + RESOURCE['view'] = {} + RESOURCE['view']['query'] = QUERY + RESOURCE['type'] = 'VIEW' + conn1 = _Connection() + client1 = _Client(project=self.PROJECT, connection=conn1) + conn2 = _Connection(RESOURCE) + client2 = _Client(project=self.PROJECT, connection=conn2) + dataset = _Dataset(client=client1) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = self._makeOne(self.TABLE_NAME, dataset=dataset, + schema=[full_name, age]) + table.friendly_name = TITLE + table.description = DESCRIPTION + table.view_query = QUERY + + table.create(client=client2) + + self.assertEqual(len(conn1._requested), 0) + self.assertEqual(len(conn2._requested), 1) + req = conn2._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'tableReference': { + 'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + 'tableId': self.TABLE_NAME}, + 'schema': {'fields': [ + {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, + 'description': DESCRIPTION, + 'friendlyName': TITLE, + 'view': {'query': QUERY}, + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(table, RESOURCE) + + def test_exists_miss_w_bound_client(self): + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + conn = _Connection() + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + self.assertFalse(table.exists()) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], {'fields': 'id'}) + + def test_exists_hit_w_alternate_client(self): + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + conn1 = _Connection() + client1 = _Client(project=self.PROJECT, connection=conn1) + conn2 = _Connection({}) + client2 = _Client(project=self.PROJECT, connection=conn2) + dataset = _Dataset(client1) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + self.assertTrue(table.exists(client=client2)) + + self.assertEqual(len(conn1._requested), 0) + self.assertEqual(len(conn2._requested), 1) + req = conn2._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self.assertEqual(req['query_params'], {'fields': 'id'}) + + def test_reload_w_bound_client(self): + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + RESOURCE = self._makeResource() + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + table.reload() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self._verifyResourceProperties(table, RESOURCE) + + def test_reload_w_alternate_client(self): + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + RESOURCE = self._makeResource() + conn1 = _Connection() + client1 = _Client(project=self.PROJECT, connection=conn1) + conn2 = _Connection(RESOURCE) + client2 = _Client(project=self.PROJECT, connection=conn2) + dataset = _Dataset(client1) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + table.reload(client=client2) + + self.assertEqual(len(conn1._requested), 0) + self.assertEqual(len(conn2._requested), 1) + req = conn2._requested[0] + self.assertEqual(req['method'], 'GET') + self.assertEqual(req['path'], '/%s' % PATH) + self._verifyResourceProperties(table, RESOURCE) + + def test_patch_w_invalid_expiration(self): + RESOURCE = self._makeResource() + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + with self.assertRaises(ValueError): + table.patch(expires='BOGUS') + + def test_patch_w_bound_client(self): + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + DESCRIPTION = 'DESCRIPTION' + TITLE = 'TITLE' + RESOURCE = self._makeResource() + RESOURCE['description'] = DESCRIPTION + RESOURCE['friendlyName'] = TITLE + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + table.patch(description=DESCRIPTION, friendly_name=TITLE) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'PATCH') + SENT = { + 'description': DESCRIPTION, + 'friendlyName': TITLE, + } + self.assertEqual(req['data'], SENT) + self.assertEqual(req['path'], '/%s' % PATH) + self._verifyResourceProperties(table, RESOURCE) + + def test_patch_w_alternate_client(self): + import datetime + import pytz + from gcloud.bigquery._helpers import _millis + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + QUERY = 'select fullname, age from person_ages' + LOCATION = 'EU' + RESOURCE = self._makeResource() + RESOURCE['view'] = {'query': QUERY} + RESOURCE['type'] = 'VIEW' + RESOURCE['location'] = LOCATION + self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59, + tzinfo=pytz.utc) + RESOURCE['expirationTime'] = _millis(self.EXP_TIME) + conn1 = _Connection() + client1 = _Client(project=self.PROJECT, connection=conn1) + conn2 = _Connection(RESOURCE) + client2 = _Client(project=self.PROJECT, connection=conn2) + dataset = _Dataset(client1) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + table.patch(client=client2, view_query=QUERY, location=LOCATION, + expires=self.EXP_TIME) + + self.assertEqual(len(conn1._requested), 0) + self.assertEqual(len(conn2._requested), 1) + req = conn2._requested[0] + self.assertEqual(req['method'], 'PATCH') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'view': {'query': QUERY}, + 'location': LOCATION, + 'expirationTime': _millis(self.EXP_TIME), + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(table, RESOURCE) + + def test_update_w_bound_client(self): + from gcloud.bigquery.table import SchemaField + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + DESCRIPTION = 'DESCRIPTION' + TITLE = 'TITLE' + RESOURCE = self._makeResource() + RESOURCE['description'] = DESCRIPTION + RESOURCE['friendlyName'] = TITLE + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = self._makeOne(self.TABLE_NAME, dataset=dataset, + schema=[full_name, age]) + table.description = DESCRIPTION + table.friendly_name = TITLE + + table.update() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'PUT') + SENT = { + 'tableReference': + {'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + 'tableId': self.TABLE_NAME}, + 'schema': {'fields': [ + {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, + 'description': DESCRIPTION, + 'friendlyName': TITLE, + } + self.assertEqual(req['data'], SENT) + self.assertEqual(req['path'], '/%s' % PATH) + self._verifyResourceProperties(table, RESOURCE) + + def test_update_w_alternate_client(self): + import datetime + import pytz + from gcloud.bigquery._helpers import _millis + from gcloud.bigquery.table import SchemaField + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + DEF_TABLE_EXP = 12345 + LOCATION = 'EU' + QUERY = 'select fullname, age from person_ages' + RESOURCE = self._makeResource() + RESOURCE['defaultTableExpirationMs'] = 12345 + RESOURCE['location'] = LOCATION + self.EXP_TIME = datetime.datetime(2015, 8, 1, 23, 59, 59, + tzinfo=pytz.utc) + RESOURCE['expirationTime'] = _millis(self.EXP_TIME) + RESOURCE['view'] = {'query': QUERY} + RESOURCE['type'] = 'VIEW' + conn1 = _Connection() + client1 = _Client(project=self.PROJECT, connection=conn1) + conn2 = _Connection(RESOURCE) + client2 = _Client(project=self.PROJECT, connection=conn2) + dataset = _Dataset(client1) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = self._makeOne(self.TABLE_NAME, dataset=dataset, + schema=[full_name, age]) + table.default_table_expiration_ms = DEF_TABLE_EXP + table.location = LOCATION + table.expires = self.EXP_TIME + table.view_query = QUERY + + table.update(client=client2) + + self.assertEqual(len(conn1._requested), 0) + self.assertEqual(len(conn2._requested), 1) + req = conn2._requested[0] + self.assertEqual(req['method'], 'PUT') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'tableReference': + {'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + 'tableId': self.TABLE_NAME}, + 'schema': {'fields': [ + {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, + 'expirationTime': _millis(self.EXP_TIME), + 'location': 'EU', + 'view': {'query': QUERY}, + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(table, RESOURCE) + + def test_delete_w_bound_client(self): + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + conn = _Connection({}) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + table.delete() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'DELETE') + self.assertEqual(req['path'], '/%s' % PATH) + + def test_delete_w_alternate_client(self): + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + conn1 = _Connection() + client1 = _Client(project=self.PROJECT, connection=conn1) + conn2 = _Connection({}) + client2 = _Client(project=self.PROJECT, connection=conn2) + dataset = _Dataset(client1) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + table.delete(client=client2) + + self.assertEqual(len(conn1._requested), 0) + self.assertEqual(len(conn2._requested), 1) + req = conn2._requested[0] + self.assertEqual(req['method'], 'DELETE') + self.assertEqual(req['path'], '/%s' % PATH) + class _Client(object): @@ -261,9 +738,31 @@ class _Dataset(object): def __init__(self, client, name=TestTable.DS_NAME): self._client = client - self._name = name + self.name = name @property def path(self): return '/projects/%s/datasets/%s' % ( - self._client.project, self._name) + self._client.project, self.name) + + @property + def project(self): + return self._client.project + + +class _Connection(object): + + def __init__(self, *responses): + self._responses = responses + self._requested = [] + + def api_request(self, **kw): + from gcloud.exceptions import NotFound + self._requested.append(kw) + + try: + response, self._responses = self._responses[0], self._responses[1:] + except: + raise NotFound('miss') + else: + return response From da9cb31e52dfd4c9105df36f1afc5df8c3aa299d Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 29 Jul 2015 16:32:57 -0400 Subject: [PATCH 2/7] Add 'Dataset.table' factory. --- gcloud/bigquery/dataset.py | 15 +++++++++++++++ gcloud/bigquery/test_dataset.py | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index aaee1c7bd3a6..df6cc4d07ffc 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -17,6 +17,7 @@ from gcloud.exceptions import NotFound from gcloud.bigquery._helpers import _datetime_from_prop +from gcloud.bigquery.table import Table class Dataset(object): @@ -353,3 +354,17 @@ def delete(self, client=None): """ client = self._require_client(client) client.connection.api_request(method='DELETE', path=self.path) + + def table(self, name, schema=()): + """Construct a table bound to this dataset. + + :type name: string + :param name: Name of the table. + + :rtype: :class:`gcloud.bigquery.table.Table` + :returns: a new ``Table`` instance + + :type schema: list of :class:`gcloud.bigquery.table.SchemaField` + :param schema: The table's schema + """ + return Table(name, dataset=self, schema=schema) diff --git a/gcloud/bigquery/test_dataset.py b/gcloud/bigquery/test_dataset.py index d15e9d751712..2a0214316f2f 100644 --- a/gcloud/bigquery/test_dataset.py +++ b/gcloud/bigquery/test_dataset.py @@ -393,6 +393,31 @@ def test_delete_w_alternate_client(self): self.assertEqual(req['method'], 'DELETE') self.assertEqual(req['path'], '/%s' % PATH) + def test_table_wo_schema(self): + from gcloud.bigquery.table import Table + conn = _Connection({}) + CLIENT = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=CLIENT) + table = dataset.table('table_name') + self.assertTrue(isinstance(table, Table)) + self.assertEqual(table.name, 'table_name') + self.assertTrue(table._dataset is dataset) + self.assertEqual(table.schema, []) + + def test_table_w_schema(self): + from gcloud.bigquery.table import SchemaField + from gcloud.bigquery.table import Table + conn = _Connection({}) + CLIENT = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=CLIENT) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = dataset.table('table_name', schema=[full_name, age]) + self.assertTrue(isinstance(table, Table)) + self.assertEqual(table.name, 'table_name') + self.assertTrue(table._dataset is dataset) + self.assertEqual(table.schema, [full_name, age]) + class _Client(object): From d9776419465baab9d61b4df97b6f6a4ed86187ec Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 4 Aug 2015 14:22:08 -0400 Subject: [PATCH 3/7] Move :rtype:/:returns: to end of docstring. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1023#discussion_r36219997 --- gcloud/bigquery/dataset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index df6cc4d07ffc..3092ed102ef2 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -361,10 +361,10 @@ def table(self, name, schema=()): :type name: string :param name: Name of the table. - :rtype: :class:`gcloud.bigquery.table.Table` - :returns: a new ``Table`` instance - :type schema: list of :class:`gcloud.bigquery.table.SchemaField` :param schema: The table's schema + + :rtype: :class:`gcloud.bigquery.table.Table` + :returns: a new ``Table`` instance """ return Table(name, dataset=self, schema=schema) From f3e5dcce1d8a9c9141a05e5eed08981fa1d344f3 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 4 Aug 2015 18:45:13 -0400 Subject: [PATCH 4/7] Replace '**kw' w/ named, defaulted args. Addresses: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1023#discussion_r36225263 --- gcloud/bigquery/table.py | 48 ++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index d4edb7a86c94..ad8b3d9987fe 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -23,6 +23,9 @@ from gcloud.bigquery._helpers import _prop_from_datetime +_MARKER = object() + + class SchemaField(object): """Describe a single field within a table schema. @@ -282,7 +285,7 @@ def view_query(self, value): """Update SQL query defining the table as a view. :type value: string - :param value: new location + :param value: new query :raises: ValueError for invalid value types. """ @@ -420,7 +423,8 @@ def reload(self, client=None): method='GET', path=self.path) self._set_properties(api_response) - def patch(self, client=None, **kw): + def patch(self, client=None, friendly_name=_MARKER, description=_MARKER, + location=_MARKER, expires=_MARKER, view_query=_MARKER): """API call: update individual table properties via a PATCH request See @@ -430,8 +434,20 @@ def patch(self, client=None, **kw): :param client: the client to use. If not passed, falls back to the ``client`` stored on the current dataset. - :type kw: ``dict`` - :param kw: properties to be patched. + :type friendly_name: string or ``NoneType`` + :param friendly_name: point in time at which the table expires. + + :type description: string or ``NoneType`` + :param description: point in time at which the table expires. + + :type location: string or ``NoneType`` + :param location: point in time at which the table expires. + + :type expires: :class:`datetime.datetime` or ``NoneType`` + :param expires: point in time at which the table expires. + + :type view_query: string + :param view_query: SQL query defining the table as a view :raises: ValueError for invalid value types. """ @@ -439,23 +455,23 @@ def patch(self, client=None, **kw): partial = {} - if 'expires' in kw: - value = kw['expires'] - if not isinstance(value, datetime.datetime) and value is not None: + if expires is not _MARKER: + if (not isinstance(expires, datetime.datetime) and + expires is not None): raise ValueError("Pass a datetime, or None") - partial['expirationTime'] = _prop_from_datetime(value) + partial['expirationTime'] = _prop_from_datetime(expires) - if 'description' in kw: - partial['description'] = kw['description'] + if description is not _MARKER: + partial['description'] = description - if 'friendly_name' in kw: - partial['friendlyName'] = kw['friendly_name'] + if friendly_name is not _MARKER: + partial['friendlyName'] = friendly_name - if 'location' in kw: - partial['location'] = kw['location'] + if location is not _MARKER: + partial['location'] = location - if 'view_query' in kw: - partial['view'] = {'query': kw['view_query']} + if view_query is not _MARKER: + partial['view'] = {'query': view_query} api_response = client.connection.api_request( method='PATCH', path=self.path, data=partial) From d22d7dae028a4b2be2533949a15bb25de7ba246b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 5 Aug 2015 11:41:44 -0400 Subject: [PATCH 5/7] Handle PATCH w/ 'view_query=None'. See: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1023#discussion_r36316570 --- gcloud/bigquery/table.py | 5 ++++- gcloud/bigquery/test_table.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index ad8b3d9987fe..b2f5f6fa13cf 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -471,7 +471,10 @@ def patch(self, client=None, friendly_name=_MARKER, description=_MARKER, partial['location'] = location if view_query is not _MARKER: - partial['view'] = {'query': view_query} + if view_query is None: + partial['view'] = None + else: + partial['view'] = {'query': view_query} api_response = client.connection.api_request( method='PATCH', path=self.path, data=partial) diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index 3edd28c773ee..e36e2d3799d8 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -549,7 +549,9 @@ def test_patch_w_bound_client(self): dataset = _Dataset(client) table = self._makeOne(self.TABLE_NAME, dataset=dataset) - table.patch(description=DESCRIPTION, friendly_name=TITLE) + table.patch(description=DESCRIPTION, + friendly_name=TITLE, + view_query=None) self.assertEqual(len(conn._requested), 1) req = conn._requested[0] @@ -557,6 +559,7 @@ def test_patch_w_bound_client(self): SENT = { 'description': DESCRIPTION, 'friendlyName': TITLE, + 'view': None, } self.assertEqual(req['data'], SENT) self.assertEqual(req['path'], '/%s' % PATH) From 22bc7cdead88a8d77351465c08e00bbb1f9e5c9e Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 5 Aug 2015 12:00:39 -0400 Subject: [PATCH 6/7] Update 'Table.patch()' to support patching schema. See: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1023#issuecomment-128046695. --- gcloud/bigquery/table.py | 20 ++++++++++++++++++-- gcloud/bigquery/test_table.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index b2f5f6fa13cf..addd1acfe11e 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -423,8 +423,14 @@ def reload(self, client=None): method='GET', path=self.path) self._set_properties(api_response) - def patch(self, client=None, friendly_name=_MARKER, description=_MARKER, - location=_MARKER, expires=_MARKER, view_query=_MARKER): + def patch(self, + client=None, + friendly_name=_MARKER, + description=_MARKER, + location=_MARKER, + expires=_MARKER, + view_query=_MARKER, + schema=_MARKER): """API call: update individual table properties via a PATCH request See @@ -449,6 +455,9 @@ def patch(self, client=None, friendly_name=_MARKER, description=_MARKER, :type view_query: string :param view_query: SQL query defining the table as a view + :type schema: list of :class:`SchemaField` + :param schema: fields describing the schema + :raises: ValueError for invalid value types. """ client = self._require_client(client) @@ -476,6 +485,13 @@ def patch(self, client=None, friendly_name=_MARKER, description=_MARKER, else: partial['view'] = {'query': view_query} + if schema is not _MARKER: + if schema is None: + partial['schema'] = None + else: + partial['schema'] = { + 'fields': self._build_schema_resource(schema)} + api_response = client.connection.api_request( method='PATCH', path=self.path, data=partial) self._set_properties(api_response) diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index e36e2d3799d8..d11c2c130d02 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -569,6 +569,7 @@ def test_patch_w_alternate_client(self): import datetime import pytz from gcloud.bigquery._helpers import _millis + from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME) QUERY = 'select fullname, age from person_ages' @@ -586,9 +587,11 @@ def test_patch_w_alternate_client(self): client2 = _Client(project=self.PROJECT, connection=conn2) dataset = _Dataset(client1) table = self._makeOne(self.TABLE_NAME, dataset=dataset) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='OPTIONAL') table.patch(client=client2, view_query=QUERY, location=LOCATION, - expires=self.EXP_TIME) + expires=self.EXP_TIME, schema=[full_name, age]) self.assertEqual(len(conn1._requested), 0) self.assertEqual(len(conn2._requested), 1) @@ -599,10 +602,38 @@ def test_patch_w_alternate_client(self): 'view': {'query': QUERY}, 'location': LOCATION, 'expirationTime': _millis(self.EXP_TIME), + 'schema': {'fields': [ + {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'age', 'type': 'INTEGER', 'mode': 'OPTIONAL'}]}, } self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) + def test_patch_w_schema_None(self): + # Simulate deleting schema: not sure if back-end will actually + # allow this operation, but the spec says it is optional. + PATH = 'projects/%s/datasets/%s/tables/%s' % ( + self.PROJECT, self.DS_NAME, self.TABLE_NAME) + DESCRIPTION = 'DESCRIPTION' + TITLE = 'TITLE' + RESOURCE = self._makeResource() + RESOURCE['description'] = DESCRIPTION + RESOURCE['friendlyName'] = TITLE + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + table = self._makeOne(self.TABLE_NAME, dataset=dataset) + + table.patch(schema=None) + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'PATCH') + SENT = {'schema': None} + self.assertEqual(req['data'], SENT) + self.assertEqual(req['path'], '/%s' % PATH) + self._verifyResourceProperties(table, RESOURCE) + def test_update_w_bound_client(self): from gcloud.bigquery.table import SchemaField PATH = 'projects/%s/datasets/%s/tables/%s' % ( From 7210c8c86c4297cd33f4272b70e0865c7a59d450 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Wed, 5 Aug 2015 12:20:39 -0400 Subject: [PATCH 7/7] Defend against missing 'creationTime'/'lastModifiedTime' from back-end. See: https://github.com/GoogleCloudPlatform/gcloud-python/pull/1023#discussion_r36223517 --- gcloud/bigquery/dataset.py | 6 ++++-- gcloud/bigquery/table.py | 6 ++++-- gcloud/bigquery/test_dataset.py | 25 +++++++++++++++++++++++ gcloud/bigquery/test_table.py | 35 +++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 4 deletions(-) diff --git a/gcloud/bigquery/dataset.py b/gcloud/bigquery/dataset.py index 3092ed102ef2..c725b8200462 100644 --- a/gcloud/bigquery/dataset.py +++ b/gcloud/bigquery/dataset.py @@ -212,8 +212,10 @@ def _set_properties(self, api_response): """ self._properties.clear() cleaned = api_response.copy() - cleaned['creationTime'] = float(cleaned['creationTime']) - cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime']) + if 'creationTime' in cleaned: + cleaned['creationTime'] = float(cleaned['creationTime']) + if 'lastModifiedTime' in cleaned: + cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime']) self._properties.update(cleaned) def _build_resource(self): diff --git a/gcloud/bigquery/table.py b/gcloud/bigquery/table.py index addd1acfe11e..bd537375ce2f 100644 --- a/gcloud/bigquery/table.py +++ b/gcloud/bigquery/table.py @@ -320,8 +320,10 @@ def _set_properties(self, api_response): """ self._properties.clear() cleaned = api_response.copy() - cleaned['creationTime'] = float(cleaned['creationTime']) - cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime']) + if 'creationTime' in cleaned: + cleaned['creationTime'] = float(cleaned['creationTime']) + if 'lastModifiedTime' in cleaned: + cleaned['lastModifiedTime'] = float(cleaned['lastModifiedTime']) if 'expirationTime' in cleaned: cleaned['expirationTime'] = float(cleaned['expirationTime']) self._properties.update(cleaned) diff --git a/gcloud/bigquery/test_dataset.py b/gcloud/bigquery/test_dataset.py index 2a0214316f2f..1ae3699c2e57 100644 --- a/gcloud/bigquery/test_dataset.py +++ b/gcloud/bigquery/test_dataset.py @@ -179,6 +179,31 @@ def test_create_w_alternate_client(self): self.assertEqual(req['data'], SENT) self._verifyResourceProperties(dataset, RESOURCE) + def test_create_w_missing_output_properties(self): + # In the wild, the resource returned from 'dataset.create' sometimes + # lacks 'creationTime' / 'lastModifiedTime' + PATH = 'projects/%s/datasets' % (self.PROJECT,) + RESOURCE = self._makeResource() + del RESOURCE['creationTime'] + del RESOURCE['lastModifiedTime'] + self.WHEN = None + conn = _Connection(RESOURCE) + CLIENT = _Client(project=self.PROJECT, connection=conn) + dataset = self._makeOne(self.DS_NAME, client=CLIENT) + + dataset.create() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'datasetReference': + {'projectId': self.PROJECT, 'datasetId': self.DS_NAME}, + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(dataset, RESOURCE) + def test_exists_miss_w_bound_client(self): PATH = 'projects/%s/datasets/%s' % (self.PROJECT, self.DS_NAME) conn = _Connection() diff --git a/gcloud/bigquery/test_table.py b/gcloud/bigquery/test_table.py index d11c2c130d02..dcc30b16e5fb 100644 --- a/gcloud/bigquery/test_table.py +++ b/gcloud/bigquery/test_table.py @@ -454,6 +454,41 @@ def test_create_w_alternate_client(self): self.assertEqual(req['data'], SENT) self._verifyResourceProperties(table, RESOURCE) + def test_create_w_missing_output_properties(self): + # In the wild, the resource returned from 'dataset.create' sometimes + # lacks 'creationTime' / 'lastModifiedTime' + from gcloud.bigquery.table import SchemaField + PATH = 'projects/%s/datasets/%s/tables' % (self.PROJECT, self.DS_NAME) + RESOURCE = self._makeResource() + del RESOURCE['creationTime'] + del RESOURCE['lastModifiedTime'] + self.WHEN = None + conn = _Connection(RESOURCE) + client = _Client(project=self.PROJECT, connection=conn) + dataset = _Dataset(client) + full_name = SchemaField('full_name', 'STRING', mode='REQUIRED') + age = SchemaField('age', 'INTEGER', mode='REQUIRED') + table = self._makeOne(self.TABLE_NAME, dataset, + schema=[full_name, age]) + + table.create() + + self.assertEqual(len(conn._requested), 1) + req = conn._requested[0] + self.assertEqual(req['method'], 'POST') + self.assertEqual(req['path'], '/%s' % PATH) + SENT = { + 'tableReference': { + 'projectId': self.PROJECT, + 'datasetId': self.DS_NAME, + 'tableId': self.TABLE_NAME}, + 'schema': {'fields': [ + {'name': 'full_name', 'type': 'STRING', 'mode': 'REQUIRED'}, + {'name': 'age', 'type': 'INTEGER', 'mode': 'REQUIRED'}]}, + } + self.assertEqual(req['data'], SENT) + self._verifyResourceProperties(table, RESOURCE) + def test_exists_miss_w_bound_client(self): PATH = 'projects/%s/datasets/%s/tables/%s' % ( self.PROJECT, self.DS_NAME, self.TABLE_NAME)