From 55cfd5c97a582ff0579f2a9f0c04fcd6958c0e96 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 31 Oct 2014 14:41:00 -0400 Subject: [PATCH] Rename key iterators to indicate they are implementation details of Key. Rename 'key.KeyIterator' -> 'key._KeyIterator' Move / rename 'iterator.KeyDataIterator' -> 'key._KeyDataIterator'. Fixes #244. --- gcloud/storage/bucket.py | 4 +- gcloud/storage/iterator.py | 96 ---------------- gcloud/storage/key.py | 102 ++++++++++++++++- gcloud/storage/test_bucket.py | 4 +- gcloud/storage/test_iterator.py | 191 ------------------------------- gcloud/storage/test_key.py | 192 +++++++++++++++++++++++++++++++- 6 files changed, 288 insertions(+), 301 deletions(-) diff --git a/gcloud/storage/bucket.py b/gcloud/storage/bucket.py index 4f82ef791ab6..4c1bbc9b66f0 100644 --- a/gcloud/storage/bucket.py +++ b/gcloud/storage/bucket.py @@ -7,7 +7,7 @@ from gcloud.storage.acl import DefaultObjectACL from gcloud.storage.iterator import Iterator from gcloud.storage.key import Key -from gcloud.storage.key import KeyIterator +from gcloud.storage.key import _KeyIterator class Bucket(object): @@ -46,7 +46,7 @@ def __repr__(self): return '' % self.name def __iter__(self): - return iter(KeyIterator(bucket=self)) + return iter(_KeyIterator(bucket=self)) def __contains__(self, key): return self.get_key(key) is not None diff --git a/gcloud/storage/iterator.py b/gcloud/storage/iterator.py index 70381f76d3ad..9f3d098df4cf 100644 --- a/gcloud/storage/iterator.py +++ b/gcloud/storage/iterator.py @@ -28,8 +28,6 @@ def get_items_from_response(self, response): >>> break """ -from gcloud.storage.exceptions import StorageError - class Iterator(object): """A generic class for iterating through Cloud Storage list responses. @@ -113,97 +111,3 @@ def get_items_from_response(self, response): :returns: Items that the iterator should yield. """ raise NotImplementedError - - -class KeyDataIterator(object): - """An iterator listing data stored in a key. - - You shouldn't have to use this directly, but instead should use the - helper methods on :class:`gcloud.storage.key.Key` objects. - - :type key: :class:`gcloud.storage.key.Key` - :param key: The key from which to list data.. - """ - - def __init__(self, key): - self.key = key - # NOTE: These variables will be initialized by reset(). - self._bytes_written = None - self._total_bytes = None - self.reset() - - def __iter__(self): - while self.has_more_data(): - yield self.get_next_chunk() - - def reset(self): - """Resets the iterator to the beginning.""" - self._bytes_written = 0 - self._total_bytes = None - - def has_more_data(self): - """Determines whether or not this iterator has more data to read. - - :rtype: bool - :returns: Whether the iterator has more data or not. - """ - if self._bytes_written == 0: - return True - elif not self._total_bytes: - # self._total_bytes **should** be set by this point. - # If it isn't, something is wrong. - raise ValueError('Size of object is unknown.') - else: - return self._bytes_written < self._total_bytes - - def get_headers(self): - """Gets range header(s) for next chunk of data. - - :rtype: dict - :returns: A dictionary of query parameters. - """ - start = self._bytes_written - end = self._bytes_written + self.key.CHUNK_SIZE - 1 - - if self._total_bytes and end > self._total_bytes: - end = '' - - return {'Range': 'bytes=%s-%s' % (start, end)} - - def get_url(self): - """Gets URL to read next chunk of data. - - :rtype: string - :returns: A URL. - """ - return self.key.connection.build_api_url( - path=self.key.path, query_params={'alt': 'media'}) - - def get_next_chunk(self): - """Gets the next chunk of data. - - Uses CHUNK_SIZE to determine how much data to get. - - :rtype: string - :returns: The chunk of data read from the key. - :raises: :class:`RuntimeError` if no more data or - :class:`gcloud.storage.exceptions.StorageError` in the - case of an unexpected response status code. - """ - if not self.has_more_data(): - raise RuntimeError('No more data in this iterator. Try resetting.') - - response, content = self.key.connection.make_request( - method='GET', url=self.get_url(), headers=self.get_headers()) - - if response.status in (200, 206): - self._bytes_written += len(content) - - if 'content-range' in response: - content_range = response['content-range'] - self._total_bytes = int(content_range.rsplit('/', 1)[1]) - - return content - - # Expected a 200 or a 206. Got something else, which is unknown. - raise StorageError(response) diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 6173ecf3c034..f8f22fa682cf 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -5,8 +5,8 @@ from StringIO import StringIO from gcloud.storage.acl import ObjectACL +from gcloud.storage.exceptions import StorageError from gcloud.storage.iterator import Iterator -from gcloud.storage.iterator import KeyDataIterator class Key(object): @@ -170,7 +170,7 @@ def get_contents_to_file(self, file_obj): :raises: :class:`gcloud.storage.exceptions.NotFoundError` """ - for chunk in KeyDataIterator(self): + for chunk in _KeyDataIterator(self): file_obj.write(chunk) def get_contents_to_filename(self, filename): @@ -433,7 +433,7 @@ def make_public(self): return self -class KeyIterator(Iterator): +class _KeyIterator(Iterator): """An iterator listing keys. You shouldn't have to use this directly, but instead should use the @@ -444,7 +444,7 @@ class KeyIterator(Iterator): """ def __init__(self, bucket): self.bucket = bucket - super(KeyIterator, self).__init__( + super(_KeyIterator, self).__init__( connection=bucket.connection, path=bucket.path + '/o') def get_items_from_response(self, response): @@ -455,3 +455,97 @@ def get_items_from_response(self, response): """ for item in response.get('items', []): yield Key.from_dict(item, bucket=self.bucket) + + +class _KeyDataIterator(object): + """An iterator listing data stored in a key. + + You shouldn't have to use this directly, but instead should use the + helper methods on :class:`gcloud.storage.key.Key` objects. + + :type key: :class:`gcloud.storage.key.Key` + :param key: The key from which to list data.. + """ + + def __init__(self, key): + self.key = key + # NOTE: These variables will be initialized by reset(). + self._bytes_written = None + self._total_bytes = None + self.reset() + + def __iter__(self): + while self.has_more_data(): + yield self.get_next_chunk() + + def reset(self): + """Resets the iterator to the beginning.""" + self._bytes_written = 0 + self._total_bytes = None + + def has_more_data(self): + """Determines whether or not this iterator has more data to read. + + :rtype: bool + :returns: Whether the iterator has more data or not. + """ + if self._bytes_written == 0: + return True + elif not self._total_bytes: + # self._total_bytes **should** be set by this point. + # If it isn't, something is wrong. + raise ValueError('Size of object is unknown.') + else: + return self._bytes_written < self._total_bytes + + def get_headers(self): + """Gets range header(s) for next chunk of data. + + :rtype: dict + :returns: A dictionary of query parameters. + """ + start = self._bytes_written + end = self._bytes_written + self.key.CHUNK_SIZE - 1 + + if self._total_bytes and end > self._total_bytes: + end = '' + + return {'Range': 'bytes=%s-%s' % (start, end)} + + def get_url(self): + """Gets URL to read next chunk of data. + + :rtype: string + :returns: A URL. + """ + return self.key.connection.build_api_url( + path=self.key.path, query_params={'alt': 'media'}) + + def get_next_chunk(self): + """Gets the next chunk of data. + + Uses CHUNK_SIZE to determine how much data to get. + + :rtype: string + :returns: The chunk of data read from the key. + :raises: :class:`RuntimeError` if no more data or + :class:`gcloud.storage.exceptions.StorageError` in the + case of an unexpected response status code. + """ + if not self.has_more_data(): + raise RuntimeError('No more data in this iterator. Try resetting.') + + response, content = self.key.connection.make_request( + method='GET', url=self.get_url(), headers=self.get_headers()) + + if response.status in (200, 206): + self._bytes_written += len(content) + + if 'content-range' in response: + content_range = response['content-range'] + self._total_bytes = int(content_range.rsplit('/', 1)[1]) + + return content + + # Expected a 200 or a 206. Got something else, which is unknown. + raise StorageError(response) diff --git a/gcloud/storage/test_bucket.py b/gcloud/storage/test_bucket.py index 8ae603a67cd0..e7869ce0cab3 100644 --- a/gcloud/storage/test_bucket.py +++ b/gcloud/storage/test_bucket.py @@ -863,7 +863,7 @@ def grant_read(self): def save_acl(self): _saved.append((self._bucket, self._name, self._granted)) - class _KeyIterator(key.KeyIterator): + class _KeyIterator(key._KeyIterator): def get_items_from_response(self, response): for item in response.get('items', []): yield _Key(self.bucket, item['name']) @@ -875,7 +875,7 @@ def get_items_from_response(self, response): after = {'acl': permissive, 'defaultObjectAcl': []} connection = _Connection(after, {'items': [{'name': KEY}]}) bucket = self._makeOne(connection, NAME, before) - with _Monkey(MUT, KeyIterator=_KeyIterator): + with _Monkey(MUT, _KeyIterator=_KeyIterator): bucket.make_public(recursive=True) self.assertEqual(bucket.metadata, after) self.assertEqual(list(bucket.acl), after['acl']) diff --git a/gcloud/storage/test_iterator.py b/gcloud/storage/test_iterator.py index 8d441b1dce5c..954bc46f1473 100644 --- a/gcloud/storage/test_iterator.py +++ b/gcloud/storage/test_iterator.py @@ -118,204 +118,13 @@ def test_get_items_from_response_raises_NotImplementedError(self): iterator.get_items_from_response, object()) -class TestKeyDataIterator(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud.storage.iterator import KeyDataIterator - return KeyDataIterator - - def _makeOne(self, *args, **kw): - return self._getTargetClass()(*args, **kw) - - def test_ctor(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - self.assertTrue(iterator.key is key) - self.assertEqual(iterator._bytes_written, 0) - self.assertEqual(iterator._total_bytes, None) - - def test__iter__(self): - response1 = _Response(status=200) - response1['content-range'] = '0-9/15' - response2 = _Response(status=200) - response2['content-range'] = '10-14/15' - connection = _Connection( - (response1, '0123456789'), - (response2, '01234'), - ) - key = _Key(connection) - iterator = self._makeOne(key) - chunks = list(iterator) - self.assertEqual(len(chunks), 2) - self.assertEqual(chunks[0], '0123456789') - self.assertEqual(chunks[1], '01234') - self.assertEqual(iterator._bytes_written, 15) - self.assertEqual(iterator._total_bytes, 15) - kws = connection._requested - self.assertEqual(kws[0]['method'], 'GET') - self.assertEqual(kws[0]['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kws[0]['headers'], {'Range': 'bytes=0-9'}) - self.assertEqual(kws[1]['method'], 'GET') - self.assertEqual(kws[1]['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kws[1]['headers'], {'Range': 'bytes=10-'}) - - def test_reset(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 - iterator._total_bytes = 1000 - iterator.reset() - self.assertEqual(iterator._bytes_written, 0) - self.assertEqual(iterator._total_bytes, None) - - def test_has_more_data_new(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - self.assertTrue(iterator.has_more_data()) - - def test_has_more_data_invalid(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 # no _total_bytes. - self.assertRaises(ValueError, iterator.has_more_data) - - def test_has_more_data_true(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 - iterator._total_bytes = 1000 - self.assertTrue(iterator.has_more_data()) - - def test_has_more_data_false(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 1000 - iterator._total_bytes = 1000 - self.assertFalse(iterator.has_more_data()) - - def test_get_headers_new(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - headers = iterator.get_headers() - self.assertEqual(len(headers), 1) - self.assertEqual(headers['Range'], 'bytes=0-9') - - def test_get_headers_ok(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 - iterator._total_bytes = 1000 - headers = iterator.get_headers() - self.assertEqual(len(headers), 1) - self.assertEqual(headers['Range'], 'bytes=10-19') - - def test_get_headers_off_end(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 95 - iterator._total_bytes = 100 - headers = iterator.get_headers() - self.assertEqual(len(headers), 1) - self.assertEqual(headers['Range'], 'bytes=95-') - - def test_get_url(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - self.assertEqual(iterator.get_url(), - 'http://example.com/b/name/o/key?alt=media') - - def test_get_next_chunk_underflow(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = iterator._total_bytes = 10 - self.assertRaises(RuntimeError, iterator.get_next_chunk) - - def test_get_next_chunk_200(self): - response = _Response(status=200) - response['content-range'] = '0-9/100' - connection = _Connection((response, 'CHUNK')) - key = _Key(connection) - iterator = self._makeOne(key) - chunk = iterator.get_next_chunk() - self.assertEqual(chunk, 'CHUNK') - self.assertEqual(iterator._bytes_written, len(chunk)) - self.assertEqual(iterator._total_bytes, 100) - kw, = connection._requested - self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'}) - - def test_get_next_chunk_206(self): - response = _Response(status=206) - connection = _Connection((response, 'CHUNK')) - key = _Key(connection) - iterator = self._makeOne(key) - iterator._total_bytes = 1000 - chunk = iterator.get_next_chunk() - self.assertEqual(chunk, 'CHUNK') - self.assertEqual(iterator._bytes_written, len(chunk)) - kw, = connection._requested - self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'}) - - def test_get_next_chunk_416(self): - from gcloud.storage.exceptions import StorageError - response = _Response(status=416) - connection = _Connection((response, '')) - key = _Key(connection) - iterator = self._makeOne(key) - iterator._total_bytes = 1000 - self.assertRaises(StorageError, iterator.get_next_chunk) - - -class _Response(dict): - @property - def status(self): - return self['status'] - - class _Connection(object): def __init__(self, *responses): self._responses = responses self._requested = [] - def make_request(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response - def api_request(self, **kw): self._requested.append(kw) response, self._responses = self._responses[0], self._responses[1:] return response - - def build_api_url(self, path, query_params=None): - from urllib import urlencode - from urlparse import urlunsplit - qs = urlencode(query_params or {}) - return urlunsplit(('http', 'example.com', path, qs, '')) - - -class _Key(object): - CHUNK_SIZE = 10 - path = '/b/name/o/key' - - def __init__(self, connection): - self.connection = connection diff --git a/gcloud/storage/test_key.py b/gcloud/storage/test_key.py index 9d02a16f8c9a..3a401c8bc016 100644 --- a/gcloud/storage/test_key.py +++ b/gcloud/storage/test_key.py @@ -152,7 +152,7 @@ def test_get_contents_to_file(self): bucket = _Bucket(connection) key = self._makeOne(bucket, KEY) fh = StringIO() - with _Monkey(MUT, KeyDataIterator=lambda self: iter(_CHUNKS)): + with _Monkey(MUT, _KeyDataIterator=lambda self: iter(_CHUNKS)): key.get_contents_to_file(fh) self.assertEqual(fh.getvalue(), ''.join(_CHUNKS)) @@ -165,7 +165,7 @@ def test_get_contents_to_filename(self): connection = _Connection() bucket = _Bucket(connection) key = self._makeOne(bucket, KEY) - with _Monkey(MUT, KeyDataIterator=lambda self: iter(_CHUNKS)): + with _Monkey(MUT, _KeyDataIterator=lambda self: iter(_CHUNKS)): with NamedTemporaryFile() as f: key.get_contents_to_filename(f.name) f.flush() @@ -181,7 +181,7 @@ def test_get_contents_as_string(self): connection = _Connection() bucket = _Bucket(connection) key = self._makeOne(bucket, KEY) - with _Monkey(MUT, KeyDataIterator=lambda self: iter(_CHUNKS)): + with _Monkey(MUT, _KeyDataIterator=lambda self: iter(_CHUNKS)): fetched = key.get_contents_as_string() self.assertEqual(fetched, ''.join(_CHUNKS)) @@ -591,11 +591,11 @@ def test_make_public(self): self.assertEqual(kw[0]['query_params'], {'projection': 'full'}) -class TestKeyIterator(unittest2.TestCase): +class Test__KeyIterator(unittest2.TestCase): def _getTargetClass(self): - from gcloud.storage.key import KeyIterator - return KeyIterator + from gcloud.storage.key import _KeyIterator + return _KeyIterator def _makeOne(self, *args, **kw): return self._getTargetClass()(*args, **kw) @@ -631,6 +631,172 @@ def test_get_items_from_response_non_empty(self): self.assertEqual(key.name, KEY) +class Test__KeyDataIterator(unittest2.TestCase): + + def _getTargetClass(self): + from gcloud.storage.key import _KeyDataIterator + return _KeyDataIterator + + def _makeOne(self, *args, **kw): + return self._getTargetClass()(*args, **kw) + + def test_ctor(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + self.assertTrue(iterator.key is key) + self.assertEqual(iterator._bytes_written, 0) + self.assertEqual(iterator._total_bytes, None) + + def test__iter__(self): + response1 = _Response(status=200) + response1['content-range'] = '0-9/15' + response2 = _Response(status=200) + response2['content-range'] = '10-14/15' + connection = _Connection( + (response1, '0123456789'), + (response2, '01234'), + ) + key = _Key(connection) + iterator = self._makeOne(key) + chunks = list(iterator) + self.assertEqual(len(chunks), 2) + self.assertEqual(chunks[0], '0123456789') + self.assertEqual(chunks[1], '01234') + self.assertEqual(iterator._bytes_written, 15) + self.assertEqual(iterator._total_bytes, 15) + kws = connection._requested + self.assertEqual(kws[0]['method'], 'GET') + self.assertEqual(kws[0]['url'], + 'http://example.com/b/name/o/key?alt=media') + self.assertEqual(kws[0]['headers'], {'Range': 'bytes=0-9'}) + self.assertEqual(kws[1]['method'], 'GET') + self.assertEqual(kws[1]['url'], + 'http://example.com/b/name/o/key?alt=media') + self.assertEqual(kws[1]['headers'], {'Range': 'bytes=10-'}) + + def test_reset(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + iterator._bytes_written = 10 + iterator._total_bytes = 1000 + iterator.reset() + self.assertEqual(iterator._bytes_written, 0) + self.assertEqual(iterator._total_bytes, None) + + def test_has_more_data_new(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + self.assertTrue(iterator.has_more_data()) + + def test_has_more_data_invalid(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + iterator._bytes_written = 10 # no _total_bytes. + self.assertRaises(ValueError, iterator.has_more_data) + + def test_has_more_data_true(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + iterator._bytes_written = 10 + iterator._total_bytes = 1000 + self.assertTrue(iterator.has_more_data()) + + def test_has_more_data_false(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + iterator._bytes_written = 1000 + iterator._total_bytes = 1000 + self.assertFalse(iterator.has_more_data()) + + def test_get_headers_new(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + headers = iterator.get_headers() + self.assertEqual(len(headers), 1) + self.assertEqual(headers['Range'], 'bytes=0-9') + + def test_get_headers_ok(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + iterator._bytes_written = 10 + iterator._total_bytes = 1000 + headers = iterator.get_headers() + self.assertEqual(len(headers), 1) + self.assertEqual(headers['Range'], 'bytes=10-19') + + def test_get_headers_off_end(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + iterator._bytes_written = 95 + iterator._total_bytes = 100 + headers = iterator.get_headers() + self.assertEqual(len(headers), 1) + self.assertEqual(headers['Range'], 'bytes=95-') + + def test_get_url(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + self.assertEqual(iterator.get_url(), + 'http://example.com/b/name/o/key?alt=media') + + def test_get_next_chunk_underflow(self): + connection = _Connection() + key = _Key(connection) + iterator = self._makeOne(key) + iterator._bytes_written = iterator._total_bytes = 10 + self.assertRaises(RuntimeError, iterator.get_next_chunk) + + def test_get_next_chunk_200(self): + response = _Response(status=200) + response['content-range'] = '0-9/100' + connection = _Connection((response, 'CHUNK')) + key = _Key(connection) + iterator = self._makeOne(key) + chunk = iterator.get_next_chunk() + self.assertEqual(chunk, 'CHUNK') + self.assertEqual(iterator._bytes_written, len(chunk)) + self.assertEqual(iterator._total_bytes, 100) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['url'], + 'http://example.com/b/name/o/key?alt=media') + self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'}) + + def test_get_next_chunk_206(self): + response = _Response(status=206) + connection = _Connection((response, 'CHUNK')) + key = _Key(connection) + iterator = self._makeOne(key) + iterator._total_bytes = 1000 + chunk = iterator.get_next_chunk() + self.assertEqual(chunk, 'CHUNK') + self.assertEqual(iterator._bytes_written, len(chunk)) + kw, = connection._requested + self.assertEqual(kw['method'], 'GET') + self.assertEqual(kw['url'], + 'http://example.com/b/name/o/key?alt=media') + self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'}) + + def test_get_next_chunk_416(self): + from gcloud.storage.exceptions import StorageError + response = _Response(status=416) + connection = _Connection((response, '')) + key = _Key(connection) + iterator = self._makeOne(key) + iterator._total_bytes = 1000 + self.assertRaises(StorageError, iterator.get_next_chunk) + + class _Connection(object): API_BASE_URL = 'http://example.com' @@ -664,6 +830,14 @@ def generate_signed_url(self, resource, expiration, **kw): '&Expiration=%s' % expiration) +class _Key(object): + CHUNK_SIZE = 10 + path = '/b/name/o/key' + + def __init__(self, connection): + self.connection = connection + + class _Bucket(object): path = '/b/name' name = 'name' @@ -683,3 +857,9 @@ def copy_key(self, key, destination_bucket, new_name): def delete_key(self, key): del self._keys[key.name] self._deleted.append(key.name) + + +class _Response(dict): + @property + def status(self): + return self['status']