From 08e0c9e122fb5e44849066b30c9bdabf2914947b Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 4 Dec 2014 00:59:13 -0500 Subject: [PATCH 1/5] Replace hand-rolled upload/download impl. w/ apitools utilities. Note that in order to use the 'Upload' / 'Download' utilities provided by apitools without its 'base_api' module (not vendored), we have to unwind some of the generated request setup code. Fixes #334. --- gcloud/storage/key.py | 158 ++++--------- gcloud/storage/test_key.py | 441 ++++++++++++++----------------------- 2 files changed, 197 insertions(+), 402 deletions(-) diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 9e2571e67c97..55647c55a60a 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -6,10 +6,14 @@ from StringIO import StringIO import urllib +from _gcloud_vendor.apitools.base.py.http_wrapper import Request +from _gcloud_vendor.apitools.base.py.transfer import Upload +from _gcloud_vendor.apitools.base.py.transfer import Download +from _gcloud_vendor.apitools.base.py.transfer import _RESUMABLE_UPLOAD + from gcloud.storage._helpers import _PropertyMixin from gcloud.storage._helpers import _scalar_property from gcloud.storage.acl import ObjectACL -from gcloud.storage.exceptions import StorageError class Key(_PropertyMixin): @@ -207,8 +211,21 @@ def download_to_file(self, file_obj): :raises: :class:`gcloud.storage.exceptions.NotFound` """ - for chunk in _KeyDataIterator(self): - file_obj.write(chunk) + # Use apitools 'Download' facility. + download = Download.FromStream(file_obj, auto_transfer=False) + download.chunksize = self.CHUNK_SIZE + download_url = self.connection.build_api_url( + path=self.path, query_params={'alt': 'media'}) + headers = {'Range': 'bytes=0-%d' % (self.CHUNK_SIZE - 1)} + request = Request(download_url, 'POST', headers) + + download.InitializeDownload(request, self.connection.http) + + # Should we be passing callbacks through from caller? We can't + # pass them as None, because apitools wants to print to the console + # by default. + download.StreamInChunks(callback=lambda *args: None, + finish_callback=lambda *args: None) # NOTE: Alias for boto-like API. get_contents_to_file = download_to_file @@ -275,10 +292,12 @@ def upload_from_file(self, file_obj, rewind=False, size=None, # Get the basic stats about the file. total_bytes = size or os.fstat(file_obj.fileno()).st_size - bytes_uploaded = 0 - - # Set up a resumable upload session. headers = { + # Base headers + 'Accept': 'application/json', + 'Accept-Encoding': 'gzip, deflate', + 'User-Agent': self.connection.USER_AGENT, + # resumable upload headers. 'X-Upload-Content-Type': content_type or 'application/unknown', 'X-Upload-Content-Length': total_bytes, } @@ -293,30 +312,21 @@ def upload_from_file(self, file_obj, rewind=False, size=None, query_params=query_params, api_base_url=self.connection.API_BASE_URL + '/upload') - response, _ = self.connection.make_request( - method='POST', url=upload_url, - headers=headers) - - # Get the resumable upload URL. - upload_url = response['location'] + # Use apitools 'Upload' facility. + request = Request(upload_url, 'POST', headers) - while bytes_uploaded < total_bytes: - # Construct the range header. - data = file_obj.read(self.CHUNK_SIZE) - chunk_size = len(data) + upload = Upload(file_obj, content_type or 'application/unknown', + total_bytes, auto_transfer=False, + chunksize=self.CHUNK_SIZE) + upload.strategy = _RESUMABLE_UPLOAD - start = bytes_uploaded - end = bytes_uploaded + chunk_size - 1 + upload.InitializeUpload(request, self.connection.http) - headers = { - 'Content-Range': 'bytes %d-%d/%d' % (start, end, total_bytes), - } - - response, _ = self.connection.make_request( - content_type='text/plain', - method='POST', url=upload_url, headers=headers, data=data) - - bytes_uploaded += chunk_size + # Should we be passing callbacks through from caller? We can't + # pass them as None, because apitools wants to print to the console + # by default. + upload.StreamInChunks(callback=lambda *args: None, + finish_callback=lambda *args: None) # NOTE: Alias for boto-like API. set_contents_from_file = upload_from_file @@ -596,97 +606,3 @@ def updated(self): :returns: timestamp in RFC 3339 format. """ return self.properties['updated'] - - -class _KeyDataIterator(object): - """An iterator listing data stored in a key. - - You shouldn't have to use this directly, but instead should use the - helper methods on :class:`gcloud.storage.key.Key` objects. - - :type key: :class:`gcloud.storage.key.Key` - :param key: The key from which to list data.. - """ - - def __init__(self, key): - self.key = key - # NOTE: These variables will be initialized by reset(). - self._bytes_written = None - self._total_bytes = None - self.reset() - - def __iter__(self): - while self.has_more_data(): - yield self.get_next_chunk() - - def reset(self): - """Resets the iterator to the beginning.""" - self._bytes_written = 0 - self._total_bytes = None - - def has_more_data(self): - """Determines whether or not this iterator has more data to read. - - :rtype: bool - :returns: Whether the iterator has more data or not. - """ - if self._bytes_written == 0: - return True - elif not self._total_bytes: - # self._total_bytes **should** be set by this point. - # If it isn't, something is wrong. - raise ValueError('Size of object is unknown.') - else: - return self._bytes_written < self._total_bytes - - def get_headers(self): - """Gets range header(s) for next chunk of data. - - :rtype: dict - :returns: A dictionary of query parameters. - """ - start = self._bytes_written - end = self._bytes_written + self.key.CHUNK_SIZE - 1 - - if self._total_bytes and end > self._total_bytes: - end = '' - - return {'Range': 'bytes=%s-%s' % (start, end)} - - def get_url(self): - """Gets URL to read next chunk of data. - - :rtype: string - :returns: A URL. - """ - return self.key.connection.build_api_url( - path=self.key.path, query_params={'alt': 'media'}) - - def get_next_chunk(self): - """Gets the next chunk of data. - - Uses CHUNK_SIZE to determine how much data to get. - - :rtype: string - :returns: The chunk of data read from the key. - :raises: :class:`RuntimeError` if no more data or - :class:`gcloud.storage.exceptions.StorageError` in the - case of an unexpected response status code. - """ - if not self.has_more_data(): - raise RuntimeError('No more data in this iterator. Try resetting.') - - response, content = self.key.connection.make_request( - method='GET', url=self.get_url(), headers=self.get_headers()) - - if response.status in (200, 206): - self._bytes_written += len(content) - - if 'content-range' in response: - content_range = response['content-range'] - self._total_bytes = int(content_range.rsplit('/', 1)[1]) - - return content - - # Expected a 200 or a 206. Got something else, which is unknown. - raise StorageError(response) diff --git a/gcloud/storage/test_key.py b/gcloud/storage/test_key.py index bc63190d9c93..41010aca4d46 100644 --- a/gcloud/storage/test_key.py +++ b/gcloud/storage/test_key.py @@ -179,58 +179,76 @@ def test_delete(self): self.assertFalse(key.exists()) def test_download_to_file(self): + import httplib from StringIO import StringIO - from gcloud._testing import _Monkey - from gcloud.storage import key as MUT - _CHUNKS = ['abc', 'def'] KEY = 'key' - connection = _Connection() + chunk1_response = {'status': httplib.PARTIAL_CONTENT, + 'content-range': 'bytes 0-2/6'} + chunk2_response = {'status': httplib.OK, + 'content-range': 'bytes 3-5/6'} + connection = _Connection( + (chunk1_response, 'abc'), + (chunk2_response, 'def'), + ) bucket = _Bucket(connection) key = self._makeOne(bucket, KEY) + key.CHUNK_SIZE = 3 fh = StringIO() - with _Monkey(MUT, _KeyDataIterator=lambda self: iter(_CHUNKS)): - key.download_to_file(fh) - self.assertEqual(fh.getvalue(), ''.join(_CHUNKS)) + key.download_to_file(fh) + self.assertEqual(fh.getvalue(), 'abcdef') def test_download_to_filename(self): + import httplib from tempfile import NamedTemporaryFile - from gcloud._testing import _Monkey - from gcloud.storage import key as MUT - _CHUNKS = ['abc', 'def'] KEY = 'key' - connection = _Connection() + chunk1_response = {'status': httplib.PARTIAL_CONTENT, + 'content-range': 'bytes 0-2/6'} + chunk2_response = {'status': httplib.OK, + 'content-range': 'bytes 3-5/6'} + connection = _Connection( + (chunk1_response, 'abc'), + (chunk2_response, 'def'), + ) bucket = _Bucket(connection) key = self._makeOne(bucket, KEY) - with _Monkey(MUT, _KeyDataIterator=lambda self: iter(_CHUNKS)): - with NamedTemporaryFile() as f: - key.download_to_filename(f.name) - f.flush() - with open(f.name) as g: - wrote = g.read() - self.assertEqual(wrote, ''.join(_CHUNKS)) + key.CHUNK_SIZE = 3 + with NamedTemporaryFile() as f: + key.download_to_filename(f.name) + f.flush() + with open(f.name) as g: + wrote = g.read() + self.assertEqual(wrote, 'abcdef') def test_download_as_string(self): - from gcloud._testing import _Monkey - from gcloud.storage import key as MUT - _CHUNKS = ['abc', 'def'] + import httplib KEY = 'key' - connection = _Connection() + chunk1_response = {'status': httplib.PARTIAL_CONTENT, + 'content-range': 'bytes 0-2/6'} + chunk2_response = {'status': httplib.OK, + 'content-range': 'bytes 3-5/6'} + connection = _Connection( + (chunk1_response, 'abc'), + (chunk2_response, 'def'), + ) bucket = _Bucket(connection) key = self._makeOne(bucket, KEY) - with _Monkey(MUT, _KeyDataIterator=lambda self: iter(_CHUNKS)): - fetched = key.download_as_string() - self.assertEqual(fetched, ''.join(_CHUNKS)) + key.CHUNK_SIZE = 3 + fetched = key.download_as_string() + self.assertEqual(fetched, 'abcdef') def test_upload_from_file(self): + import httplib from tempfile import NamedTemporaryFile from urlparse import parse_qsl from urlparse import urlsplit + from _gcloud_vendor.apitools.base.py import http_wrapper KEY = 'key' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = 'ABCDEF' - loc_response = {'location': UPLOAD_URL} - chunk1_response = {} - chunk2_response = {} + loc_response = {'status': httplib.OK, 'location': UPLOAD_URL} + chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE, + 'range': 'bytes 0-4'} + chunk2_response = {'status': httplib.OK} connection = _Connection( (loc_response, ''), (chunk1_response, ''), @@ -243,40 +261,49 @@ def test_upload_from_file(self): fh.write(DATA) fh.flush() key.upload_from_file(fh, rewind=True) - rq = connection._requested + rq = connection.http._requested self.assertEqual(len(rq), 3) self.assertEqual(rq[0]['method'], 'POST') - uri = rq[0]['url'] + uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') self.assertEqual(path, '/b/name/o') self.assertEqual(dict(parse_qsl(qs)), {'uploadType': 'resumable', 'name': 'key'}) - self.assertEqual(rq[0]['headers'], - {'X-Upload-Content-Length': 6, - 'X-Upload-Content-Type': 'application/unknown'}) - self.assertEqual(rq[1]['method'], 'POST') - self.assertEqual(rq[1]['url'], UPLOAD_URL) - self.assertEqual(rq[1]['content_type'], 'text/plain') - self.assertEqual(rq[1]['data'], DATA[:5]) - self.assertEqual(rq[1]['headers'], {'Content-Range': 'bytes 0-4/6'}) - self.assertEqual(rq[2]['method'], 'POST') - self.assertEqual(rq[2]['url'], UPLOAD_URL) - self.assertEqual(rq[2]['content_type'], 'text/plain') - self.assertEqual(rq[2]['data'], DATA[5:]) - self.assertEqual(rq[2]['headers'], {'Content-Range': 'bytes 5-5/6'}) + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + self.assertEqual(headers['X-Upload-Content-Length'], '6') + self.assertEqual(headers['X-Upload-Content-Type'], + 'application/unknown') + self.assertEqual(rq[1]['method'], 'PUT') + self.assertEqual(rq[1]['uri'], UPLOAD_URL) + headers = dict( + [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) + self.assertEqual(rq[1]['body'], DATA[:5]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 0-4/6') + self.assertEqual(rq[2]['method'], 'PUT') + self.assertEqual(rq[2]['uri'], UPLOAD_URL) + self.assertEqual(rq[2]['body'], DATA[5:]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[2]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 5-5/6') def test_upload_from_file_w_slash_in_name(self): + import httplib from tempfile import NamedTemporaryFile from urlparse import parse_qsl from urlparse import urlsplit + from _gcloud_vendor.apitools.base.py import http_wrapper KEY = 'parent/child' UPLOAD_URL = 'http://example.com/upload/name/parent%2Fchild' DATA = 'ABCDEF' - loc_response = {'location': UPLOAD_URL} - chunk1_response = {} - chunk2_response = {} + loc_response = {'status': httplib.OK, 'location': UPLOAD_URL} + chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE, + 'range': 'bytes 0-4'} + chunk2_response = {'status': httplib.OK} connection = _Connection( (loc_response, ''), (chunk1_response, ''), @@ -289,40 +316,47 @@ def test_upload_from_file_w_slash_in_name(self): fh.write(DATA) fh.flush() key.upload_from_file(fh, rewind=True) - rq = connection._requested + rq = connection.http._requested self.assertEqual(len(rq), 3) self.assertEqual(rq[0]['method'], 'POST') - uri = rq[0]['url'] + uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') self.assertEqual(path, '/b/name/o') self.assertEqual(dict(parse_qsl(qs)), {'uploadType': 'resumable', 'name': 'parent/child'}) - self.assertEqual(rq[0]['headers'], - {'X-Upload-Content-Length': 6, - 'X-Upload-Content-Type': 'application/unknown'}) - self.assertEqual(rq[1]['method'], 'POST') - self.assertEqual(rq[1]['url'], UPLOAD_URL) - self.assertEqual(rq[1]['content_type'], 'text/plain') - self.assertEqual(rq[1]['data'], DATA[:5]) - self.assertEqual(rq[1]['headers'], {'Content-Range': 'bytes 0-4/6'}) - self.assertEqual(rq[2]['method'], 'POST') - self.assertEqual(rq[2]['url'], UPLOAD_URL) - self.assertEqual(rq[2]['content_type'], 'text/plain') - self.assertEqual(rq[2]['data'], DATA[5:]) - self.assertEqual(rq[2]['headers'], {'Content-Range': 'bytes 5-5/6'}) + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + self.assertEqual(headers['X-Upload-Content-Length'], '6') + self.assertEqual(headers['X-Upload-Content-Type'], + 'application/unknown') + self.assertEqual(rq[1]['method'], 'PUT') + self.assertEqual(rq[1]['uri'], UPLOAD_URL) + self.assertEqual(rq[1]['body'], DATA[:5]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 0-4/6') + self.assertEqual(rq[2]['method'], 'PUT') + self.assertEqual(rq[2]['uri'], UPLOAD_URL) + self.assertEqual(rq[2]['body'], DATA[5:]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[2]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 5-5/6') def test_upload_from_filename(self): + import httplib from tempfile import NamedTemporaryFile from urlparse import parse_qsl from urlparse import urlsplit + from _gcloud_vendor.apitools.base.py import http_wrapper KEY = 'key' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = 'ABCDEF' - loc_response = {'location': UPLOAD_URL} - chunk1_response = {} - chunk2_response = {} + loc_response = {'status': httplib.OK, 'location': UPLOAD_URL} + chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE, + 'range': 'bytes 0-4'} + chunk2_response = {'status': httplib.OK} connection = _Connection( (loc_response, ''), (chunk1_response, ''), @@ -335,39 +369,46 @@ def test_upload_from_filename(self): fh.write(DATA) fh.flush() key.upload_from_filename(fh.name) - rq = connection._requested + rq = connection.http._requested self.assertEqual(len(rq), 3) self.assertEqual(rq[0]['method'], 'POST') - uri = rq[0]['url'] + uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') self.assertEqual(path, '/b/name/o') self.assertEqual(dict(parse_qsl(qs)), {'uploadType': 'resumable', 'name': 'key'}) - self.assertEqual(rq[0]['headers'], - {'X-Upload-Content-Length': 6, - 'X-Upload-Content-Type': 'image/jpeg'}) - self.assertEqual(rq[1]['method'], 'POST') - self.assertEqual(rq[1]['url'], UPLOAD_URL) - self.assertEqual(rq[1]['content_type'], 'text/plain') - self.assertEqual(rq[1]['data'], DATA[:5]) - self.assertEqual(rq[1]['headers'], {'Content-Range': 'bytes 0-4/6'}) - self.assertEqual(rq[2]['method'], 'POST') - self.assertEqual(rq[2]['url'], UPLOAD_URL) - self.assertEqual(rq[2]['content_type'], 'text/plain') - self.assertEqual(rq[2]['data'], DATA[5:]) - self.assertEqual(rq[2]['headers'], {'Content-Range': 'bytes 5-5/6'}) + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + self.assertEqual(headers['X-Upload-Content-Length'], '6') + self.assertEqual(headers['X-Upload-Content-Type'], + 'image/jpeg') + self.assertEqual(rq[1]['method'], 'PUT') + self.assertEqual(rq[1]['uri'], UPLOAD_URL) + self.assertEqual(rq[1]['body'], DATA[:5]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 0-4/6') + self.assertEqual(rq[2]['method'], 'PUT') + self.assertEqual(rq[2]['uri'], UPLOAD_URL) + self.assertEqual(rq[2]['body'], DATA[5:]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[2]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 5-5/6') def test_upload_from_string(self): + import httplib from urlparse import parse_qsl from urlparse import urlsplit + from _gcloud_vendor.apitools.base.py import http_wrapper KEY = 'key' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = 'ABCDEF' - loc_response = {'location': UPLOAD_URL} - chunk1_response = {} - chunk2_response = {} + loc_response = {'status': httplib.OK, 'location': UPLOAD_URL} + chunk1_response = {'status': http_wrapper.RESUME_INCOMPLETE, + 'range': 'bytes 0-4'} + chunk2_response = {'status': httplib.OK} connection = _Connection( (loc_response, ''), (chunk1_response, ''), @@ -377,29 +418,33 @@ def test_upload_from_string(self): key = self._makeOne(bucket, KEY) key.CHUNK_SIZE = 5 key.upload_from_string(DATA) - rq = connection._requested + rq = connection.http._requested self.assertEqual(len(rq), 3) self.assertEqual(rq[0]['method'], 'POST') - uri = rq[0]['url'] + uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') self.assertEqual(path, '/b/name/o') self.assertEqual(dict(parse_qsl(qs)), {'uploadType': 'resumable', 'name': 'key'}) - self.assertEqual(rq[0]['headers'], - {'X-Upload-Content-Length': 6, - 'X-Upload-Content-Type': 'text/plain'}) - self.assertEqual(rq[1]['method'], 'POST') - self.assertEqual(rq[1]['url'], UPLOAD_URL) - self.assertEqual(rq[1]['content_type'], 'text/plain') - self.assertEqual(rq[1]['data'], DATA[:5]) - self.assertEqual(rq[1]['headers'], {'Content-Range': 'bytes 0-4/6'}) - self.assertEqual(rq[2]['method'], 'POST') - self.assertEqual(rq[2]['url'], UPLOAD_URL) - self.assertEqual(rq[2]['content_type'], 'text/plain') - self.assertEqual(rq[2]['data'], DATA[5:]) - self.assertEqual(rq[2]['headers'], {'Content-Range': 'bytes 5-5/6'}) + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + self.assertEqual(headers['X-Upload-Content-Length'], '6') + self.assertEqual(headers['X-Upload-Content-Type'], + 'text/plain') + self.assertEqual(rq[1]['method'], 'PUT') + self.assertEqual(rq[1]['uri'], UPLOAD_URL) + self.assertEqual(rq[1]['body'], DATA[:5]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 0-4/6') + self.assertEqual(rq[2]['method'], 'PUT') + self.assertEqual(rq[2]['uri'], UPLOAD_URL) + self.assertEqual(rq[2]['body'], DATA[5:]) + headers = dict( + [(x.title(), str(y)) for x, y in rq[2]['headers'].items()]) + self.assertEqual(headers['Content-Range'], 'bytes 5-5/6') def test_make_public(self): from gcloud.storage.acl import _ACLEntity @@ -737,189 +782,30 @@ def test_updated(self): self.assertEqual(key.updated, UPDATED) -class Test__KeyDataIterator(unittest2.TestCase): +class _Responder(object): - def _getTargetClass(self): - from gcloud.storage.key import _KeyDataIterator - return _KeyDataIterator + def __init__(self, *responses): + self._responses = responses[:] + self._requested = [] - def _makeOne(self, *args, **kw): - return self._getTargetClass()(*args, **kw) + def _respond(self, **kw): + self._requested.append(kw) + response, self._responses = self._responses[0], self._responses[1:] + return response - def test_ctor(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - self.assertTrue(iterator.key is key) - self.assertEqual(iterator._bytes_written, 0) - self.assertEqual(iterator._total_bytes, None) - def test__iter__(self): - response1 = _Response(status=200) - response1['content-range'] = '0-9/15' - response2 = _Response(status=200) - response2['content-range'] = '10-14/15' - connection = _Connection( - (response1, '0123456789'), - (response2, '01234'), - ) - key = _Key(connection) - iterator = self._makeOne(key) - chunks = list(iterator) - self.assertEqual(len(chunks), 2) - self.assertEqual(chunks[0], '0123456789') - self.assertEqual(chunks[1], '01234') - self.assertEqual(iterator._bytes_written, 15) - self.assertEqual(iterator._total_bytes, 15) - kws = connection._requested - self.assertEqual(kws[0]['method'], 'GET') - self.assertEqual(kws[0]['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kws[0]['headers'], {'Range': 'bytes=0-9'}) - self.assertEqual(kws[1]['method'], 'GET') - self.assertEqual(kws[1]['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kws[1]['headers'], {'Range': 'bytes=10-'}) - - def test_reset(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 - iterator._total_bytes = 1000 - iterator.reset() - self.assertEqual(iterator._bytes_written, 0) - self.assertEqual(iterator._total_bytes, None) - - def test_has_more_data_new(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - self.assertTrue(iterator.has_more_data()) - - def test_has_more_data_invalid(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 # no _total_bytes. - self.assertRaises(ValueError, iterator.has_more_data) - - def test_has_more_data_true(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 - iterator._total_bytes = 1000 - self.assertTrue(iterator.has_more_data()) - - def test_has_more_data_false(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 1000 - iterator._total_bytes = 1000 - self.assertFalse(iterator.has_more_data()) - - def test_get_headers_new(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - headers = iterator.get_headers() - self.assertEqual(len(headers), 1) - self.assertEqual(headers['Range'], 'bytes=0-9') - - def test_get_headers_ok(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 10 - iterator._total_bytes = 1000 - headers = iterator.get_headers() - self.assertEqual(len(headers), 1) - self.assertEqual(headers['Range'], 'bytes=10-19') - - def test_get_headers_off_end(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = 95 - iterator._total_bytes = 100 - headers = iterator.get_headers() - self.assertEqual(len(headers), 1) - self.assertEqual(headers['Range'], 'bytes=95-') - - def test_get_url(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - self.assertEqual(iterator.get_url(), - 'http://example.com/b/name/o/key?alt=media') - - def test_get_next_chunk_underflow(self): - connection = _Connection() - key = _Key(connection) - iterator = self._makeOne(key) - iterator._bytes_written = iterator._total_bytes = 10 - self.assertRaises(RuntimeError, iterator.get_next_chunk) - - def test_get_next_chunk_200(self): - response = _Response(status=200) - response['content-range'] = '0-9/100' - connection = _Connection((response, 'CHUNK')) - key = _Key(connection) - iterator = self._makeOne(key) - chunk = iterator.get_next_chunk() - self.assertEqual(chunk, 'CHUNK') - self.assertEqual(iterator._bytes_written, len(chunk)) - self.assertEqual(iterator._total_bytes, 100) - kw, = connection._requested - self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'}) - - def test_get_next_chunk_206(self): - response = _Response(status=206) - connection = _Connection((response, 'CHUNK')) - key = _Key(connection) - iterator = self._makeOne(key) - iterator._total_bytes = 1000 - chunk = iterator.get_next_chunk() - self.assertEqual(chunk, 'CHUNK') - self.assertEqual(iterator._bytes_written, len(chunk)) - kw, = connection._requested - self.assertEqual(kw['method'], 'GET') - self.assertEqual(kw['url'], - 'http://example.com/b/name/o/key?alt=media') - self.assertEqual(kw['headers'], {'Range': 'bytes=0-9'}) - - def test_get_next_chunk_416(self): - from gcloud.storage.exceptions import StorageError - response = _Response(status=416) - connection = _Connection((response, '')) - key = _Key(connection) - iterator = self._makeOne(key) - iterator._total_bytes = 1000 - self.assertRaises(StorageError, iterator.get_next_chunk) - - -class _Connection(object): +class _Connection(_Responder): + API_BASE_URL = 'http://example.com' + USER_AGENT = 'testing 1.2.3' def __init__(self, *responses): - self._responses = responses - self._requested = [] + super(_Connection, self).__init__(*responses) self._signed = [] - - def make_request(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response + self.http = _HTTP(*responses) def api_request(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response + return self._respond(**kw) def build_api_url(self, path, query_params=None, api_base_url=API_BASE_URL): @@ -936,12 +822,11 @@ def generate_signed_url(self, resource, expiration, **kw): '&Expiration=%s' % expiration) -class _Key(object): - CHUNK_SIZE = 10 - path = '/b/name/o/key' +class _HTTP(_Responder): - def __init__(self, connection): - self.connection = connection + def request(self, uri, method, headers, body, **kw): + return self._respond(uri=uri, method=method, headers=headers, + body=body, **kw) class _Bucket(object): @@ -963,9 +848,3 @@ def copy_key(self, key, destination_bucket, new_name): def delete_key(self, key): del self._keys[key.name] self._deleted.append(key.name) - - -class _Response(dict): - @property - def status(self): - return self['status'] From 2a306feb54e33ad221c8fd8c3be96efcfd94a103 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 5 Dec 2014 12:36:46 -0500 Subject: [PATCH 2/5] Use 'Upload.ConfigureRequest' to decide 'simple' vs. 'resumable'. --- gcloud/storage/key.py | 89 +++++++++++++++++++--------- gcloud/storage/test_key.py | 118 ++++++++++++++++++------------------- 2 files changed, 118 insertions(+), 89 deletions(-) diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 55647c55a60a..096c9dd5dcec 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -6,10 +6,8 @@ from StringIO import StringIO import urllib -from _gcloud_vendor.apitools.base.py.http_wrapper import Request -from _gcloud_vendor.apitools.base.py.transfer import Upload -from _gcloud_vendor.apitools.base.py.transfer import Download -from _gcloud_vendor.apitools.base.py.transfer import _RESUMABLE_UPLOAD +from _gcloud_vendor.apitools.base.py import http_wrapper +from _gcloud_vendor.apitools.base.py import transfer from gcloud.storage._helpers import _PropertyMixin from gcloud.storage._helpers import _scalar_property @@ -212,12 +210,12 @@ def download_to_file(self, file_obj): :raises: :class:`gcloud.storage.exceptions.NotFound` """ # Use apitools 'Download' facility. - download = Download.FromStream(file_obj, auto_transfer=False) + download = transfer.Download.FromStream(file_obj, auto_transfer=False) download.chunksize = self.CHUNK_SIZE download_url = self.connection.build_api_url( path=self.path, query_params={'alt': 'media'}) headers = {'Range': 'bytes=0-%d' % (self.CHUNK_SIZE - 1)} - request = Request(download_url, 'POST', headers) + request = http_wrapper.Request(download_url, 'POST', headers) download.InitializeDownload(request, self.connection.http) @@ -259,7 +257,7 @@ def download_as_string(self): get_contents_as_string = download_as_string def upload_from_file(self, file_obj, rewind=False, size=None, - content_type=None): + content_type=None, num_retries=6): """Upload the contents of this key from a file-like object. .. note:: @@ -292,41 +290,43 @@ def upload_from_file(self, file_obj, rewind=False, size=None, # Get the basic stats about the file. total_bytes = size or os.fstat(file_obj.fileno()).st_size + conn = self.connection headers = { - # Base headers 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate', - 'User-Agent': self.connection.USER_AGENT, - # resumable upload headers. - 'X-Upload-Content-Type': content_type or 'application/unknown', - 'X-Upload-Content-Length': total_bytes, + 'User-Agent': conn.USER_AGENT, } - query_params = { - 'uploadType': 'resumable', - 'name': self.name, - } + upload = transfer.Upload(file_obj, + content_type or 'application/unknown', + total_bytes, auto_transfer=False, + chunksize=self.CHUNK_SIZE) - upload_url = self.connection.build_api_url( - path=self.bucket.path + '/o', - query_params=query_params, - api_base_url=self.connection.API_BASE_URL + '/upload') + url_builder = _UrlBuilder(bucket_name=self.bucket.name, + object_name=self.name) + upload_config = _UploadConfig() - # Use apitools 'Upload' facility. - request = Request(upload_url, 'POST', headers) + # Temporary URL, until we know simple vs. resumable. + upload_url = conn.build_api_url( + path=self.bucket.path + '/o') - upload = Upload(file_obj, content_type or 'application/unknown', - total_bytes, auto_transfer=False, - chunksize=self.CHUNK_SIZE) - upload.strategy = _RESUMABLE_UPLOAD + # Use apitools 'Upload' facility. + request = http_wrapper.Request(upload_url, 'POST', headers) - upload.InitializeUpload(request, self.connection.http) + upload.ConfigureRequest(upload_config, request, url_builder) + path = url_builder.relative_path.format(bucket=self.bucket.name) + query_params = url_builder.query_params + request.url = conn.build_api_url(path=path, query_params=query_params) + upload.InitializeUpload(request, conn.http) # Should we be passing callbacks through from caller? We can't # pass them as None, because apitools wants to print to the console # by default. - upload.StreamInChunks(callback=lambda *args: None, - finish_callback=lambda *args: None) + if upload.strategy == transfer._RESUMABLE_UPLOAD: + upload.StreamInChunks(callback=lambda *args: None, + finish_callback=lambda *args: None) + else: + http_wrapper.MakeRequest(conn.http, request, retries=num_retries) # NOTE: Alias for boto-like API. set_contents_from_file = upload_from_file @@ -606,3 +606,34 @@ def updated(self): :returns: timestamp in RFC 3339 format. """ return self.properties['updated'] + + +class _UploadConfig(object): + """ Faux message FBO apitools' 'ConfigureRequest'. + + Values extracted from apitools + 'samples/storage_sample/storage/storage_v1_client.py' + """ + accept = ['*/*'] + max_size = None + resumable_multipart = True + resumable_path = u'/resumable/upload/storage/v1/b/{bucket}/o' + simple_multipart = True + simple_path = u'/upload/storage/v1/b/{bucket}/o' + + +class _UrlBuilder(object): + """Faux builder FBO apitools' 'ConfigureRequest' + """ + def __init__(self, bucket_name, object_name): + self.query_params = {'name': object_name} + self._bucket_name = bucket_name + self._relative_path = '' + + @property + def relative_path(self): + return self._relative_path.format(bucket=self._bucket_name) + + @relative_path.setter + def relative_path(self, value): + self._relative_path = value diff --git a/gcloud/storage/test_key.py b/gcloud/storage/test_key.py index 41010aca4d46..a0d95b706a1e 100644 --- a/gcloud/storage/test_key.py +++ b/gcloud/storage/test_key.py @@ -236,12 +236,47 @@ def test_download_as_string(self): fetched = key.download_as_string() self.assertEqual(fetched, 'abcdef') - def test_upload_from_file(self): + def test_upload_from_file_simple(self): import httplib from tempfile import NamedTemporaryFile from urlparse import parse_qsl from urlparse import urlsplit + KEY = 'key' + DATA = 'ABCDEF' + response = {'status': httplib.OK} + connection = _Connection( + (response, ''), + ) + bucket = _Bucket(connection) + key = self._makeOne(bucket, KEY) + key.CHUNK_SIZE = 5 + with NamedTemporaryFile() as fh: + fh.write(DATA) + fh.flush() + key.upload_from_file(fh, rewind=True) + rq = connection.http._requested + self.assertEqual(len(rq), 1) + self.assertEqual(rq[0]['method'], 'POST') + uri = rq[0]['uri'] + scheme, netloc, path, qs, _ = urlsplit(uri) + self.assertEqual(scheme, 'http') + self.assertEqual(netloc, 'example.com') + self.assertEqual(path, '/upload/storage/v1/b/name/o') + self.assertEqual(dict(parse_qsl(qs)), + {'uploadType': 'media', 'name': 'key'}) + headers = dict( + [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) + self.assertEqual(headers['Content-Length'], '6') + self.assertEqual(headers['Content-Type'], 'application/unknown') + + def test_upload_from_file_resumable(self): + import httplib + from tempfile import NamedTemporaryFile + from urlparse import parse_qsl + from urlparse import urlsplit + from gcloud._testing import _Monkey from _gcloud_vendor.apitools.base.py import http_wrapper + from _gcloud_vendor.apitools.base.py import transfer KEY = 'key' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = 'ABCDEF' @@ -257,10 +292,12 @@ def test_upload_from_file(self): bucket = _Bucket(connection) key = self._makeOne(bucket, KEY) key.CHUNK_SIZE = 5 - with NamedTemporaryFile() as fh: - fh.write(DATA) - fh.flush() - key.upload_from_file(fh, rewind=True) + # Set the threshhold low enough that we force a resumable uploada. + with _Monkey(transfer, _RESUMABLE_UPLOAD_THRESHOLD=5): + with NamedTemporaryFile() as fh: + fh.write(DATA) + fh.flush() + key.upload_from_file(fh, rewind=True) rq = connection.http._requested self.assertEqual(len(rq), 3) self.assertEqual(rq[0]['method'], 'POST') @@ -268,7 +305,7 @@ def test_upload_from_file(self): scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') - self.assertEqual(path, '/b/name/o') + self.assertEqual(path, '/resumable/upload/storage/v1/b/name/o') self.assertEqual(dict(parse_qsl(qs)), {'uploadType': 'resumable', 'name': 'key'}) headers = dict( @@ -317,32 +354,19 @@ def test_upload_from_file_w_slash_in_name(self): fh.flush() key.upload_from_file(fh, rewind=True) rq = connection.http._requested - self.assertEqual(len(rq), 3) + self.assertEqual(len(rq), 1) self.assertEqual(rq[0]['method'], 'POST') uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') - self.assertEqual(path, '/b/name/o') + self.assertEqual(path, '/upload/storage/v1/b/name/o') self.assertEqual(dict(parse_qsl(qs)), - {'uploadType': 'resumable', 'name': 'parent/child'}) + {'uploadType': 'media', 'name': 'parent/child'}) headers = dict( [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) - self.assertEqual(headers['X-Upload-Content-Length'], '6') - self.assertEqual(headers['X-Upload-Content-Type'], - 'application/unknown') - self.assertEqual(rq[1]['method'], 'PUT') - self.assertEqual(rq[1]['uri'], UPLOAD_URL) - self.assertEqual(rq[1]['body'], DATA[:5]) - headers = dict( - [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) - self.assertEqual(headers['Content-Range'], 'bytes 0-4/6') - self.assertEqual(rq[2]['method'], 'PUT') - self.assertEqual(rq[2]['uri'], UPLOAD_URL) - self.assertEqual(rq[2]['body'], DATA[5:]) - headers = dict( - [(x.title(), str(y)) for x, y in rq[2]['headers'].items()]) - self.assertEqual(headers['Content-Range'], 'bytes 5-5/6') + self.assertEqual(headers['Content-Length'], '6') + self.assertEqual(headers['Content-Type'], 'application/unknown') def test_upload_from_filename(self): import httplib @@ -370,32 +394,19 @@ def test_upload_from_filename(self): fh.flush() key.upload_from_filename(fh.name) rq = connection.http._requested - self.assertEqual(len(rq), 3) + self.assertEqual(len(rq), 1) self.assertEqual(rq[0]['method'], 'POST') uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') - self.assertEqual(path, '/b/name/o') + self.assertEqual(path, '/upload/storage/v1/b/name/o') self.assertEqual(dict(parse_qsl(qs)), - {'uploadType': 'resumable', 'name': 'key'}) + {'uploadType': 'media', 'name': 'key'}) headers = dict( [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) - self.assertEqual(headers['X-Upload-Content-Length'], '6') - self.assertEqual(headers['X-Upload-Content-Type'], - 'image/jpeg') - self.assertEqual(rq[1]['method'], 'PUT') - self.assertEqual(rq[1]['uri'], UPLOAD_URL) - self.assertEqual(rq[1]['body'], DATA[:5]) - headers = dict( - [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) - self.assertEqual(headers['Content-Range'], 'bytes 0-4/6') - self.assertEqual(rq[2]['method'], 'PUT') - self.assertEqual(rq[2]['uri'], UPLOAD_URL) - self.assertEqual(rq[2]['body'], DATA[5:]) - headers = dict( - [(x.title(), str(y)) for x, y in rq[2]['headers'].items()]) - self.assertEqual(headers['Content-Range'], 'bytes 5-5/6') + self.assertEqual(headers['Content-Length'], '6') + self.assertEqual(headers['Content-Type'], 'image/jpeg') def test_upload_from_string(self): import httplib @@ -419,32 +430,19 @@ def test_upload_from_string(self): key.CHUNK_SIZE = 5 key.upload_from_string(DATA) rq = connection.http._requested - self.assertEqual(len(rq), 3) + self.assertEqual(len(rq), 1) self.assertEqual(rq[0]['method'], 'POST') uri = rq[0]['uri'] scheme, netloc, path, qs, _ = urlsplit(uri) self.assertEqual(scheme, 'http') self.assertEqual(netloc, 'example.com') - self.assertEqual(path, '/b/name/o') + self.assertEqual(path, '/upload/storage/v1/b/name/o') self.assertEqual(dict(parse_qsl(qs)), - {'uploadType': 'resumable', 'name': 'key'}) + {'uploadType': 'media', 'name': 'key'}) headers = dict( [(x.title(), str(y)) for x, y in rq[0]['headers'].items()]) - self.assertEqual(headers['X-Upload-Content-Length'], '6') - self.assertEqual(headers['X-Upload-Content-Type'], - 'text/plain') - self.assertEqual(rq[1]['method'], 'PUT') - self.assertEqual(rq[1]['uri'], UPLOAD_URL) - self.assertEqual(rq[1]['body'], DATA[:5]) - headers = dict( - [(x.title(), str(y)) for x, y in rq[1]['headers'].items()]) - self.assertEqual(headers['Content-Range'], 'bytes 0-4/6') - self.assertEqual(rq[2]['method'], 'PUT') - self.assertEqual(rq[2]['uri'], UPLOAD_URL) - self.assertEqual(rq[2]['body'], DATA[5:]) - headers = dict( - [(x.title(), str(y)) for x, y in rq[2]['headers'].items()]) - self.assertEqual(headers['Content-Range'], 'bytes 5-5/6') + self.assertEqual(headers['Content-Length'], '6') + self.assertEqual(headers['Content-Type'], 'text/plain') def test_make_public(self): from gcloud.storage.acl import _ACLEntity From 71780e3754511ee4bb9c60bbb1bfd818a7f7ff39 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 5 Dec 2014 12:48:27 -0500 Subject: [PATCH 3/5] Appease lint. --- gcloud/storage/key.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gcloud/storage/key.py b/gcloud/storage/key.py index 096c9dd5dcec..dbdae912ff57 100644 --- a/gcloud/storage/key.py +++ b/gcloud/storage/key.py @@ -623,8 +623,7 @@ class _UploadConfig(object): class _UrlBuilder(object): - """Faux builder FBO apitools' 'ConfigureRequest' - """ + """Faux builder FBO apitools' 'ConfigureRequest'""" def __init__(self, bucket_name, object_name): self.query_params = {'name': object_name} self._bucket_name = bucket_name @@ -632,8 +631,13 @@ def __init__(self, bucket_name, object_name): @property def relative_path(self): + """Inject bucket name into path.""" return self._relative_path.format(bucket=self._bucket_name) @relative_path.setter def relative_path(self, value): + """Allow update of path template. + + ``value`` should be a string template taking ``{bucket}``. + """ self._relative_path = value From 0416e93fce65c1c2e99fcac71bff79e06d8d4284 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 8 Dec 2014 14:27:04 -0500 Subject: [PATCH 4/5] Garden in 'python-modernize' updates to apitools. Catches up our vendored-in source to commit: 5a1bab1df7a474cff57f7f5cc066b19a1c286a21. --- .../apitools/base/py/http_wrapper.py | 19 ++++----- _gcloud_vendor/apitools/base/py/transfer.py | 40 ++++++++++--------- _gcloud_vendor/apitools/base/py/util.py | 11 ++--- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/_gcloud_vendor/apitools/base/py/http_wrapper.py b/_gcloud_vendor/apitools/base/py/http_wrapper.py index 80454f495752..aa47fbc65602 100644 --- a/_gcloud_vendor/apitools/base/py/http_wrapper.py +++ b/_gcloud_vendor/apitools/base/py/http_wrapper.py @@ -6,13 +6,14 @@ """ import collections -import httplib import logging import socket import time import urlparse import httplib2 +from six.moves import http_client +from six.moves import range from _gcloud_vendor.apitools.base.py import exceptions from _gcloud_vendor.apitools.base.py import util @@ -28,10 +29,10 @@ RESUME_INCOMPLETE = 308 TOO_MANY_REQUESTS = 429 _REDIRECT_STATUS_CODES = ( - httplib.MOVED_PERMANENTLY, - httplib.FOUND, - httplib.SEE_OTHER, - httplib.TEMPORARY_REDIRECT, + http_client.MOVED_PERMANENTLY, + http_client.FOUND, + http_client.SEE_OTHER, + http_client.TEMPORARY_REDIRECT, RESUME_INCOMPLETE, ) @@ -129,7 +130,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): url_scheme = urlparse.urlsplit(http_request.url).scheme if url_scheme and url_scheme in http.connections: connection_type = http.connections[url_scheme] - for retry in xrange(retries + 1): + for retry in range(retries + 1): # Note that the str() calls here are important for working around # some funny business with message construction and unicode in # httplib itself. See, eg, @@ -140,7 +141,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): str(http_request.url), method=str(http_request.http_method), body=http_request.body, headers=http_request.headers, redirections=redirections, connection_type=connection_type) - except httplib.BadStatusLine as e: + except http_client.BadStatusLine as e: logging.error('Caught BadStatusLine from httplib, retrying: %s', e) exc = e except socket.error as e: @@ -148,7 +149,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): raise logging.error('Caught socket error, retrying: %s', e) exc = e - except httplib.IncompleteRead as e: + except http_client.IncompleteRead as e: if http_request.http_method != 'GET': raise logging.error('Caught IncompleteRead error, retrying: %s', e) @@ -161,7 +162,7 @@ def MakeRequest(http, http_request, retries=5, redirections=5): break logging.info('Retrying request to url <%s> after status code %s.', response.request_url, response.status_code) - elif isinstance(exc, httplib.IncompleteRead): + elif isinstance(exc, http_client.IncompleteRead): logging.info('Retrying request to url <%s> after incomplete read.', str(http_request.url)) else: diff --git a/_gcloud_vendor/apitools/base/py/transfer.py b/_gcloud_vendor/apitools/base/py/transfer.py index 46dbc7f22c82..1b55651ab1aa 100644 --- a/_gcloud_vendor/apitools/base/py/transfer.py +++ b/_gcloud_vendor/apitools/base/py/transfer.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """Upload and download support for apitools.""" +from __future__ import print_function import email.generator as email_generator import email.mime.multipart as mime_multipart import email.mime.nonmultipart as mime_nonmultipart -import httplib import io import json import mimetypes @@ -12,6 +12,8 @@ import StringIO import threading +from six.moves import http_client + from _gcloud_vendor.apitools.base.py import exceptions from _gcloud_vendor.apitools.base.py import http_wrapper from _gcloud_vendor.apitools.base.py import util @@ -38,7 +40,7 @@ def __init__(self, stream, close_stream=False, chunksize=None, self.__url = None self.auto_transfer = auto_transfer - self.chunksize = chunksize or 1048576L + self.chunksize = chunksize or 1048576 def __repr__(self): return str(self) @@ -121,10 +123,10 @@ class Download(_Transfer): chunksize: default chunksize to use for transfers. """ _ACCEPTABLE_STATUSES = set(( - httplib.OK, - httplib.NO_CONTENT, - httplib.PARTIAL_CONTENT, - httplib.REQUESTED_RANGE_NOT_SATISFIABLE, + http_client.OK, + http_client.NO_CONTENT, + http_client.PARTIAL_CONTENT, + http_client.REQUESTED_RANGE_NOT_SATISFIABLE, )) _REQUIRED_SERIALIZATION_KEYS = set(( 'auto_transfer', 'progress', 'total_size', 'url')) @@ -242,13 +244,13 @@ def InitializeDownload(self, http_request, http=None, client=None): @staticmethod def _ArgPrinter(response, unused_download): if 'content-range' in response.info: - print 'Received %s' % response.info['content-range'] + print('Received %s' % response.info['content-range']) else: - print 'Received %d bytes' % len(response) + print('Received %d bytes' % len(response)) @staticmethod def _CompletePrinter(*unused_args): - print 'Download complete' + print('Download complete') def __NormalizeStartEnd(self, start, end=None): if end is not None: @@ -290,10 +292,10 @@ def __ProcessResponse(self, response): """Process this response (by updating self and writing to self.stream).""" if response.status_code not in self._ACCEPTABLE_STATUSES: raise exceptions.TransferInvalidError(response.content) - if response.status_code in (httplib.OK, httplib.PARTIAL_CONTENT): + if response.status_code in (http_client.OK, http_client.PARTIAL_CONTENT): self.stream.write(response.content) self.__progress += len(response) - elif response.status_code == httplib.NO_CONTENT: + elif response.status_code == http_client.NO_CONTENT: # It's important to write something to the stream for the case # of a 0-byte download to a file, as otherwise python won't # create the file. @@ -348,7 +350,7 @@ def StreamInChunks(self, callback=None, finish_callback=None, additional_headers=additional_headers) response = self.__ProcessResponse(response) self._ExecuteCallback(callback, response) - if (response.status_code == httplib.OK or + if (response.status_code == http_client.OK or self.progress >= self.total_size): break self._ExecuteCallback(finish_callback, response) @@ -591,7 +593,7 @@ def _RefreshResumableUploadState(self): self.http, refresh_request, redirections=0) range_header = refresh_response.info.get( 'Range', refresh_response.info.get('range')) - if refresh_response.status_code in (httplib.OK, httplib.CREATED): + if refresh_response.status_code in (http_client.OK, http_client.CREATED): self.__complete = True elif refresh_response.status_code == http_wrapper.RESUME_INCOMPLETE: if range_header is None: @@ -619,7 +621,7 @@ def InitializeUpload(self, http_request, http=None, client=None): http_request.url = client.FinalizeTransferUrl(http_request.url) self.EnsureUninitialized() http_response = http_wrapper.MakeRequest(http, http_request) - if http_response.status_code != httplib.OK: + if http_response.status_code != http_client.OK: raise exceptions.HttpError.FromResponse(http_response) self.__server_chunk_granularity = http_response.info.get( @@ -651,11 +653,11 @@ def __ValidateChunksize(self, chunksize=None): @staticmethod def _ArgPrinter(response, unused_upload): - print 'Sent %s' % response.info['range'] + print('Sent %s' % response.info['range']) @staticmethod def _CompletePrinter(*unused_args): - print 'Upload complete' + print('Upload complete') def StreamInChunks(self, callback=None, finish_callback=None, additional_headers=None): @@ -674,7 +676,7 @@ def StreamInChunks(self, callback=None, finish_callback=None, while not self.complete: response = self.__SendChunk(self.stream.tell(), additional_headers=additional_headers) - if response.status_code in (httplib.OK, httplib.CREATED): + if response.status_code in (http_client.OK, http_client.CREATED): self.__complete = True break self.__progress = self.__GetLastByte(response.info['range']) @@ -703,10 +705,10 @@ def __SendChunk(self, start, additional_headers=None, data=None): request.headers.update(additional_headers) response = http_wrapper.MakeRequest(self.bytes_http, request) - if response.status_code not in (httplib.OK, httplib.CREATED, + if response.status_code not in (http_client.OK, http_client.CREATED, http_wrapper.RESUME_INCOMPLETE): raise exceptions.HttpError.FromResponse(response) - if response.status_code in (httplib.OK, httplib.CREATED): + if response.status_code in (http_client.OK, http_client.CREATED): return response # TODO(craigcitro): Add retries on no progress? last_byte = self.__GetLastByte(response.info['range']) diff --git a/_gcloud_vendor/apitools/base/py/util.py b/_gcloud_vendor/apitools/base/py/util.py index 4d64bedf538e..c2444c4e1948 100644 --- a/_gcloud_vendor/apitools/base/py/util.py +++ b/_gcloud_vendor/apitools/base/py/util.py @@ -2,13 +2,14 @@ """Assorted utilities shared between parts of apitools.""" import collections -import httplib import os import random -import types import urllib import urllib2 +import six +from six.moves import http_client + from _gcloud_vendor.apitools.base.py import exceptions __all__ = [ @@ -46,13 +47,13 @@ def DetectGce(): o = urllib2.urlopen('http://metadata.google.internal') except urllib2.URLError: return False - return (o.getcode() == httplib.OK and + return (o.getcode() == http_client.OK and o.headers.get('metadata-flavor') == 'Google') def NormalizeScopes(scope_spec): """Normalize scope_spec to a set of strings.""" - if isinstance(scope_spec, types.StringTypes): + if isinstance(scope_spec, six.string_types): return set(scope_spec.split(' ')) elif isinstance(scope_spec, collections.Iterable): return set(scope_spec) @@ -99,7 +100,7 @@ def ExpandRelativePath(method_config, params, relative_path=None): raise exceptions.InvalidUserInputError( 'Request missing required parameter %s' % param) try: - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): value = str(value) path = path.replace(param_template, urllib.quote(value.encode('utf_8'), reserved_chars)) From 9a68d900ba7b3f9eb530eb278e573937f69cf1e0 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 9 Dec 2014 15:56:33 -0500 Subject: [PATCH 5/5] Add a regression test for small-file writes. In conjunction with the existing large-file test, this one ensures that we exercise both the 'simple'/'media' upload API, and the 'resumable' one. --- regression/storage.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/regression/storage.py b/regression/storage.py index 7ab574b121f4..7427d5f72b31 100644 --- a/regression/storage.py +++ b/regression/storage.py @@ -123,6 +123,17 @@ def test_large_file_write_from_stream(self): key._properties.clear() # force a reload self.assertEqual(key.md5_hash, file_data['hash']) + def test_small_file_write_from_filename(self): + key = self.bucket.new_key('LargeFile') + self.assertEqual(key._properties, {}) + + file_data = self.FILES['simple'] + key.upload_from_filename(file_data['path']) + self.case_keys_to_delete.append(key) + + key._properties.clear() # force a reload + self.assertEqual(key.md5_hash, file_data['hash']) + def test_write_metadata(self): key = self.bucket.upload_file(self.FILES['logo']['path']) self.case_keys_to_delete.append(key)