From 2da75d4e76b3161d3d81869cab9f58613742ddc9 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 6 Oct 2016 15:05:23 -0400 Subject: [PATCH 1/2] Make 'encryption_key' an attribute of the Blob. Avoids the need to plumb it through all the 'upload' and 'download' methods. Convert '_set_encryption_headers(key, headers)' into a pure function, '_get_encryption_headers(key)', returning a new dict. Preparing for use of encryption in to-be-added 'Blob.rewrite(source_blob)' method. See: #1960. --- storage/google/cloud/storage/blob.py | 108 +++++++++++---------------- storage/unit_tests/test_blob.py | 30 +++++--- 2 files changed, 63 insertions(+), 75 deletions(-) diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index e0c8eb536020..f2e8c96edb37 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -61,6 +61,11 @@ class Blob(_PropertyMixin): :param chunk_size: The size of a chunk of data whenever iterating (1 MB). This must be a multiple of 256 KB per the API specification. + + :type encryption_key: bytes + :param encryption_key: + Optional 32 byte encryption key for customer-supplied encryption. + See https://cloud.google.com/storage/docs/encryption#customer-supplied """ _chunk_size = None # Default value for each instance. @@ -68,12 +73,13 @@ class Blob(_PropertyMixin): _CHUNK_SIZE_MULTIPLE = 256 * 1024 """Number (256 KB, in bytes) that must divide the chunk size.""" - def __init__(self, name, bucket, chunk_size=None): + def __init__(self, name, bucket, chunk_size=None, encryption_key=None): super(Blob, self).__init__(name=name) self.chunk_size = chunk_size # Check that setter accepts value. self.bucket = bucket self._acl = ObjectACL(self) + self._encryption_key = encryption_key @property def chunk_size(self): @@ -284,7 +290,7 @@ def delete(self, client=None): """ return self.bucket.delete_blob(self.name, client=client) - def download_to_file(self, file_obj, encryption_key=None, client=None): + def download_to_file(self, file_obj, client=None): """Download the contents of this blob into a file-like object. .. note:: @@ -301,10 +307,10 @@ def download_to_file(self, file_obj, encryption_key=None, client=None): >>> client = storage.Client(project='my-project') >>> bucket = client.get_bucket('my-bucket') >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' - >>> blob = Blob('secure-data', bucket) + >>> blob = Blob('secure-data', bucket, + ... encryption_key=encryption_key) >>> with open('/tmp/my-secure-file', 'wb') as file_obj: - >>> blob.download_to_file(file_obj, - ... encryption_key=encryption_key) + >>> blob.download_to_file(file_obj) The ``encryption_key`` should be a str or bytes with a length of at least 32. @@ -315,10 +321,6 @@ def download_to_file(self, file_obj, encryption_key=None, client=None): :type file_obj: file :param file_obj: A file handle to which to write the blob's data. - :type encryption_key: str or bytes - :param encryption_key: Optional 32 byte encryption key for - customer-supplied encryption. - :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back @@ -338,9 +340,7 @@ def download_to_file(self, file_obj, encryption_key=None, client=None): if self.chunk_size is not None: download.chunksize = self.chunk_size - headers = {} - if encryption_key: - _set_encryption_headers(encryption_key, headers) + headers = _get_encryption_headers(self._encryption_key) request = Request(download_url, 'GET', headers) @@ -352,16 +352,12 @@ def download_to_file(self, file_obj, encryption_key=None, client=None): # it has all three (http, API_BASE_URL and build_api_url). download.initialize_download(request, client._connection.http) - def download_to_filename(self, filename, encryption_key=None, client=None): + def download_to_filename(self, filename, client=None): """Download the contents of this blob into a named file. :type filename: string :param filename: A filename to be passed to ``open``. - :type encryption_key: str or bytes - :param encryption_key: Optional 32 byte encryption key for - customer-supplied encryption. - :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back @@ -370,19 +366,14 @@ def download_to_filename(self, filename, encryption_key=None, client=None): :raises: :class:`google.cloud.exceptions.NotFound` """ with open(filename, 'wb') as file_obj: - self.download_to_file(file_obj, encryption_key=encryption_key, - client=client) + self.download_to_file(file_obj, client=client) mtime = time.mktime(self.updated.timetuple()) os.utime(file_obj.name, (mtime, mtime)) - def download_as_string(self, encryption_key=None, client=None): + def download_as_string(self, client=None): """Download the contents of this blob as a string. - :type encryption_key: str or bytes - :param encryption_key: Optional 32 byte encryption key for - customer-supplied encryption. - :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back @@ -393,8 +384,7 @@ def download_as_string(self, encryption_key=None, client=None): :raises: :class:`google.cloud.exceptions.NotFound` """ string_buffer = BytesIO() - self.download_to_file(string_buffer, encryption_key=encryption_key, - client=client) + self.download_to_file(string_buffer, client=client) return string_buffer.getvalue() @staticmethod @@ -409,8 +399,7 @@ def _check_response_error(request, http_response): # pylint: disable=too-many-locals def upload_from_file(self, file_obj, rewind=False, size=None, - encryption_key=None, content_type=None, num_retries=6, - client=None): + content_type=None, num_retries=6, client=None): """Upload the contents of this blob from a file-like object. The content type of the upload will either be @@ -437,10 +426,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None, >>> client = storage.Client(project='my-project') >>> bucket = client.get_bucket('my-bucket') >>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19' - >>> blob = Blob('secure-data', bucket) + >>> blob = Blob('secure-data', bucket, + ... encryption_key=encryption_key) >>> with open('my-file', 'rb') as my_file: - >>> blob.upload_from_file(my_file, - ... encryption_key=encryption_key) + >>> blob.upload_from_file(my_file) The ``encryption_key`` should be a str or bytes with a length of at least 32. @@ -461,10 +450,6 @@ def upload_from_file(self, file_obj, rewind=False, size=None, :func:`os.fstat`. (If the file handle is not from the filesystem this won't be possible.) - :type encryption_key: str or bytes - :param encryption_key: Optional 32 byte encryption key for - customer-supplied encryption. - :type content_type: string or ``NoneType`` :param content_type: Optional type of content being uploaded. @@ -510,8 +495,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None, 'User-Agent': connection.USER_AGENT, } - if encryption_key: - _set_encryption_headers(encryption_key, headers) + headers.update(_get_encryption_headers(self._encryption_key)) upload = Upload(file_obj, content_type, total_bytes, auto_transfer=False) @@ -561,8 +545,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None, self._set_properties(json.loads(response_content)) # pylint: enable=too-many-locals - def upload_from_filename(self, filename, content_type=None, - encryption_key=None, client=None): + def upload_from_filename(self, filename, content_type=None, client=None): """Upload this blob's contents from the content of a named file. The content type of the upload will either be @@ -587,10 +570,6 @@ def upload_from_filename(self, filename, content_type=None, :type content_type: string or ``NoneType`` :param content_type: Optional type of content being uploaded. - :type encryption_key: str or bytes - :param encryption_key: Optional 32 byte encryption key for - customer-supplied encryption. - :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back @@ -601,11 +580,10 @@ def upload_from_filename(self, filename, content_type=None, content_type, _ = mimetypes.guess_type(filename) with open(filename, 'rb') as file_obj: - self.upload_from_file(file_obj, content_type=content_type, - encryption_key=encryption_key, client=client) + self.upload_from_file( + file_obj, content_type=content_type, client=client) - def upload_from_string(self, data, content_type='text/plain', - encryption_key=None, client=None): + def upload_from_string(self, data, content_type='text/plain', client=None): """Upload contents of this blob from the provided string. .. note:: @@ -627,10 +605,6 @@ def upload_from_string(self, data, content_type='text/plain', :param content_type: Optional type of content being uploaded. Defaults to ``'text/plain'``. - :type encryption_key: str or bytes - :param encryption_key: Optional 32 byte encryption key for - customer-supplied encryption. - :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` :param client: Optional. The client to use. If not passed, falls back @@ -640,9 +614,9 @@ def upload_from_string(self, data, content_type='text/plain', data = data.encode('utf-8') string_buffer = BytesIO() string_buffer.write(data) - self.upload_from_file(file_obj=string_buffer, rewind=True, - size=len(data), content_type=content_type, - encryption_key=encryption_key, client=client) + self.upload_from_file( + file_obj=string_buffer, rewind=True, size=len(data), + content_type=content_type, client=client) def make_public(self, client=None): """Make this blob public giving all users read access. @@ -964,19 +938,25 @@ def __init__(self, bucket_name, object_name): self._relative_path = '' -def _set_encryption_headers(key, headers): +def _get_encryption_headers(key): """Builds customer encryption key headers - :type key: str or bytes + :type key: bytes :param key: 32 byte key to build request key and hash. - :type headers: dict - :param headers: dict of HTTP headers being sent in request. + :rtype: dict + :returns: dict of HTTP headers being sent in request. """ + if key is None: + return {} + key = _to_bytes(key) - sha256_key = hashlib.sha256(key).digest() - key_hash = base64.b64encode(sha256_key).rstrip() - encoded_key = base64.b64encode(key).rstrip() - headers['X-Goog-Encryption-Algorithm'] = 'AES256' - headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key) - headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash) + key_hash = hashlib.sha256(key).digest() + key_hash = base64.b64encode(key_hash).rstrip() + key = base64.b64encode(key).rstrip() + + return { + 'X-Goog-Encryption-Algorithm': 'AES256', + 'X-Goog-Encryption-Key': _bytes_to_unicode(key), + 'X-Goog-Encryption-Key-Sha256': _bytes_to_unicode(key_hash), + } diff --git a/storage/unit_tests/test_blob.py b/storage/unit_tests/test_blob.py index 47622cedc766..9c5d3527dd45 100644 --- a/storage/unit_tests/test_blob.py +++ b/storage/unit_tests/test_blob.py @@ -24,7 +24,7 @@ def _makeOne(self, *args, **kw): blob._properties = properties or {} return blob - def test_ctor(self): + def test_ctor_wo_encryption_key(self): BLOB_NAME = 'blob-name' bucket = _Bucket() properties = {'key': 'value'} @@ -34,6 +34,14 @@ def test_ctor(self): self.assertEqual(blob._properties, properties) self.assertFalse(blob._acl.loaded) self.assertIs(blob._acl.blob, blob) + self.assertEqual(blob._encryption_key, None) + + def test_ctor_w_encryption_key(self): + KEY = b'01234567890123456789012345678901' # 32 bytes + BLOB_NAME = 'blob-name' + bucket = _Bucket() + blob = self._makeOne(BLOB_NAME, bucket=bucket, encryption_key=KEY) + self.assertEqual(blob._encryption_key, KEY) def test_chunk_size_ctor(self): from google.cloud.storage.blob import Blob @@ -391,7 +399,7 @@ def test_download_to_filename_w_key(self): from google.cloud._testing import _NamedTemporaryFile BLOB_NAME = 'blob-name' - KEY = 'aa426195405adee2c8081bb9e7e74b19' + KEY = b'aa426195405adee2c8081bb9e7e74b19' HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk=' HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0=' chunk1_response = {'status': PARTIAL_CONTENT, @@ -407,12 +415,13 @@ def test_download_to_filename_w_key(self): MEDIA_LINK = 'http://example.com/media/' properties = {'mediaLink': MEDIA_LINK, 'updated': '2014-12-06T13:13:50.690Z'} - blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties) + blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties, + encryption_key=KEY) blob._CHUNK_SIZE_MULTIPLE = 1 blob.chunk_size = 3 with _NamedTemporaryFile() as temp: - blob.download_to_filename(temp.name, encryption_key=KEY) + blob.download_to_filename(temp.name) with open(temp.name, 'rb') as file_obj: wrote = file_obj.read() mtime = os.path.getmtime(temp.name) @@ -835,7 +844,7 @@ def test_upload_from_filename_w_key(self): BLOB_NAME = 'blob-name' UPLOAD_URL = 'http://example.com/upload/name/key' DATA = b'ABCDEF' - KEY = 'aa426195405adee2c8081bb9e7e74b19' + KEY = b'aa426195405adee2c8081bb9e7e74b19' HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk=' HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0=' EXPECTED_CONTENT_TYPE = 'foo/bar' @@ -852,7 +861,7 @@ def test_upload_from_filename_w_key(self): client = _Client(connection) bucket = _Bucket(client) blob = self._makeOne(BLOB_NAME, bucket=bucket, - properties=properties) + properties=properties, encryption_key=KEY) blob._CHUNK_SIZE_MULTIPLE = 1 blob.chunk_size = 5 @@ -860,8 +869,7 @@ def test_upload_from_filename_w_key(self): with open(temp.name, 'wb') as file_obj: file_obj.write(DATA) blob.upload_from_filename(temp.name, - content_type=EXPECTED_CONTENT_TYPE, - encryption_key=KEY) + content_type=EXPECTED_CONTENT_TYPE) rq = connection.http._requested self.assertEqual(len(rq), 1) @@ -1040,7 +1048,7 @@ def test_upload_from_string_text_w_key(self): from six.moves.urllib.parse import urlsplit from google.cloud.streaming import http_wrapper BLOB_NAME = 'blob-name' - KEY = 'aa426195405adee2c8081bb9e7e74b19' + KEY = b'aa426195405adee2c8081bb9e7e74b19' HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk=' HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0=' UPLOAD_URL = 'http://example.com/upload/name/key' @@ -1057,10 +1065,10 @@ def test_upload_from_string_text_w_key(self): ) client = _Client(connection) bucket = _Bucket(client=client) - blob = self._makeOne(BLOB_NAME, bucket=bucket) + blob = self._makeOne(BLOB_NAME, bucket=bucket, encryption_key=KEY) blob._CHUNK_SIZE_MULTIPLE = 1 blob.chunk_size = 5 - blob.upload_from_string(DATA, encryption_key=KEY) + blob.upload_from_string(DATA) rq = connection.http._requested self.assertEqual(len(rq), 1) self.assertEqual(rq[0]['method'], 'POST') From cf451cf12590cb9ae94f3c074486109fce81a5fc Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 6 Oct 2016 16:40:22 -0400 Subject: [PATCH 2/2] Pass 'encryption_key' through 'Bucket.blob' factory. --- storage/google/cloud/storage/bucket.py | 9 +++++++-- storage/unit_tests/test_bucket.py | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index 77f86106f4e7..3068bfb2fe06 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -111,7 +111,7 @@ def client(self): """The client bound to this bucket.""" return self._client - def blob(self, blob_name, chunk_size=None): + def blob(self, blob_name, chunk_size=None, encryption_key=None): """Factory constructor for blob object. .. note:: @@ -126,10 +126,15 @@ def blob(self, blob_name, chunk_size=None): (1 MB). This must be a multiple of 256 KB per the API specification. + :type encryption_key: bytes + :param encryption_key: + Optional 32 byte encryption key for customer-supplied encryption. + :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The blob object created. """ - return Blob(name=blob_name, bucket=self, chunk_size=chunk_size) + return Blob(name=blob_name, bucket=self, chunk_size=chunk_size, + encryption_key=encryption_key) def exists(self, client=None): """Determines whether or not this bucket exists. diff --git a/storage/unit_tests/test_bucket.py b/storage/unit_tests/test_bucket.py index c8e598d0a27d..a0b8c1847b1d 100644 --- a/storage/unit_tests/test_bucket.py +++ b/storage/unit_tests/test_bucket.py @@ -113,14 +113,17 @@ def test_blob(self): BUCKET_NAME = 'BUCKET_NAME' BLOB_NAME = 'BLOB_NAME' CHUNK_SIZE = 1024 * 1024 + KEY = b'01234567890123456789012345678901' # 32 bytes bucket = self._makeOne(name=BUCKET_NAME) - blob = bucket.blob(BLOB_NAME, chunk_size=CHUNK_SIZE) + blob = bucket.blob( + BLOB_NAME, chunk_size=CHUNK_SIZE, encryption_key=KEY) self.assertIsInstance(blob, Blob) self.assertIs(blob.bucket, bucket) self.assertIs(blob.client, bucket.client) self.assertEqual(blob.name, BLOB_NAME) self.assertEqual(blob.chunk_size, CHUNK_SIZE) + self.assertEqual(blob._encryption_key, KEY) def test_exists_miss(self): from google.cloud.exceptions import NotFound