diff --git a/src/spring/azext_spring/_buildservices_factory.py b/src/spring/azext_spring/_buildservices_factory.py index c84e7dca670..437893b4932 100644 --- a/src/spring/azext_spring/_buildservices_factory.py +++ b/src/spring/azext_spring/_buildservices_factory.py @@ -40,7 +40,7 @@ def build_and_get_result(self, total_steps, **kwargs): logger.warning("[1/{}] Requesting for upload URL.".format(total_steps)) upload_info = self._get_upload_info() logger.warning("[2/{}] Uploading package to blob.".format(total_steps)) - uploader_selector(upload_url=upload_info.upload_url, **kwargs).upload_and_build(**kwargs) + uploader_selector(cli_ctx=self.cmd.cli_ctx, upload_url=upload_info.upload_url, **kwargs).upload_and_build(**kwargs) logger.warning("[3/{}] Creating or Updating build '{}'.".format(total_steps, kwargs['app'])) build_result_id = self._queue_build(upload_info.relative_path, **kwargs) logger.warning("[4/{}] Waiting for building docker image to finish. This may take a few minutes.".format(total_steps)) diff --git a/src/spring/azext_spring/_deployment_deployable_factory.py b/src/spring/azext_spring/_deployment_deployable_factory.py index 2271e6d2dfa..b631b5e0daf 100644 --- a/src/spring/azext_spring/_deployment_deployable_factory.py +++ b/src/spring/azext_spring/_deployment_deployable_factory.py @@ -71,7 +71,7 @@ def build_deployable_path(self, **kwargs): return upload_info.relative_path def _get_uploader(self, upload_url=None): - return FileUpload(upload_url=upload_url) + return FileUpload(upload_url=upload_url, cli_ctx=self.cmd.cli_ctx) class SourceBuildDeployableBuilder(UploadDeployableBuilder): @@ -82,7 +82,7 @@ def build_deployable_path(self, **kwargs): return relative_path def _get_uploader(self, upload_url=None): - return FolderUpload(upload_url=upload_url) + return FolderUpload(upload_url=upload_url, cli_ctx=self.cmd.cli_ctx) def get_source_type(self, **_): return 'Source' diff --git a/src/spring/azext_spring/_deployment_uploadable_factory.py b/src/spring/azext_spring/_deployment_uploadable_factory.py index 1c9ba9a219b..2014543390b 100644 --- a/src/spring/azext_spring/_deployment_uploadable_factory.py +++ b/src/spring/azext_spring/_deployment_uploadable_factory.py @@ -9,7 +9,7 @@ import tempfile import uuid from azure.cli.core.azclierror import InvalidArgumentValueError -from .azure_storage_file import FileService +from azure.cli.core.profiles import ResourceType, get_sdk from ._utils import (get_azure_files_info, _pack_source_code) @@ -22,13 +22,14 @@ class FileUpload: ''' Upload a file in local file system to upload url ''' - def __init__(self, upload_url): + def __init__(self, upload_url, cli_ctx): account_name, endpoint_suffix, share_name, relative_name, sas_token = get_azure_files_info(upload_url) self.account_name = account_name self.endpoint_suffix = endpoint_suffix self.share_name = share_name self.relative_name = relative_name self.sas_token = sas_token + self.cli_ctx = cli_ctx def upload_and_build(self, artifact_path, **_): if not artifact_path: @@ -36,6 +37,7 @@ def upload_and_build(self, artifact_path, **_): self._upload(artifact_path) def _upload(self, artifact_path): + FileService = get_sdk(self.cli_ctx, ResourceType.DATA_STORAGE, 'file#FileService') file_service = FileService(self.account_name, sas_token=self.sas_token, endpoint_suffix=self.endpoint_suffix) file_service.create_file_from_path(self.share_name, None, self.relative_name, artifact_path) @@ -56,9 +58,9 @@ def _compress_folder(self, folder): return file_path -def uploader_selector(source_path=None, artifact_path=None, upload_url=None, **_): +def uploader_selector(cli_ctx, source_path=None, artifact_path=None, upload_url=None, **_): if source_path: - return FolderUpload(upload_url) + return FolderUpload(upload_url, cli_ctx) if artifact_path: - return FileUpload(upload_url) + return FileUpload(upload_url, cli_ctx) return Empty() diff --git a/src/spring/azext_spring/azure_storage_file/__init__.py b/src/spring/azext_spring/azure_storage_file/__init__.py deleted file mode 100644 index c4de86a5bb6..00000000000 --- a/src/spring/azext_spring/azure_storage_file/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -# pylint: disable=unused-import - -from .fileservice import FileService -from .models import ( - Share, - ShareProperties, - File, - FileProperties, - Directory, - DirectoryProperties, - FileRange, - ContentSettings, - CopyProperties, - SharePermissions, - FilePermissions, - DeleteSnapshot, - SMBProperties, - NTFSAttributes, -) -from ._constants import __version__ diff --git a/src/spring/azext_spring/azure_storage_file/_constants.py b/src/spring/azext_spring/azure_storage_file/_constants.py deleted file mode 100644 index d71baea3083..00000000000 --- a/src/spring/azext_spring/azure_storage_file/_constants.py +++ /dev/null @@ -1,11 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -__author__ = 'Microsoft Corp. ' -__version__ = '2.1.0' - -# x-ms-version for storage service. -X_MS_VERSION = '2019-02-02' diff --git a/src/spring/azext_spring/azure_storage_file/_deserialization.py b/src/spring/azext_spring/azure_storage_file/_deserialization.py deleted file mode 100644 index 92870d6922a..00000000000 --- a/src/spring/azext_spring/azure_storage_file/_deserialization.py +++ /dev/null @@ -1,334 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -# pylint: disable=wrong-import-order - -from dateutil import parser # pylint: disable=import-error - -try: - from xml.etree import cElementTree as ETree -except ImportError: - from xml.etree import ElementTree as ETree -from .models import ( - Share, - Directory, - File, - Handle, - FileProperties, - FileRange, - ShareProperties, - DirectoryProperties, -) -from azure.storage.common.models import ( - _list, -) -from azure.storage.common._deserialization import ( - _parse_properties, - _parse_metadata, -) -from azure.storage.common._error import _validate_content_match -from azure.storage.common._common_conversion import ( - _get_content_md5, - _to_str, -) - - -def _parse_snapshot_share(response, name): - ''' - Extracts snapshot return header. - ''' - snapshot = response.headers.get('x-ms-snapshot') - - return _parse_share(response, name, snapshot) - - -def _parse_share(response, name, snapshot=None): - if response is None: - return None - - metadata = _parse_metadata(response) - props = _parse_properties(response, ShareProperties) - return Share(name, props, metadata, snapshot) - - -def _parse_directory(response, name): - if response is None: - return None - - metadata = _parse_metadata(response) - props = _parse_properties(response, DirectoryProperties) - return Directory(name, props, metadata) - - -def _parse_permission_key(response): - ''' - Extracts out file permission key - ''' - - if response is None or response.headers is None: - return None - return response.headers.get('x-ms-file-permission-key', None) - - -def _parse_permission(response): - ''' - Extracts out file permission - ''' - - return response.body - - -def _parse_file(response, name, validate_content=False): - if response is None: - return None - - metadata = _parse_metadata(response) - props = _parse_properties(response, FileProperties) - - # For range gets, only look at 'x-ms-content-md5' for overall MD5 - content_settings = getattr(props, 'content_settings') - if 'content-range' in response.headers: - if 'x-ms-content-md5' in response.headers: - setattr(content_settings, 'content_md5', _to_str( - response.headers['x-ms-content-md5'])) - else: - delattr(content_settings, 'content_md5') - - if validate_content: - computed_md5 = _get_content_md5(response.body) - _validate_content_match(response.headers['content-md5'], computed_md5) - - return File(name, response.body, props, metadata) - - -def _convert_xml_to_shares(response): - ''' - - - string-value - string-value - int-value - - - share-name - date-time-value - - date/time-value - etag - max-share-size - - - value - - - - marker-value - - ''' - if response is None or response.body is None: - return None - - shares = _list() - list_element = ETree.fromstring(response.body) - - # Set next marker - next_marker = list_element.findtext('NextMarker') or None - setattr(shares, 'next_marker', next_marker) - - shares_element = list_element.find('Shares') - - for share_element in shares_element.findall('Share'): - # Name element - share = Share() - share.name = share_element.findtext('Name') - - # Snapshot - share.snapshot = share_element.findtext('Snapshot') - - # Metadata - metadata_root_element = share_element.find('Metadata') - if metadata_root_element is not None: - share.metadata = dict() - for metadata_element in metadata_root_element: - share.metadata[metadata_element.tag] = metadata_element.text - - # Properties - properties_element = share_element.find('Properties') - share.properties.last_modified = parser.parse( - properties_element.findtext('Last-Modified')) - share.properties.etag = properties_element.findtext('Etag') - share.properties.quota = int(properties_element.findtext('Quota')) - - # Add share to list - shares.append(share) - - return shares - - -# pylint: disable=line-too-long -def _convert_xml_to_directories_and_files(response): - ''' - - - string-value - int-value - - - file-name - - size-in-bytes - - - - directory-name - - - - - ''' - if response is None or response.body is None: - return None - - entries = _list() - list_element = ETree.fromstring(response.body) - - # Set next marker - next_marker = list_element.findtext('NextMarker') or None - setattr(entries, 'next_marker', next_marker) - - entries_element = list_element.find('Entries') - - for file_element in entries_element.findall('File'): - # Name element - _file = File() - _file.name = file_element.findtext('Name') - - # Properties - properties_element = file_element.find('Properties') - _file.properties.content_length = int( - properties_element.findtext('Content-Length')) - - # Add file to list - entries.append(_file) - - for directory_element in entries_element.findall('Directory'): - # Name element - directory = Directory() - directory.name = directory_element.findtext('Name') - - # Add directory to list - entries.append(directory) - - return entries - - -def _convert_xml_to_handles(response): - """ - - - - - 21123954401 - - 0 - 0 - 9385737614310506553 - 167.220.2.92:27553 - Fri, 03 May 2019 05:59:43 GMT - - ... - - - ' - """ - if response is None or response.body is None: - return None - - entries = _list() - list_element = ETree.fromstring(response.body) - - # Set next marker - next_marker = list_element.findtext('NextMarker') or None - setattr(entries, 'next_marker', next_marker) - - handles_list_element = list_element.find('Entries') - - for handle_element in handles_list_element.findall('Handle'): - # Name element - handle = Handle() - handle.handle_id = handle_element.findtext('HandleId') - handle.path = handle_element.findtext('Path') - handle.file_id = handle_element.findtext('FileId') - handle.parent_id = handle_element.findtext('ParentId') - handle.session_id = handle_element.findtext('SessionId') - handle.client_ip = handle_element.findtext('ClientIp') - handle.open_time = parser.parse(handle_element.findtext('OpenTime')) - - last_connect_time_string = handle_element.findtext('LastReconnectTime') - if last_connect_time_string is not None: - handle.last_reconnect_time = parser.parse(last_connect_time_string) - - # Add file to list - entries.append(handle) - - return entries - - -def _parse_close_handle_response(response): - if response is None or response.body is None: - return 0 - - results = _list() - results.append(int(response.headers['x-ms-number-of-handles-closed'])) - - next_marker = None if 'x-ms-marker' not in response.headers else response.headers['x-ms-marker'] - setattr(results, 'next_marker', next_marker) - return results - - -def _convert_xml_to_ranges(response): - ''' - - - - Start Byte - End Byte - - - Start Byte - End Byte - - - ''' - if response is None or response.body is None: - return None - - ranges = list() - ranges_element = ETree.fromstring(response.body) - - for range_element in ranges_element.findall('Range'): - # Parse range - rangeObj = FileRange(int(range_element.findtext('Start')), - int(range_element.findtext('End'))) - - # Add range to list - ranges.append(rangeObj) - - return ranges - - -def _convert_xml_to_share_stats(response): - ''' - - - 15 - - ''' - if response is None or response.body is None: - return None - - share_stats_element = ETree.fromstring(response.body) - return int(share_stats_element.findtext('ShareUsageBytes')) diff --git a/src/spring/azext_spring/azure_storage_file/_download_chunking.py b/src/spring/azext_spring/azure_storage_file/_download_chunking.py deleted file mode 100644 index e8f9cebd510..00000000000 --- a/src/spring/azext_spring/azure_storage_file/_download_chunking.py +++ /dev/null @@ -1,165 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -# pylint: disable=too-few-public-methods, too-many-instance-attributes - -import threading -import concurrent.futures - - -def _download_file_chunks(file_service, share_name, directory_name, file_name, - download_size, block_size, progress, start_range, end_range, - stream, max_connections, progress_callback, validate_content, - timeout, operation_context, snapshot): - - downloader_class = _ParallelFileChunkDownloader if max_connections > 1 else _SequentialFileChunkDownloader - - downloader = downloader_class( - file_service, - share_name, - directory_name, - file_name, - download_size, - block_size, - progress, - start_range, - end_range, - stream, - progress_callback, - validate_content, - timeout, - operation_context, - snapshot, - ) - - if max_connections > 1: - executor = concurrent.futures.ThreadPoolExecutor(max_connections) - list(executor.map(downloader.process_chunk, downloader.get_chunk_offsets())) - else: - for chunk in downloader.get_chunk_offsets(): - downloader.process_chunk(chunk) - - -class _FileChunkDownloader(object): - def __init__(self, file_service, share_name, directory_name, file_name, - download_size, chunk_size, progress, start_range, end_range, - stream, progress_callback, validate_content, timeout, operation_context, snapshot): - # identifiers for the file - self.file_service = file_service - self.share_name = share_name - self.directory_name = directory_name - self.file_name = file_name - - # information on the download range/chunk size - self.chunk_size = chunk_size - self.download_size = download_size - self.start_index = start_range - self.file_end = end_range - - # the destination that we will write to - self.stream = stream - - # progress related - self.progress_callback = progress_callback - self.progress_total = progress - - # parameters for each get file operation - self.validate_content = validate_content - self.timeout = timeout - self.operation_context = operation_context - self.snapshot = snapshot - - def get_chunk_offsets(self): - index = self.start_index - while index < self.file_end: - yield index - index += self.chunk_size - - def process_chunk(self, chunk_start): - if chunk_start + self.chunk_size > self.file_end: - chunk_end = self.file_end - else: - chunk_end = chunk_start + self.chunk_size - - chunk_data = self._download_chunk(chunk_start, chunk_end).content - length = chunk_end - chunk_start - if length > 0: - self._write_to_stream(chunk_data, chunk_start) - self._update_progress(length) - - # should be provided by the subclass - def _update_progress(self, length): - pass - - # should be provided by the subclass - def _write_to_stream(self, chunk_data, chunk_start): - pass - - # pylint: disable=protected-access - def _download_chunk(self, chunk_start, chunk_end): - return self.file_service._get_file( - self.share_name, - self.directory_name, - self.file_name, - start_range=chunk_start, - end_range=chunk_end - 1, - validate_content=self.validate_content, - timeout=self.timeout, - _context=self.operation_context, - snapshot=self.snapshot - ) - - -class _ParallelFileChunkDownloader(_FileChunkDownloader): - def __init__(self, file_service, share_name, directory_name, file_name, - download_size, chunk_size, progress, start_range, end_range, - stream, progress_callback, validate_content, timeout, operation_context, snapshot): - super(_ParallelFileChunkDownloader, self).__init__(file_service, share_name, directory_name, file_name, - download_size, chunk_size, progress, start_range, end_range, - stream, progress_callback, validate_content, timeout, - operation_context, snapshot) - - # for a parallel download, the stream is always seekable, so we note down the current position - # in order to seek to the right place when out-of-order chunks come in - self.stream_start = stream.tell() - - # since parallel operations are going on - # it is essential to protect the writing and progress reporting operations - self.stream_lock = threading.Lock() - self.progress_lock = threading.Lock() - - def _update_progress(self, length): - if self.progress_callback is not None: - with self.progress_lock: - self.progress_total += length - total_so_far = self.progress_total - self.progress_callback(total_so_far, self.download_size) - - def _write_to_stream(self, chunk_data, chunk_start): - with self.stream_lock: - self.stream.seek(self.stream_start + - (chunk_start - self.start_index)) - self.stream.write(chunk_data) - - -# pylint: disable=useless-super-delegation -class _SequentialFileChunkDownloader(_FileChunkDownloader): - def __init__(self, file_service, share_name, directory_name, file_name, download_size, chunk_size, progress, - start_range, end_range, stream, progress_callback, validate_content, timeout, operation_context, - snapshot): - super(_SequentialFileChunkDownloader, self).__init__(file_service, share_name, directory_name, file_name, - download_size, chunk_size, progress, start_range, - end_range, stream, progress_callback, validate_content, - timeout, operation_context, snapshot) - - def _update_progress(self, length): - if self.progress_callback is not None: - self.progress_total += length - self.progress_callback(self.progress_total, self.download_size) - - def _write_to_stream(self, chunk_data, chunk_start): - # chunk_start is ignored in the case of sequential download since we cannot seek the destination stream - self.stream.write(chunk_data) diff --git a/src/spring/azext_spring/azure_storage_file/_serialization.py b/src/spring/azext_spring/azure_storage_file/_serialization.py deleted file mode 100644 index e0a07c0baef..00000000000 --- a/src/spring/azext_spring/azure_storage_file/_serialization.py +++ /dev/null @@ -1,96 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from azure.storage.common._common_conversion import _str -from azure.storage.common._error import ( - _validate_not_none, - _ERROR_START_END_NEEDED_FOR_MD5, - _ERROR_RANGE_TOO_LARGE_FOR_MD5, -) -_ERROR_TOO_MANY_FILE_PERMISSIONS = 'file_permission and file_permission_key should not be set at the same time' -_FILE_PERMISSION_TOO_LONG = 'Size of file_permission is too large. file_permission should be <=8KB, else' \ - 'please use file_permission_key' - - -def _get_path(share_name=None, directory_name=None, file_name=None): - ''' - Creates the path to access a file resource. - - share_name: - Name of share. - directory_name: - The path to the directory. - file_name: - Name of file. - ''' - if share_name and directory_name and file_name: - return '/{0}/{1}/{2}'.format( - _str(share_name), - _str(directory_name), - _str(file_name)) - if share_name and directory_name: - return '/{0}/{1}'.format( - _str(share_name), - _str(directory_name)) - if share_name and file_name: - return '/{0}/{1}'.format( - _str(share_name), - _str(file_name)) - if share_name: - return '/{0}'.format(_str(share_name)) - - return '/' - - -def _validate_and_format_range_headers(request, start_range, end_range, start_range_required=True, - end_range_required=True, check_content_md5=False, is_source=False): - # If end range is provided, start range must be provided - if start_range_required or end_range is not None: - _validate_not_none('start_range', start_range) - if end_range_required: - _validate_not_none('end_range', end_range) - - # Format based on whether end_range is present - request.headers = request.headers or {} - header_name = 'x-ms-source-range' if is_source else 'x-ms-range' - if end_range is not None: - request.headers[header_name] = 'bytes={0}-{1}'.format( - start_range, end_range) - elif start_range is not None: - request.headers[header_name] = 'bytes={0}-'.format(start_range) - - # Content MD5 can only be provided for a complete range less than 4MB in size - if check_content_md5: - if start_range is None or end_range is None: - raise ValueError(_ERROR_START_END_NEEDED_FOR_MD5) - if end_range - start_range > 4 * 1024 * 1024: - raise ValueError(_ERROR_RANGE_TOO_LARGE_FOR_MD5) - - request.headers['x-ms-range-get-content-md5'] = 'true' - - -def _validate_and_return_file_permission(file_permission, file_permission_key, default_permission): - # if file_permission and file_permission_key are both empty, then use the default_permission - # value as file permission, file_permission size should be <= 8KB, else file permission_key should be used - empty_file_permission = file_permission is None or len( - file_permission) == 0 - empty_file_permission_key = file_permission_key is None or len( - file_permission_key) == 0 - file_permission_size_too_big = False if file_permission is None \ - else len(str(file_permission).encode('utf-8')) > 8 * 1024 - - if file_permission_size_too_big: - raise ValueError(_FILE_PERMISSION_TOO_LONG) - - if empty_file_permission: - if empty_file_permission_key: - return default_permission - return None - - if empty_file_permission_key: - return file_permission - - raise ValueError(_ERROR_TOO_MANY_FILE_PERMISSIONS) diff --git a/src/spring/azext_spring/azure_storage_file/_upload_chunking.py b/src/spring/azext_spring/azure_storage_file/_upload_chunking.py deleted file mode 100644 index ea56427e29c..00000000000 --- a/src/spring/azext_spring/azure_storage_file/_upload_chunking.py +++ /dev/null @@ -1,138 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -# pylint: disable=too-few-public-methods, too-many-instance-attributes - -import threading -import concurrent.futures - - -def _upload_file_chunks(file_service, share_name, directory_name, file_name, - file_size, block_size, stream, max_connections, - progress_callback, validate_content, timeout): - uploader = _FileChunkUploader( - file_service, - share_name, - directory_name, - file_name, - file_size, - block_size, - stream, - max_connections > 1, - progress_callback, - validate_content, - timeout - ) - - if progress_callback is not None: - progress_callback(0, file_size) - - if max_connections > 1: - executor = concurrent.futures.ThreadPoolExecutor(max_connections) - range_ids = list(executor.map(uploader.process_chunk, - uploader.get_chunk_offsets())) - else: - if file_size is not None: - range_ids = [uploader.process_chunk( - start) for start in uploader.get_chunk_offsets()] - else: - range_ids = uploader.process_all_unknown_size() - - return range_ids - - -class _FileChunkUploader(object): - def __init__(self, file_service, share_name, directory_name, file_name, - file_size, chunk_size, stream, parallel, progress_callback, - validate_content, timeout): - self.file_service = file_service - self.share_name = share_name - self.directory_name = directory_name - self.file_name = file_name - self.file_size = file_size - self.chunk_size = chunk_size - self.stream = stream - self.stream_start = stream.tell() if parallel else None - self.stream_lock = threading.Lock() if parallel else None - self.progress_callback = progress_callback - self.progress_total = 0 - self.progress_lock = threading.Lock() if parallel else None - self.validate_content = validate_content - self.timeout = timeout - - def get_chunk_offsets(self): - index = 0 - if self.file_size is None: - # we don't know the size of the stream, so we have no - # choice but to seek - while True: - data = self._read_from_stream(index, 1) - if not data: - break - yield index - index += self.chunk_size - else: - while index < self.file_size: - yield index - index += self.chunk_size - - def process_chunk(self, chunk_offset): - size = self.chunk_size - if self.file_size is not None: - size = min(size, self.file_size - chunk_offset) - chunk_data = self._read_from_stream(chunk_offset, size) - return self._upload_chunk_with_progress(chunk_offset, chunk_data) - - def process_all_unknown_size(self): - assert self.stream_lock is None - range_ids = [] - index = 0 - while True: - data = self._read_from_stream(None, self.chunk_size) - if data: - index += len(data) - range_id = self._upload_chunk_with_progress(index, data) - range_ids.append(range_id) - else: - break - - return range_ids - - def _read_from_stream(self, offset, count): - if self.stream_lock is not None: - with self.stream_lock: - self.stream.seek(self.stream_start + offset) - data = self.stream.read(count) - else: - data = self.stream.read(count) - return data - - def _update_progress(self, length): - if self.progress_callback is not None: - if self.progress_lock is not None: - with self.progress_lock: - self.progress_total += length - total = self.progress_total - else: - self.progress_total += length - total = self.progress_total - self.progress_callback(total, self.file_size) - - def _upload_chunk_with_progress(self, chunk_start, chunk_data): - chunk_end = chunk_start + len(chunk_data) - 1 - self.file_service.update_range( - self.share_name, - self.directory_name, - self.file_name, - chunk_data, - chunk_start, - chunk_end, - self.validate_content, - timeout=self.timeout - ) - range_id = 'bytes={0}-{1}'.format(chunk_start, chunk_end) - self._update_progress(len(chunk_data)) - return range_id diff --git a/src/spring/azext_spring/azure_storage_file/fileservice.py b/src/spring/azext_spring/azure_storage_file/fileservice.py deleted file mode 100644 index 1e0789a81f5..00000000000 --- a/src/spring/azext_spring/azure_storage_file/fileservice.py +++ /dev/null @@ -1,2925 +0,0 @@ -# coding: utf-8 - -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -# pylint: disable=too-many-lines, import-error, redefined-builtin, protected-access, too-many-public-methods - -import sys -from datetime import datetime - -import math -from os import path - -from azure.common import AzureHttpError - -from azure.storage.common._auth import ( - _StorageSharedKeyAuthentication, - _StorageSASAuthentication, -) -from azure.storage.common._common_conversion import ( - _int_to_str, - _to_str, - _get_content_md5, -) -from azure.storage.common._connection import _ServiceParameters -from azure.storage.common._constants import ( - SERVICE_HOST_BASE, - DEFAULT_PROTOCOL, - DEV_ACCOUNT_NAME, -) -from azure.storage.common._deserialization import ( - _convert_xml_to_service_properties, - _convert_xml_to_signed_identifiers, - _parse_metadata, - _parse_properties, - _parse_length_from_content_range, -) -from azure.storage.common._error import ( - _dont_fail_not_exist, - _dont_fail_on_exist, - _validate_not_none, - _validate_type_bytes, - _ERROR_VALUE_NEGATIVE, - _ERROR_STORAGE_MISSING_INFO, - _ERROR_EMULATOR_DOES_NOT_SUPPORT_FILES, - _ERROR_PARALLEL_NOT_SEEKABLE, - _validate_access_policies, -) -from azure.storage.common._http import HTTPRequest -from azure.storage.common._serialization import ( - _get_request_body, - _get_data_bytes_only, - _convert_signed_identifiers_to_xml, - _convert_service_properties_to_xml, - _add_metadata_headers, -) -from azure.storage.common.models import ( - Services, - ListGenerator, - _OperationContext, -) -from azure.storage.common.storageclient import StorageClient -from .sharedaccesssignature import ( - FileSharedAccessSignature, -) -from ._deserialization import ( - _convert_xml_to_shares, - _convert_xml_to_directories_and_files, - _convert_xml_to_handles, - _parse_close_handle_response, - _convert_xml_to_ranges, - _convert_xml_to_share_stats, - _parse_file, - _parse_share, - _parse_snapshot_share, - _parse_directory, - _parse_permission_key, _parse_permission) -from ._download_chunking import _download_file_chunks -from ._serialization import ( - _get_path, - _validate_and_format_range_headers, - _validate_and_return_file_permission) -from ._upload_chunking import _upload_file_chunks -from .models import ( - FileProperties, - SMBProperties) - -from ._constants import ( - X_MS_VERSION, - __version__ as package_version, -) - -_SHARE_NOT_FOUND_ERROR_CODE = 'ShareNotFound' -_PARENT_NOT_FOUND_ERROR_CODE = 'ParentNotFound' -_RESOURCE_NOT_FOUND_ERROR_CODE = 'ResourceNotFound' -_RESOURCE_ALREADY_EXISTS_ERROR_CODE = 'ResourceAlreadyExists' -_SHARE_ALREADY_EXISTS_ERROR_CODE = 'ShareAlreadyExists' - -_GB = 1024 * 1024 * 1024 - -if sys.version_info >= (3,): - from io import BytesIO -else: - from cStringIO import StringIO as BytesIO - - -class FileService(StorageClient): - ''' - The Server Message Block (SMB) protocol is the preferred file share protocol - used on premise today. The Microsoft Azure File service enables customers to - leverage the availability and scalability of Azure's Cloud Infrastructure as - a Service (IaaS) SMB without having to rewrite SMB client applications. - - The Azure File service also offers a compelling alternative to traditional - Direct Attached Storage (DAS) and Storage Area Network (SAN) solutions, which - are often complex and expensive to install, configure, and operate. - - :ivar int MAX_SINGLE_GET_SIZE: - The size of the first range get performed by get_file_to_* methods if - max_connections is greater than 1. Less data will be returned if the - file is smaller than this. - :ivar int MAX_CHUNK_GET_SIZE: - The size of subsequent range gets performed by get_file_to_* methods if - max_connections is greater than 1 and the file is larger than MAX_SINGLE_GET_SIZE. - Less data will be returned if the remainder of the file is smaller than - this. If this is set to larger than 4MB, content_validation will throw an - error if enabled. However, if content_validation is not desired a size - greater than 4MB may be optimal. Setting this below 4MB is not recommended. - :ivar int MAX_RANGE_SIZE: - The size of the ranges put by create_file_from_* methods. Smaller ranges - may be put if there is less data provided. The maximum range size the service - supports is 4MB. - ''' - MAX_SINGLE_GET_SIZE = 32 * 1024 * 1024 - MAX_CHUNK_GET_SIZE = 8 * 1024 * 1024 - MAX_RANGE_SIZE = 4 * 1024 * 1024 - - def __init__(self, account_name=None, account_key=None, sas_token=None, - protocol=DEFAULT_PROTOCOL, endpoint_suffix=SERVICE_HOST_BASE, - request_session=None, connection_string=None, socket_timeout=None): - ''' - :param str account_name: - The storage account name. This is used to authenticate requests - signed with an account key and to construct the storage endpoint. It - is required unless a connection string is given. - :param str account_key: - The storage account key. This is used for shared key authentication. - :param str sas_token: - A shared access signature token to use to authenticate requests - instead of the account key. If account key and sas token are both - specified, account key will be used to sign. - :param str protocol: - The protocol to use for requests. Defaults to https. - :param str endpoint_suffix: - The host base component of the url, minus the account name. Defaults - to Azure (core.windows.net). Override this to use the China cloud - (core.chinacloudapi.cn). - :param requests.Session request_session: - The session object to use for http requests. - :param str connection_string: - If specified, this will override all other parameters besides - request session. See - http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ - for the connection string format. - :param int socket_timeout: - If specified, this will override the default socket timeout. The timeout specified is in seconds. - See DEFAULT_SOCKET_TIMEOUT in _constants.py for the default value. - ''' - service_params = _ServiceParameters.get_service_parameters( - 'file', - account_name=account_name, - account_key=account_key, - sas_token=sas_token, - protocol=protocol, - endpoint_suffix=endpoint_suffix, - request_session=request_session, - connection_string=connection_string, - socket_timeout=socket_timeout) - - super(FileService, self).__init__(service_params) - - if self.account_name == DEV_ACCOUNT_NAME: - raise ValueError(_ERROR_EMULATOR_DOES_NOT_SUPPORT_FILES) - - if self.account_key: - self.authentication = _StorageSharedKeyAuthentication( - self.account_name, - self.account_key, - ) - elif self.sas_token: - self.authentication = _StorageSASAuthentication(self.sas_token) - else: - raise ValueError(_ERROR_STORAGE_MISSING_INFO) - self._X_MS_VERSION = X_MS_VERSION - self._update_user_agent_string(package_version) - - def make_file_url(self, share_name, directory_name, file_name, - protocol=None, sas_token=None): - ''' - Creates the url to access a file. - - :param str share_name: - Name of share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of file. - :param str protocol: - Protocol to use: 'http' or 'https'. If not specified, uses the - protocol specified when FileService was initialized. - :param str sas_token: - Shared access signature token created with - generate_shared_access_signature. - :return: file access URL. - :rtype: str - ''' - - if directory_name is None: - url = '{}://{}/{}/{}'.format( - protocol or self.protocol, - self.primary_endpoint, - share_name, - file_name, - ) - else: - url = '{}://{}/{}/{}/{}'.format( - protocol or self.protocol, - self.primary_endpoint, - share_name, - directory_name, - file_name, - ) - - if sas_token: - url += (sas_token if sas_token.startswith('?') else '?' + sas_token) - - return url - - def generate_account_shared_access_signature(self, resource_types, permission, - expiry, start=None, ip=None, protocol=None): - ''' - Generates a shared access signature for the file service. - Use the returned signature with the sas_token parameter of the FileService. - - :param ResourceTypes resource_types: - Specifies the resource types that are accessible with the account SAS. - :param AccountPermissions permission: - The permissions associated with the shared access signature. The - user is restricted to operations allowed by the permissions. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has been - specified in an associated stored access policy. - :param expiry: - The time at which the shared access signature becomes invalid. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has - been specified in an associated stored access policy. Azure will always - convert values to UTC. If a date is passed in without timezone info, it - is assumed to be UTC. - :type expiry: datetime or str - :param start: - The time at which the shared access signature becomes valid. If - omitted, start time for this call is assumed to be the time when the - storage service receives the request. Azure will always convert values - to UTC. If a date is passed in without timezone info, it is assumed to - be UTC. - :type start: datetime or str - :param str ip: - Specifies an IP address or a range of IP addresses from which to accept requests. - If the IP address from which the request originates does not match the IP address - or address range specified on the SAS token, the request is not authenticated. - For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS - restricts the request to those IP addresses. - :param str protocol: - Specifies the protocol permitted for a request made. Possible values are - both HTTPS and HTTP (https,http) or HTTPS only (https). The default value - is https,http. Note that HTTP only is not a permitted value. - :return: A Shared Access Signature (sas) token. - :rtype: str - ''' - _validate_not_none('self.account_name', self.account_name) - _validate_not_none('self.account_key', self.account_key) - - sas = FileSharedAccessSignature(self.account_name, self.account_key) - return sas.generate_account(Services.FILE, resource_types, permission, - expiry, start=start, ip=ip, protocol=protocol) - - def generate_share_shared_access_signature(self, share_name, - permission=None, - expiry=None, - start=None, - id=None, - ip=None, - protocol=None, - cache_control=None, - content_disposition=None, - content_encoding=None, - content_language=None, - content_type=None): - ''' - Generates a shared access signature for the share. - Use the returned signature with the sas_token parameter of FileService. - - :param str share_name: - Name of share. - :param SharePermissions permission: - The permissions associated with the shared access signature. The - user is restricted to operations allowed by the permissions. - Permissions must be ordered read, create, write, delete, list. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has been - specified in an associated stored access policy. - :param expiry: - The time at which the shared access signature becomes invalid. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has - been specified in an associated stored access policy. Azure will always - convert values to UTC. If a date is passed in without timezone info, it - is assumed to be UTC. - :type expiry: datetime or str - :param start: - The time at which the shared access signature becomes valid. If - omitted, start time for this call is assumed to be the time when the - storage service receives the request. Azure will always convert values - to UTC. If a date is passed in without timezone info, it is assumed to - be UTC. - :type start: datetime or str - :param str id: - A unique value up to 64 characters in length that correlates to a - stored access policy. To create a stored access policy, use :func:`~set_share_acl`. - :param str ip: - Specifies an IP address or a range of IP addresses from which to accept requests. - If the IP address from which the request originates does not match the IP address - or address range specified on the SAS token, the request is not authenticated. - For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS - restricts the request to those IP addresses. - :param str protocol: - Specifies the protocol permitted for a request made. Possible values are - both HTTPS and HTTP (https,http) or HTTPS only (https). The default value - is https,http. Note that HTTP only is not a permitted value. - :param str cache_control: - Response header value for Cache-Control when resource is accessed - using this shared access signature. - :param str content_disposition: - Response header value for Content-Disposition when resource is accessed - using this shared access signature. - :param str content_encoding: - Response header value for Content-Encoding when resource is accessed - using this shared access signature. - :param str content_language: - Response header value for Content-Language when resource is accessed - using this shared access signature. - :param str content_type: - Response header value for Content-Type when resource is accessed - using this shared access signature. - :return: A Shared Access Signature (sas) token. - :rtype: str - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('self.account_name', self.account_name) - _validate_not_none('self.account_key', self.account_key) - - sas = FileSharedAccessSignature(self.account_name, self.account_key) - return sas.generate_share( - share_name, - permission, - expiry, - start=start, - id=id, - ip=ip, - protocol=protocol, - cache_control=cache_control, - content_disposition=content_disposition, - content_encoding=content_encoding, - content_language=content_language, - content_type=content_type, - ) - - def generate_file_shared_access_signature(self, share_name, - directory_name=None, - file_name=None, - permission=None, - expiry=None, - start=None, - id=None, - ip=None, - protocol=None, - cache_control=None, - content_disposition=None, - content_encoding=None, - content_language=None, - content_type=None): - ''' - Generates a shared access signature for the file. - Use the returned signature with the sas_token parameter of FileService. - - :param str share_name: - Name of share. - :param str directory_name: - Name of directory. SAS tokens cannot be created for directories, so - this parameter should only be present if file_name is provided. - :param str file_name: - Name of file. - :param FilePermissions permission: - The permissions associated with the shared access signature. The - user is restricted to operations allowed by the permissions. - Permissions must be ordered read, create, write, delete, list. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has been - specified in an associated stored access policy. - :param expiry: - The time at which the shared access signature becomes invalid. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has - been specified in an associated stored access policy. Azure will always - convert values to UTC. If a date is passed in without timezone info, it - is assumed to be UTC. - :type expiry: datetime or str - :param start: - The time at which the shared access signature becomes valid. If - omitted, start time for this call is assumed to be the time when the - storage service receives the request. Azure will always convert values - to UTC. If a date is passed in without timezone info, it is assumed to - be UTC. - :type start: datetime or str - :param str id: - A unique value up to 64 characters in length that correlates to a - stored access policy. To create a stored access policy, use - set_file_service_properties. - :param str ip: - Specifies an IP address or a range of IP addresses from which to accept requests. - If the IP address from which the request originates does not match the IP address - or address range specified on the SAS token, the request is not authenticated. - For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS - restricts the request to those IP addresses. - :param str protocol: - Specifies the protocol permitted for a request made. Possible values are - both HTTPS and HTTP (https,http) or HTTPS only (https). The default value - is https,http. Note that HTTP only is not a permitted value. - :param str cache_control: - Response header value for Cache-Control when resource is accessed - using this shared access signature. - :param str content_disposition: - Response header value for Content-Disposition when resource is accessed - using this shared access signature. - :param str content_encoding: - Response header value for Content-Encoding when resource is accessed - using this shared access signature. - :param str content_language: - Response header value for Content-Language when resource is accessed - using this shared access signature. - :param str content_type: - Response header value for Content-Type when resource is accessed - using this shared access signature. - :return: A Shared Access Signature (sas) token. - :rtype: str - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('self.account_name', self.account_name) - _validate_not_none('self.account_key', self.account_key) - - sas = FileSharedAccessSignature(self.account_name, self.account_key) - return sas.generate_file( - share_name, - directory_name, - file_name, - permission, - expiry, - start=start, - id=id, - ip=ip, - protocol=protocol, - cache_control=cache_control, - content_disposition=content_disposition, - content_encoding=content_encoding, - content_language=content_language, - content_type=content_type, - ) - - def set_file_service_properties(self, hour_metrics=None, minute_metrics=None, - cors=None, timeout=None): - ''' - Sets the properties of a storage account's File service, including - Azure Storage Analytics. If an element (ex HourMetrics) is left as None, the - existing settings on the service for that functionality are preserved. - - :param Metrics hour_metrics: - The hour metrics settings provide a summary of request - statistics grouped by API in hourly aggregates for files. - :param Metrics minute_metrics: - The minute metrics settings provide request statistics - for each minute for files. - :param cors: - You can include up to five CorsRule elements in the - list. If an empty list is specified, all CORS rules will be deleted, - and CORS will be disabled for the service. - :type cors: list(:class:`~azure.storage.common.models.CorsRule`) - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path() - request.query = { - 'restype': 'service', - 'comp': 'properties', - 'timeout': _int_to_str(timeout), - } - request.body = _get_request_body( - _convert_service_properties_to_xml(None, hour_metrics, minute_metrics, cors)) - - self._perform_request(request) - - def get_file_service_properties(self, timeout=None): - ''' - Gets the properties of a storage account's File service, including - Azure Storage Analytics. - - :param int timeout: - The timeout parameter is expressed in seconds. - :return: The file service properties. - :rtype: - :class:`~azure.storage.common.models.ServiceProperties` - ''' - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path() - request.query = { - 'restype': 'service', - 'comp': 'properties', - 'timeout': _int_to_str(timeout), - } - - return self._perform_request(request, _convert_xml_to_service_properties) - - def list_shares(self, prefix=None, marker=None, num_results=None, - include_metadata=False, timeout=None, include_snapshots=False): - ''' - Returns a generator to list the shares under the specified account. - The generator will lazily follow the continuation tokens returned by - the service and stop when all shares have been returned or num_results - is reached. - - If num_results is specified and the account has more than that number of - shares, the generator will have a populated next_marker field once it - finishes. This marker can be used to create a new generator if more - results are desired. - - :param str prefix: - Filters the results to return only shares whose names - begin with the specified prefix. - :param int num_results: - Specifies the maximum number of shares to return. - :param bool include_metadata: - Specifies that share metadata be returned in the response. - :param str marker: - An opaque continuation token. This value can be retrieved from the - next_marker field of a previous generator object if num_results was - specified and that generator has finished enumerating results. If - specified, this generator will begin returning results from the point - where the previous generator stopped. - :param int timeout: - The timeout parameter is expressed in seconds. - :param bool include_snapshots: - Specifies that share snapshots be returned in the response. - ''' - include = 'snapshots' if include_snapshots else None - if include_metadata: - if include is not None: - include = include + ',metadata' - else: - include = 'metadata' - operation_context = _OperationContext(location_lock=True) - kwargs = {'prefix': prefix, 'marker': marker, 'max_results': num_results, - 'include': include, 'timeout': timeout, '_context': operation_context} - resp = self._list_shares(**kwargs) - - return ListGenerator(resp, self._list_shares, (), kwargs) - - def _list_shares(self, prefix=None, marker=None, max_results=None, - include=None, timeout=None, _context=None): - ''' - Returns a list of the shares under the specified account. - - :param str prefix: - Filters the results to return only shares whose names - begin with the specified prefix. - :param str marker: - A string value that identifies the portion of the list - to be returned with the next list operation. The operation returns - a next_marker value within the response body if the list returned was - not complete. The marker value may then be used in a subsequent - call to request the next set of list items. The marker value is - opaque to the client. - :param int max_results: - Specifies the maximum number of shares to return. A single list - request may return up to 1000 shares and potentially a continuation - token which should be followed to get additional resutls. - :param string include: - Include this parameter to specify that either the share's - metadata, snapshots or both be returned as part of the response body. set this - parameter to string 'metadata' to get share's metadata. set this parameter to 'snapshots' - to get all the share snapshots. for both use 'snapshots,metadata'. - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path() - request.query = { - 'comp': 'list', - 'prefix': _to_str(prefix), - 'marker': _to_str(marker), - 'maxresults': _int_to_str(max_results), - 'include': _to_str(include), - 'timeout': _int_to_str(timeout), - } - - return self._perform_request(request, _convert_xml_to_shares, operation_context=_context) - - def create_share(self, share_name, metadata=None, quota=None, - fail_on_exist=False, timeout=None): - ''' - Creates a new share under the specified account. If the share - with the same name already exists, the operation fails on the - service. By default, the exception is swallowed by the client. - To expose the exception, specify True for fail_on_exists. - - :param str share_name: - Name of share to create. - :param metadata: - A dict with name_value pairs to associate with the - share as metadata. Example:{'Category':'test'} - :type metadata: dict(str, str) - :param int quota: - Specifies the maximum size of the share, in gigabytes. Must be - greater than 0, and less than or equal to 5TB (5120). - :param bool fail_on_exist: - Specify whether to throw an exception when the share exists. - False by default. - :param int timeout: - The timeout parameter is expressed in seconds. - :return: True if share is created, False if share already exists. - :rtype: bool - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'timeout': _int_to_str(timeout), - } - request.headers = { - 'x-ms-share-quota': _int_to_str(quota) - } - _add_metadata_headers(metadata, request) - - if not fail_on_exist: - try: - self._perform_request(request, expected_errors=[_SHARE_ALREADY_EXISTS_ERROR_CODE]) - return True - except AzureHttpError as ex: - _dont_fail_on_exist(ex) - return False - else: - self._perform_request(request) - return True - - def snapshot_share(self, share_name, metadata=None, quota=None, timeout=None): - ''' - Creates a snapshot of an existing share under the specified account. - - :param str share_name: - The name of the share to create a snapshot of. - :param metadata: - A dict with name_value pairs to associate with the - share as metadata. Example:{'Category':'test'} - :type metadata: a dict of str to str: - :param int quota: - Specifies the maximum size of the share, in gigabytes. Must be - greater than 0, and less than or equal to 5TB (5120). - :param int timeout: - The timeout parameter is expressed in seconds. - :return: snapshot properties - :rtype: azure.storage.file.models.Share - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'snapshot', - 'timeout': _int_to_str(timeout), - } - request.headers = { - 'x-ms-share-quota': _int_to_str(quota) - } - _add_metadata_headers(metadata, request) - - return self._perform_request(request, _parse_snapshot_share, [share_name]) - - def get_share_properties(self, share_name, timeout=None, snapshot=None): - ''' - Returns all user-defined metadata and system properties for the - specified share. The data returned does not include the shares's - list of files or directories. - - :param str share_name: - Name of existing share. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: A Share that exposes properties and metadata. - :rtype: :class:`~azure.storage.file.models.Share` - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot) - } - - return self._perform_request(request, _parse_share, [share_name]) - - def set_share_properties(self, share_name, quota, timeout=None): - ''' - Sets service-defined properties for the specified share. - - :param str share_name: - Name of existing share. - :param int quota: - Specifies the maximum size of the share, in gigabytes. Must be - greater than 0, and less than or equal to 5 TB (5120 GB). - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('quota', quota) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'properties', - 'timeout': _int_to_str(timeout), - } - request.headers = { - 'x-ms-share-quota': _int_to_str(quota) - } - - self._perform_request(request) - - def get_share_metadata(self, share_name, timeout=None, snapshot=None): - ''' - Returns all user-defined metadata for the specified share. - - :param str share_name: - Name of existing share. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: - A dictionary representing the share metadata name, value pairs. - :rtype: dict(str, str) - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'metadata', - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot), - } - - return self._perform_request(request, _parse_metadata) - - def set_share_metadata(self, share_name, metadata=None, timeout=None): - ''' - Sets one or more user-defined name-value pairs for the specified - share. Each call to this operation replaces all existing metadata - attached to the share. To remove all metadata from the share, - call this operation with no metadata dict. - - :param str share_name: - Name of existing share. - :param metadata: - A dict containing name-value pairs to associate with the share as - metadata. Example: {'category':'test'} - :type metadata: dict(str, str) - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'metadata', - 'timeout': _int_to_str(timeout), - } - _add_metadata_headers(metadata, request) - - self._perform_request(request) - - def get_share_acl(self, share_name, timeout=None): - ''' - Gets the permissions for the specified share. - - :param str share_name: - Name of existing share. - :param int timeout: - The timeout parameter is expressed in seconds. - :return: A dictionary of access policies associated with the share. - :rtype: dict(str, :class:`~azure.storage.common.models.AccessPolicy`) - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'acl', - 'timeout': _int_to_str(timeout), - } - - return self._perform_request(request, _convert_xml_to_signed_identifiers) - - def set_share_acl(self, share_name, signed_identifiers=None, timeout=None): - ''' - Sets the permissions for the specified share or stored access - policies that may be used with Shared Access Signatures. - - :param str share_name: - Name of existing share. - :param signed_identifiers: - A dictionary of access policies to associate with the share. The - dictionary may contain up to 5 elements. An empty dictionary - will clear the access policies set on the service. - :type signed_identifiers: dict(str, :class:`~azure.storage.common.models.AccessPolicy`) - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_access_policies(signed_identifiers) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'acl', - 'timeout': _int_to_str(timeout), - } - request.body = _get_request_body( - _convert_signed_identifiers_to_xml(signed_identifiers)) - - self._perform_request(request) - - def get_share_stats(self, share_name, timeout=None): - ''' - Gets the approximate size of the data stored on the share, - rounded up to the nearest gigabyte. - - Note that this value may not include all recently created - or recently re-sized files. - - :param str share_name: - Name of existing share. - :param int timeout: - The timeout parameter is expressed in seconds. - :return: the approximate size of the data stored on the share. - :rtype: int - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'stats', - 'timeout': _int_to_str(timeout), - } - - usage = self._perform_request(request, _convert_xml_to_share_stats) - return int(math.ceil(float(usage) / _GB)) - - def get_share_stats_in_bytes(self, share_name, timeout=None): - """ - Gets the approximate size of the data stored on the share in bytes. - - Note that this value may not include all recently created - or recently re-sized files. - - :param str share_name: - Name of existing share. - :param int timeout: - The timeout parameter is expressed in seconds. - :return: the approximate size of the data stored on the share. - :rtype: int - """ - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'stats', - 'timeout': _int_to_str(timeout), - } - - return self._perform_request(request, _convert_xml_to_share_stats) - - def delete_share(self, share_name, fail_not_exist=False, timeout=None, snapshot=None, delete_snapshots=None): - ''' - Marks the specified share for deletion. If the share - does not exist, the operation fails on the service. By - default, the exception is swallowed by the client. - To expose the exception, specify True for fail_not_exist. - - :param str share_name: - Name of share to delete. - :param bool fail_not_exist: - Specify whether to throw an exception when the share doesn't - exist. False by default. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - Specify this argument to delete a specific snapshot only. - delete_snapshots must be None if this is specified. - :param ~azure.storage.file.models.DeleteSnapshot delete_snapshots: - To delete a share that has snapshots, this must be specified as DeleteSnapshot.Include. - :return: True if share is deleted, False share doesn't exist. - :rtype: bool - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'DELETE' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.headers = { - 'x-ms-delete-snapshots': _to_str(delete_snapshots) - } - request.query = { - 'restype': 'share', - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot), - } - - if not fail_not_exist: - try: - self._perform_request(request, expected_errors=[_SHARE_NOT_FOUND_ERROR_CODE]) - return True - except AzureHttpError as ex: - _dont_fail_not_exist(ex) - return False - else: - self._perform_request(request) - return True - - def create_directory(self, share_name, directory_name, metadata=None, - fail_on_exist=False, timeout=None, file_permission=None, smb_properties=SMBProperties()): - ''' - Creates a new directory under the specified share or parent directory. - If the directory with the same name already exists, the operation fails - on the service. By default, the exception is swallowed by the client. - To expose the exception, specify True for fail_on_exists. - - :param str share_name: - Name of existing share. - :param str directory_name: - Name of directory to create, including the path to the parent - directory. - :param metadata: - A dict with name_value pairs to associate with the - share as metadata. Example:{'Category':'test'} - :type metadata: dict(str, str): - :param bool fail_on_exist: - specify whether to throw an exception when the directory exists. - False by default. - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. - :return: True if directory is created, False if directory already exists. - :rtype: bool - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('directory_name', directory_name) - current_time = datetime.utcnow() - if smb_properties.ntfs_attributes is None: - smb_properties.ntfs_attributes = 'Directory' - if smb_properties.creation_time is None: - smb_properties.creation_time = current_time - if smb_properties.last_write_time is None: - smb_properties.last_write_time = current_time - file_permission = _validate_and_return_file_permission(file_permission, - smb_properties.permission_key, - 'Inherit') - - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name) - request.query = { - 'restype': 'directory', - 'timeout': _int_to_str(timeout), - } - _add_metadata_headers(metadata, request) - request.headers.update({'x-ms-file-permission': file_permission}) - request.headers.update(smb_properties._to_request_headers()) - - if not fail_on_exist: - try: - self._perform_request( - request, expected_errors=_RESOURCE_ALREADY_EXISTS_ERROR_CODE) - return True - except AzureHttpError as ex: - _dont_fail_on_exist(ex) - return False - else: - self._perform_request(request) - return True - - def set_directory_properties(self, share_name, directory_name, file_permission=None, - smb_properties=SMBProperties(), timeout=None): - """ - - :param share_name: - Name of the share - :param directory_name: - Name of the directory - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. - """ - _validate_not_none('share_name', share_name) - _validate_not_none('directory_name', directory_name) - request = self._get_basic_set_file_or_directory_properties_http_request(share_name, directory_name, None, - file_permission, smb_properties, - timeout) - request.query.update({'restype': 'directory'}) - self._perform_request(request) - - def delete_directory(self, share_name, directory_name, - fail_not_exist=False, timeout=None): - ''' - Deletes the specified empty directory. Note that the directory must - be empty before it can be deleted. Attempting to delete directories - that are not empty will fail. - - If the directory does not exist, the operation fails on the - service. By default, the exception is swallowed by the client. - To expose the exception, specify True for fail_not_exist. - - :param str share_name: - Name of existing share. - :param str directory_name: - Name of directory to delete, including the path to the parent - directory. - :param bool fail_not_exist: - Specify whether to throw an exception when the directory doesn't - exist. - :param int timeout: - The timeout parameter is expressed in seconds. - :return: True if directory is deleted, False otherwise. - :rtype: bool - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('directory_name', directory_name) - request = HTTPRequest() - request.method = 'DELETE' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name) - request.query = { - 'restype': 'directory', - 'timeout': _int_to_str(timeout), - } - - if not fail_not_exist: - try: - self._perform_request(request, expected_errors=[_RESOURCE_NOT_FOUND_ERROR_CODE]) - return True - except AzureHttpError as ex: - _dont_fail_not_exist(ex) - return False - else: - self._perform_request(request) - return True - - def get_directory_properties(self, share_name, directory_name, timeout=None, snapshot=None): - ''' - Returns all user-defined metadata and system properties for the - specified directory. The data returned does not include the directory's - list of files. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to an existing directory. - :param int timeout: - The timeout parameter is expressed in seconds. - :return: properties for the specified directory within a directory object. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :rtype: :class:`~azure.storage.file.models.Directory` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('directory_name', directory_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name) - request.query = { - 'restype': 'directory', - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot) - } - - return self._perform_request(request, _parse_directory, [directory_name]) - - def get_directory_metadata(self, share_name, directory_name, timeout=None, snapshot=None): - ''' - Returns all user-defined metadata for the specified directory. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: - A dictionary representing the directory metadata name, value pairs. - :rtype: dict(str, str) - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('directory_name', directory_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name) - request.query = { - 'restype': 'directory', - 'comp': 'metadata', - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot) - } - - return self._perform_request(request, _parse_metadata) - - def set_directory_metadata(self, share_name, directory_name, metadata=None, timeout=None): - ''' - Sets one or more user-defined name-value pairs for the specified - directory. Each call to this operation replaces all existing metadata - attached to the directory. To remove all metadata from the directory, - call this operation with no metadata dict. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param metadata: - A dict containing name-value pairs to associate with the directory - as metadata. Example: {'category':'test'} - :type metadata: dict(str, str). - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('directory_name', directory_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name) - request.query = { - 'restype': 'directory', - 'comp': 'metadata', - 'timeout': _int_to_str(timeout), - } - _add_metadata_headers(metadata, request) - - self._perform_request(request) - - def list_directories_and_files(self, share_name, directory_name=None, - num_results=None, marker=None, timeout=None, - prefix=None, snapshot=None): - ''' - Returns a generator to list the directories and files under the specified share. - The generator will lazily follow the continuation tokens returned by - the service and stop when all directories and files have been returned or - num_results is reached. - - If num_results is specified and the share has more than that number of - files and directories, the generator will have a populated next_marker - field once it finishes. This marker can be used to create a new generator - if more results are desired. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param int num_results: - Specifies the maximum number of files to return, - including all directory elements. If the request does not specify - num_results or specifies a value greater than 5,000, the server will - return up to 5,000 items. Setting num_results to a value less than - or equal to zero results in error response code 400 (Bad Request). - :param str marker: - An opaque continuation token. This value can be retrieved from the - next_marker field of a previous generator object if num_results was - specified and that generator has finished enumerating results. If - specified, this generator will begin returning results from the point - where the previous generator stopped. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str prefix: - List only the files and/or directories with the given prefix. - :param str snapshot: - A string that represents the snapshot version, if applicable. - ''' - operation_context = _OperationContext(location_lock=True) - args = (share_name, directory_name) - kwargs = {'marker': marker, 'max_results': num_results, 'timeout': timeout, - '_context': operation_context, 'prefix': prefix, 'snapshot': snapshot} - - resp = self._list_directories_and_files(*args, **kwargs) - - return ListGenerator(resp, self._list_directories_and_files, args, kwargs) - - def _list_directories_and_files(self, share_name, directory_name=None, - marker=None, max_results=None, timeout=None, - prefix=None, _context=None, snapshot=None): - ''' - Returns a list of the directories and files under the specified share. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str marker: - A string value that identifies the portion of the list - to be returned with the next list operation. The operation returns - a next_marker value within the response body if the list returned was - not complete. The marker value may then be used in a subsequent - call to request the next set of list items. The marker value is - opaque to the client. - :param int max_results: - Specifies the maximum number of files to return, - including all directory elements. If the request does not specify - max_results or specifies a value greater than 5,000, the server will - return up to 5,000 items. Setting max_results to a value less than - or equal to zero results in error response code 400 (Bad Request). - :param int timeout: - The timeout parameter is expressed in seconds. - :param str prefix: - List only the files and/or directories with the given prefix. - :param str snapshot: - A string that represents the snapshot version, if applicable. - ''' - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name) - request.query = { - 'restype': 'directory', - 'comp': 'list', - 'prefix': _to_str(prefix), - 'marker': _to_str(marker), - 'maxresults': _int_to_str(max_results), - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot) - } - - return self._perform_request(request, _convert_xml_to_directories_and_files, - operation_context=_context) - - def list_handles(self, share_name, directory_name=None, file_name=None, recursive=None, - max_results=None, marker=None, snapshot=None, timeout=None): - """ - Returns a generator to list opened handles on a directory or a file under the specified share. - The generator will lazily follow the continuation tokens returned by - the service and stop when all handles have been returned or - num_results is reached. - - If num_results is specified and the share has more than that number of - files and directories, the generator will have a populated next_marker - field once it finishes. This marker can be used to create a new generator - if more results are desired. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param bool recursive: - Boolean that specifies if operation should apply to the directory specified in the URI, - its files, its subdirectories and their files. - :param int max_results: - Specifies the maximum number of handles taken on files and/or directories to return. - If the request does not specify max_results or specifies a value greater than 5,000, - the server will return up to 5,000 items. - Setting max_results to a value less than or equal to zero results in error response code 400 (Bad Request). - :param str marker: - An opaque continuation token. This value can be retrieved from the - next_marker field of a previous generator object if max_results was - specified and that generator has finished enumerating results. If - specified, this generator will begin returning results from the point - where the previous generator stopped. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :param int timeout: - The timeout parameter is expressed in seconds. - """ - operation_context = _OperationContext(location_lock=True) - args = (share_name, directory_name, file_name) - kwargs = {'marker': marker, 'max_results': max_results, 'timeout': timeout, 'recursive': recursive, - '_context': operation_context, 'snapshot': snapshot} - - resp = self._list_handles(*args, **kwargs) - - return ListGenerator(resp, self._list_handles, args, kwargs) - - def _list_handles(self, share_name, directory_name=None, file_name=None, recursive=None, - marker=None, max_results=None, timeout=None, _context=None, snapshot=None): - """ - Returns a list of the directories and files under the specified share. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param bool recursive: - Boolean that specifies if operation should apply to the directory specified in the URI, - its files, its subdirectories and their files. - :param str marker: - An opaque continuation token. This value can be retrieved from the - next_marker field of a previous generator object if max_results was - specified and that generator has finished enumerating results. If - specified, this generator will begin returning results from the point - where the previous generator stopped. - :param int max_results: - Specifies the maximum number of handles to return, - including all directory elements. If the request does not specify - max_results or specifies a value greater than 5,000, the server will - return up to 5,000 items. Setting max_results to a value less than - or equal to zero results in error response code 400 (Bad Request). - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - """ - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'listhandles', - 'marker': _to_str(marker), - 'maxresults': _int_to_str(max_results), - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot) - } - request.headers = { - 'x-ms-recursive': _to_str(recursive) - } - - return self._perform_request(request, _convert_xml_to_handles, - operation_context=_context) - - def close_handles(self, share_name, directory_name=None, file_name=None, recursive=None, - handle_id=None, marker=None, snapshot=None, timeout=None): - """ - Returns a generator to close opened handles on a directory or a file under the specified share. - The generator will lazily follow the continuation tokens returned by - the service and stop when all handles have been closed. - The yielded values represent the number of handles that were closed in each transaction. - - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param bool recursive: - Boolean that specifies if operation should apply to the directory specified in the URI, - its files, its subdirectories and their files. - :param str handle_id: - Required. Specifies handle ID opened on the file or directory to be closed. - Astrix (‘*’) is a wildcard that specifies all handles. - :param str marker: - An opaque continuation token. This value can be retrieved from the - next_marker field of a previous generator object if it has not finished closing handles. If - specified, this generator will begin closing handles from the point - where the previous generator stopped. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :param int timeout: - The timeout parameter is expressed in seconds. - """ - operation_context = _OperationContext(location_lock=True) - args = (share_name, directory_name, file_name) - kwargs = {'marker': marker, 'handle_id': handle_id, 'timeout': timeout, 'recursive': recursive, - '_context': operation_context, 'snapshot': snapshot} - - resp = self._close_handles(*args, **kwargs) - - return ListGenerator(resp, self._close_handles, args, kwargs) - - def _close_handles(self, share_name, directory_name=None, file_name=None, recursive=None, handle_id=None, - marker=None, timeout=None, _context=None, snapshot=None): - """ - Returns the number of handles that got closed. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param bool recursive: - Boolean that specifies if operation should apply to the directory specified in the URI, - its files, its subdirectories and their files. - :param str handle_id: - Required. Specifies handle ID opened on the file or directory to be closed. - Astrix (‘*’) is a wildcard that specifies all handles. - :param str marker: - Specifies the maximum number of handles taken on files and/or directories to return. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - """ - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'forceclosehandles', - 'marker': _to_str(marker), - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot) - } - request.headers = { - 'x-ms-recursive': _to_str(recursive), - 'x-ms-handle-id': _to_str(handle_id), - } - - return self._perform_request(request, _parse_close_handle_response, operation_context=_context) - - def get_file_properties(self, share_name, directory_name, file_name, timeout=None, snapshot=None): - ''' - Returns all user-defined metadata, standard HTTP properties, and - system properties for the file. Returns an instance of :class:`~azure.storage.file.models.File` with - :class:`~azure.storage.file.models.FileProperties` and a metadata dict. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: a file object including properties and metadata. - :rtype: :class:`~azure.storage.file.models.File` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'HEAD' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = {'timeout': _int_to_str( - timeout), 'sharesnapshot': _to_str(snapshot)} - - return self._perform_request(request, _parse_file, [file_name]) - - def exists(self, share_name, directory_name=None, file_name=None, timeout=None, snapshot=None): - ''' - Returns a boolean indicating whether the share exists if only share name is - given. If directory_name is specificed a boolean will be returned indicating - if the directory exists. If file_name is specified as well, a boolean will be - returned indicating if the file exists. - - :param str share_name: - Name of a share. - :param str directory_name: - The path to a directory. - :param str file_name: - Name of a file. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: A boolean indicating whether the resource exists. - :rtype: bool - ''' - _validate_not_none('share_name', share_name) - try: - request = HTTPRequest() - request.method = 'HEAD' if file_name is not None else 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - - if file_name is not None: - restype = None - expected_errors = [ - _RESOURCE_NOT_FOUND_ERROR_CODE, _PARENT_NOT_FOUND_ERROR_CODE] - elif directory_name is not None: - restype = 'directory' - expected_errors = [_RESOURCE_NOT_FOUND_ERROR_CODE, _SHARE_NOT_FOUND_ERROR_CODE, - _PARENT_NOT_FOUND_ERROR_CODE] - else: - restype = 'share' - expected_errors = [_SHARE_NOT_FOUND_ERROR_CODE] - - request.query = { - 'restype': restype, - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot) - } - self._perform_request(request, expected_errors=expected_errors) - return True - except AzureHttpError as ex: - _dont_fail_not_exist(ex) - return False - - def resize_file(self, share_name, directory_name, file_name, content_length, timeout=None): - ''' - Resizes a file to the specified size. If the specified byte - value is less than the current size of the file, then all - ranges above the specified byte value are cleared. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int content_length: - The length to resize the file to. - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('content_length', content_length) - request = self._get_basic_set_file_or_directory_properties_http_request(share_name, directory_name, file_name, - None, SMBProperties(), timeout) - request.headers.update( - {'x-ms-content-length': _to_str(content_length)}) - - self._perform_request(request) - - def set_file_properties(self, share_name, directory_name, file_name, - content_settings, timeout=None, file_permission=None, smb_properties=SMBProperties()): - ''' - Sets system properties on the file. If one property is set for the - content_settings, all properties will be overriden. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param ~azure.storage.file.models.ContentSettings content_settings: - ContentSettings object used to set the file properties. - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('content_settings', content_settings) - request = self._get_basic_set_file_or_directory_properties_http_request(share_name, directory_name, file_name, - file_permission, smb_properties, - timeout) - request.headers.update(content_settings._to_headers()) - - self._perform_request(request) - - def _get_basic_set_file_or_directory_properties_http_request(self, share_name, directory_name, file_name, - file_permission, smb_properties, timeout): - common_default_value = 'Preserve' - if smb_properties.ntfs_attributes is None: - smb_properties.ntfs_attributes = common_default_value - if smb_properties.creation_time is None: - smb_properties.creation_time = common_default_value - if smb_properties.last_write_time is None: - smb_properties.last_write_time = common_default_value - file_permission = _validate_and_return_file_permission(file_permission, - smb_properties.permission_key, - common_default_value) - - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'properties', - 'timeout': _int_to_str(timeout), - } - request.headers = {'x-ms-file-permission': file_permission} - request.headers.update(smb_properties._to_request_headers()) - - return request - - def get_file_metadata(self, share_name, directory_name, file_name, timeout=None, snapshot=None): - ''' - Returns all user-defined metadata for the specified file. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: - A dictionary representing the file metadata name, value pairs. - :rtype: dict(str, str) - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'metadata', - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot), - } - - return self._perform_request(request, _parse_metadata) - - def set_file_metadata(self, share_name, directory_name, - file_name, metadata=None, timeout=None): - ''' - Sets user-defined metadata for the specified file as one or more - name-value pairs. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param metadata: - Dict containing name and value pairs. Each call to this operation - replaces all existing metadata attached to the file. To remove all - metadata from the file, call this operation with no metadata headers. - :type metadata: dict(str, str) - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'metadata', - 'timeout': _int_to_str(timeout), - } - _add_metadata_headers(metadata, request) - - self._perform_request(request) - - def copy_file(self, share_name, directory_name, file_name, copy_source, - metadata=None, timeout=None): - ''' - Copies a file asynchronously. This operation returns a copy operation - properties object, including a copy ID you can use to check or abort the - copy operation. The File service copies files on a best-effort basis. - - If the destination file exists, it will be overwritten. The destination - file cannot be modified while the copy operation is in progress. - - :param str share_name: - Name of the destination share. The share must exist. - :param str directory_name: - Name of the destination directory. The directory must exist. - :param str file_name: - Name of the destination file. If the destination file exists, it will - be overwritten. Otherwise, it will be created. - :param str copy_source: - A URL of up to 2 KB in length that specifies an Azure file or blob. - The value should be URL-encoded as it would appear in a request URI. - If the source is in another account, the source must either be public - or must be authenticated via a shared access signature. If the source - is public, no authentication is required. - Examples: - https://myaccount.file.core.windows.net/myshare/mydir/myfile - https://otheraccount.file.core.windows.net/myshare/mydir/myfile?sastoken - :param metadata: - Name-value pairs associated with the file as metadata. If no name-value - pairs are specified, the operation will copy the metadata from the - source blob or file to the destination file. If one or more name-value - pairs are specified, the destination file is created with the specified - metadata, and the metadata is not copied from the source blob or file. - :type metadata: dict(str, str). - :param int timeout: - The timeout parameter is expressed in seconds. - :return: Copy operation properties such as status, source, and ID. - :rtype: :class:`~azure.storage.file.models.CopyProperties` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('copy_source', copy_source) - - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = {'timeout': _int_to_str(timeout)} - request.headers = { - 'x-ms-copy-source': _to_str(copy_source), - } - _add_metadata_headers(metadata, request) - - return self._perform_request(request, _parse_properties, [FileProperties]).copy - - def abort_copy_file(self, share_name, directory_name, file_name, copy_id, timeout=None): - ''' - Aborts a pending copy_file operation, and leaves a destination file - with zero length and full metadata. - - :param str share_name: - Name of destination share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of destination file. - :param str copy_id: - Copy identifier provided in the copy.id of the original - copy_file operation. - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('copy_id', copy_id) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'copy', - 'copyid': _to_str(copy_id), - 'timeout': _int_to_str(timeout), - } - request.headers = { - 'x-ms-copy-action': 'abort', - } - - self._perform_request(request) - - def delete_file(self, share_name, directory_name, file_name, timeout=None): - ''' - Marks the specified file for deletion. The file is later - deleted during garbage collection. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'DELETE' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = {'timeout': _int_to_str(timeout)} - - self._perform_request(request) - - def create_file(self, share_name, directory_name, file_name, - content_length, content_settings=None, metadata=None, timeout=None, - file_permission=None, smb_properties=SMBProperties()): - ''' - Creates a new file. - - See create_file_from_* for high level functions that handle the - creation and upload of large files with automatic chunking and - progress notifications. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of file to create or update. - :param int content_length: - Length of the file in bytes. - :param ~azure.storage.file.models.ContentSettings content_settings: - ContentSettings object used to set file properties. - :param metadata: - Name-value pairs associated with the file as metadata. - :type metadata: dict(str, str) - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('content_length', content_length) - current_time = datetime.utcnow() - if smb_properties.ntfs_attributes is None: - smb_properties.ntfs_attributes = 'Archive' - if smb_properties.creation_time is None: - smb_properties.creation_time = current_time - if smb_properties.last_write_time is None: - smb_properties.last_write_time = current_time - - file_permission = _validate_and_return_file_permission(file_permission, - smb_properties.permission_key, - 'Inherit') - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = {'timeout': _int_to_str(timeout)} - request.headers = { - 'x-ms-content-length': _to_str(content_length), - 'x-ms-type': 'file', - 'x-ms-file-permission': file_permission - } - _add_metadata_headers(metadata, request) - if content_settings is not None: - request.headers.update(content_settings._to_headers()) - request.headers.update(smb_properties._to_request_headers()) - - self._perform_request(request) - - def create_file_from_path(self, share_name, directory_name, file_name, - local_file_path, content_settings=None, - metadata=None, validate_content=False, progress_callback=None, - max_connections=2, file_permission=None, smb_properties=SMBProperties(), timeout=None): - ''' - Creates a new azure file from a local file path, or updates the content of an - existing file, with automatic chunking and progress notifications. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of file to create or update. - :param str local_file_path: - Path of the local file to upload as the file content. - :param ~azure.storage.file.models.ContentSettings content_settings: - ContentSettings object used for setting file properties. - :param metadata: - Name-value pairs associated with the file as metadata. - :type metadata: dict(str, str) - :param bool validate_content: - If true, calculates an MD5 hash for each range of the file. The storage - service checks the hash of the content that has arrived with the hash - that was sent. This is primarily valuable for detecting bitflips on - the wire if using http instead of https as https (the default) will - already validate. Note that this MD5 hash is not stored with the - file. - :param progress_callback: - Callback for progress with signature function(current, total) where - current is the number of bytes transfered so far and total is the - size of the file, or None if the total size is unknown. - :type progress_callback: func(current, total) - :param int max_connections: - Maximum number of parallel connections to use. - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('local_file_path', local_file_path) - - count = path.getsize(local_file_path) - with open(local_file_path, 'rb') as stream: - self.create_file_from_stream( - share_name, directory_name, file_name, stream, - count, content_settings, metadata, validate_content, progress_callback, - max_connections, file_permission=file_permission, smb_properties=smb_properties, timeout=timeout) - - def create_file_from_text(self, share_name, directory_name, file_name, - text, encoding='utf-8', content_settings=None, - metadata=None, validate_content=False, timeout=None, file_permission=None, - smb_properties=SMBProperties()): - ''' - Creates a new file from str/unicode, or updates the content of an - existing file, with automatic chunking and progress notifications. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of file to create or update. - :param str text: - Text to upload to the file. - :param str encoding: - Python encoding to use to convert the text to bytes. - :param ~azure.storage.file.models.ContentSettings content_settings: - ContentSettings object used to set file properties. - :param metadata: - Name-value pairs associated with the file as metadata. - :type metadata: dict(str, str) - :param bool validate_content: - If true, calculates an MD5 hash for each range of the file. The storage - service checks the hash of the content that has arrived with the hash - that was sent. This is primarily valuable for detecting bitflips on - the wire if using http instead of https as https (the default) will - already validate. Note that this MD5 hash is not stored with the - file. - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('text', text) - - if not isinstance(text, bytes): - _validate_not_none('encoding', encoding) - text = text.encode(encoding) - - self.create_file_from_bytes( - share_name, directory_name, file_name, text, count=len(text), - content_settings=content_settings, metadata=metadata, - validate_content=validate_content, file_permission=file_permission, smb_properties=smb_properties, - timeout=timeout) - - def create_file_from_bytes( - self, share_name, directory_name, file_name, file, - index=0, count=None, content_settings=None, metadata=None, - validate_content=False, progress_callback=None, max_connections=2, timeout=None, - file_permission=None, smb_properties=SMBProperties()): - ''' - Creates a new file from an array of bytes, or updates the content - of an existing file, with automatic chunking and progress - notifications. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of file to create or update. - :param str file: - Content of file as an array of bytes. - :param int index: - Start index in the array of bytes. - :param int count: - Number of bytes to upload. Set to None or negative value to upload - all bytes starting from index. - :param ~azure.storage.file.models.ContentSettings content_settings: - ContentSettings object used to set file properties. - :param metadata: - Name-value pairs associated with the file as metadata. - :type metadata: dict(str, str) - :param bool validate_content: - If true, calculates an MD5 hash for each range of the file. The storage - service checks the hash of the content that has arrived with the hash - that was sent. This is primarily valuable for detecting bitflips on - the wire if using http instead of https as https (the default) will - already validate. Note that this MD5 hash is not stored with the - file. - :param progress_callback: - Callback for progress with signature function(current, total) where - current is the number of bytes transfered so far and total is the - size of the file, or None if the total size is unknown. - :type progress_callback: func(current, total) - :param int max_connections: - Maximum number of parallel connections to use. - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('file', file) - _validate_type_bytes('file', file) - - if index < 0: - raise TypeError(_ERROR_VALUE_NEGATIVE.format('index')) - - if count is None or count < 0: - count = len(file) - index - - stream = BytesIO(file) - stream.seek(index) - - self.create_file_from_stream( - share_name, directory_name, file_name, stream, count, - content_settings, metadata, validate_content, progress_callback, - max_connections, file_permission=file_permission, smb_properties=smb_properties, timeout=timeout) - - def create_file_from_stream( - self, share_name, directory_name, file_name, stream, count, - content_settings=None, metadata=None, validate_content=False, - progress_callback=None, max_connections=2, timeout=None, - file_permission=None, smb_properties=SMBProperties()): - ''' - Creates a new file from a file/stream, or updates the content of an - existing file, with automatic chunking and progress notifications. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of file to create or update. - :param io.IOBase stream: - Opened file/stream to upload as the file content. - :param int count: - Number of bytes to read from the stream. This is required, a - file cannot be created if the count is unknown. - :param ~azure.storage.file.models.ContentSettings content_settings: - ContentSettings object used to set file properties. - :param metadata: - Name-value pairs associated with the file as metadata. - :type metadata: dict(str, str) - :param bool validate_content: - If true, calculates an MD5 hash for each range of the file. The storage - service checks the hash of the content that has arrived with the hash - that was sent. This is primarily valuable for detecting bitflips on - the wire if using http instead of https as https (the default) will - already validate. Note that this MD5 hash is not stored with the - file. - :param progress_callback: - Callback for progress with signature function(current, total) where - current is the number of bytes transfered so far and total is the - size of the file, or None if the total size is unknown. - :type progress_callback: func(current, total) - :param int max_connections: - Maximum number of parallel connections to use. Note that parallel upload - requires the stream to be seekable. - :param str file_permission: - File permission, a portable SDDL - :param ~azure.storage.file.models.SMBProperties smb_properties: - Sets the SMB related file properties - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('stream', stream) - _validate_not_none('count', count) - - if count < 0: - raise TypeError(_ERROR_VALUE_NEGATIVE.format('count')) - - self.create_file( - share_name, - directory_name, - file_name, - count, - content_settings, - metadata, - file_permission=file_permission, - smb_properties=smb_properties, - timeout=timeout - ) - - _upload_file_chunks( - self, - share_name, - directory_name, - file_name, - count, - self.MAX_RANGE_SIZE, - stream, - max_connections, - progress_callback, - validate_content, - timeout - ) - - def _get_file(self, share_name, directory_name, file_name, - start_range=None, end_range=None, validate_content=False, - timeout=None, _context=None, snapshot=None): - ''' - Downloads a file's content, metadata, and properties. You can specify a - range if you don't need to download the file in its entirety. If no range - is specified, the full file will be downloaded. - - See get_file_to_* for high level functions that handle the download - of large files with automatic chunking and progress notifications. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int start_range: - Start of byte range to use for downloading a section of the file. - If no end_range is given, all bytes after the start_range will be downloaded. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for downloading a section of the file. - If end_range is given, start_range must be provided. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param bool validate_content: - When this is set to True and specified together with the Range header, - the service returns the MD5 hash for the range, as long as the range - is less than or equal to 4 MB in size. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: A File with content, properties, and metadata. - :rtype: :class:`~azure.storage.file.models.File` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = {'timeout': _int_to_str( - timeout), 'sharesnapshot': _to_str(snapshot)} - _validate_and_format_range_headers( - request, - start_range, - end_range, - start_range_required=False, - end_range_required=False, - check_content_md5=validate_content) - - return self._perform_request(request, _parse_file, - [file_name, validate_content], - operation_context=_context) - - def get_file_to_path(self, share_name, directory_name, file_name, file_path, - open_mode='wb', start_range=None, end_range=None, - validate_content=False, progress_callback=None, - max_connections=2, timeout=None, snapshot=None): - ''' - Downloads a file to a file path, with automatic chunking and progress - notifications. Returns an instance of File with properties and metadata. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param str file_path: - Path of file to write to. - :param str open_mode: - Mode to use when opening the file. Note that specifying append only - open_mode prevents parallel download. So, max_connections must be set - to 1 if this open_mode is used. - :param int start_range: - Start of byte range to use for downloading a section of the file. - If no end_range is given, all bytes after the start_range will be downloaded. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for downloading a section of the file. - If end_range is given, start_range must be provided. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param bool validate_content: - If set to true, validates an MD5 hash for each retrieved portion of - the file. This is primarily valuable for detecting bitflips on the wire - if using http instead of https as https (the default) will already - validate. Note that the service will only return transactional MD5s - for chunks 4MB or less so the first get request will be of size - self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If - self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be - thrown. As computing the MD5 takes processing time and more requests - will need to be done due to the reduced chunk size there may be some - increase in latency. - :param progress_callback: - Callback for progress with signature function(current, total) - where current is the number of bytes transfered so far, and total is - the size of the file if known. - :type progress_callback: func(current, total) - :param int max_connections: - If set to 2 or greater, an initial get will be done for the first - self.MAX_SINGLE_GET_SIZE bytes of the file. If this is the entire file, - the method returns at this point. If it is not, it will download the - remaining data parallel using the number of threads equal to - max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. - If set to 1, a single large get request will be done. This is not - generally recommended but available if very few threads should be - used, network requests are very expensive, or a non-seekable stream - prevents parallel download. This may also be valuable if the file is - being concurrently modified to enforce atomicity or if many files are - expected to be empty as an extra request is required for empty files - if max_connections is greater than 1. - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: A File with properties and metadata. - :rtype: :class:`~azure.storage.file.models.File` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('file_path', file_path) - _validate_not_none('open_mode', open_mode) - - if max_connections > 1 and 'a' in open_mode: - raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) - - with open(file_path, open_mode) as stream: - file = self.get_file_to_stream( - share_name, directory_name, file_name, stream, - start_range, end_range, validate_content, - progress_callback, max_connections, timeout, snapshot) - - return file - - def get_file_to_stream( - self, share_name, directory_name, file_name, stream, - start_range=None, end_range=None, validate_content=False, - progress_callback=None, max_connections=2, timeout=None, snapshot=None): - ''' - Downloads a file to a stream, with automatic chunking and progress - notifications. Returns an instance of :class:`~azure.storage.file.models.File` with properties - and metadata. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param io.IOBase stream: - Opened file/stream to write to. - :param int start_range: - Start of byte range to use for downloading a section of the file. - If no end_range is given, all bytes after the start_range will be downloaded. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for downloading a section of the file. - If end_range is given, start_range must be provided. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param bool validate_content: - If set to true, validates an MD5 hash for each retrieved portion of - the file. This is primarily valuable for detecting bitflips on the wire - if using http instead of https as https (the default) will already - validate. Note that the service will only return transactional MD5s - for chunks 4MB or less so the first get request will be of size - self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If - self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be - thrown. As computing the MD5 takes processing time and more requests - will need to be done due to the reduced chunk size there may be some - increase in latency. - :param progress_callback: - Callback for progress with signature function(current, total) - where current is the number of bytes transfered so far, and total is - the size of the file if known. - :type progress_callback: func(current, total) - :param int max_connections: - If set to 2 or greater, an initial get will be done for the first - self.MAX_SINGLE_GET_SIZE bytes of the file. If this is the entire file, - the method returns at this point. If it is not, it will download the - remaining data parallel using the number of threads equal to - max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. - If set to 1, a single large get request will be done. This is not - generally recommended but available if very few threads should be - used, network requests are very expensive, or a non-seekable stream - prevents parallel download. This may also be valuable if the file is - being concurrently modified to enforce atomicity or if many files are - expected to be empty as an extra request is required for empty files - if max_connections is greater than 1. - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: A File with properties and metadata. - :rtype: :class:`~azure.storage.file.models.File` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('stream', stream) - - if end_range is not None: - _validate_not_none("start_range", start_range) - - # the stream must be seekable if parallel download is required - if max_connections > 1: - if sys.version_info >= (3,) and not stream.seekable(): - raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) - try: - stream.seek(stream.tell()) - except (NotImplementedError, AttributeError): - raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) - - # The service only provides transactional MD5s for chunks under 4MB. - # If validate_content is on, get only self.MAX_CHUNK_GET_SIZE for the first - # chunk so a transactional MD5 can be retrieved. - first_get_size = self.MAX_SINGLE_GET_SIZE if not validate_content else self.MAX_CHUNK_GET_SIZE - - initial_request_start = start_range if start_range is not None else 0 - - if end_range is not None and end_range - start_range < first_get_size: - initial_request_end = end_range - else: - initial_request_end = initial_request_start + first_get_size - 1 - - # Send a context object to make sure we always retry to the initial location - operation_context = _OperationContext(location_lock=True) - try: - file = self._get_file(share_name, - directory_name, - file_name, - start_range=initial_request_start, - end_range=initial_request_end, - validate_content=validate_content, - timeout=timeout, - _context=operation_context, - snapshot=snapshot) - - # Parse the total file size and adjust the download size if ranges - # were specified - file_size = _parse_length_from_content_range( - file.properties.content_range) - if end_range is not None: - # Use the end_range unless it is over the end of the file - download_size = min(file_size, end_range - start_range + 1) - elif start_range is not None: - download_size = file_size - start_range - else: - download_size = file_size - except AzureHttpError as ex: - if start_range is None and ex.status_code == 416: - # Get range will fail on an empty file. If the user did not - # request a range, do a regular get request in order to get - # any properties. - file = self._get_file(share_name, - directory_name, - file_name, - validate_content=validate_content, - timeout=timeout, - _context=operation_context, - snapshot=snapshot) - - # Set the download size to empty - download_size = 0 - else: - raise ex - - # Mark the first progress chunk. If the file is small, this is the only call - if progress_callback: - progress_callback(file.properties.content_length, download_size) - - # Write the content to the user stream - # Clear file content since output has been written to user stream - if file.content is not None: - stream.write(file.content) - file.content = None - - # If the file is small, the download is complete at this point. - # If file size is large, download the rest of the blob in chunks. - if file.properties.content_length != download_size: - # At this point we would like to lock on something like the etag so that - # if the file is modified, we do not get a corrupted download. However, - # this feature is not yet available on the file service. - - end_file = file_size - if end_range is not None: - # Use the end_range unless it is over the end of the file - end_file = min(file_size, end_range + 1) - - _download_file_chunks( - self, - share_name, - directory_name, - file_name, - download_size, - self.MAX_CHUNK_GET_SIZE, - first_get_size, - initial_request_end + 1, # start where the first download ended - end_file, - stream, - max_connections, - progress_callback, - validate_content, - timeout, - operation_context, - snapshot - ) - - # Set the content length to the download size instead of the size of - # the last range - file.properties.content_length = download_size - - # Overwrite the content range to the user requested range - file.properties.content_range = 'bytes {0}-{1}/{2}'.format( - start_range, end_range, file_size) - - # Overwrite the content MD5 as it is the MD5 for the last range instead - # of the stored MD5 - # TODO: Set to the stored MD5 when the service returns this - file.properties.content_md5 = None - - return file - - def get_file_to_bytes(self, share_name, directory_name, file_name, - start_range=None, end_range=None, validate_content=False, - progress_callback=None, max_connections=2, timeout=None, snapshot=None): - ''' - Downloads a file as an array of bytes, with automatic chunking and - progress notifications. Returns an instance of :class:`~azure.storage.file.models.File` with - properties, metadata, and content. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int start_range: - Start of byte range to use for downloading a section of the file. - If no end_range is given, all bytes after the start_range will be downloaded. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for downloading a section of the file. - If end_range is given, start_range must be provided. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param bool validate_content: - If set to true, validates an MD5 hash for each retrieved portion of - the file. This is primarily valuable for detecting bitflips on the wire - if using http instead of https as https (the default) will already - validate. Note that the service will only return transactional MD5s - for chunks 4MB or less so the first get request will be of size - self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If - self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be - thrown. As computing the MD5 takes processing time and more requests - will need to be done due to the reduced chunk size there may be some - increase in latency. - :param progress_callback: - Callback for progress with signature function(current, total) - where current is the number of bytes transfered so far, and total is - the size of the file if known. - :type progress_callback: func(current, total) - :param int max_connections: - If set to 2 or greater, an initial get will be done for the first - self.MAX_SINGLE_GET_SIZE bytes of the file. If this is the entire file, - the method returns at this point. If it is not, it will download the - remaining data parallel using the number of threads equal to - max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. - If set to 1, a single large get request will be done. This is not - generally recommended but available if very few threads should be - used, network requests are very expensive, or a non-seekable stream - prevents parallel download. This may also be valuable if the file is - being concurrently modified to enforce atomicity or if many files are - expected to be empty as an extra request is required for empty files - if max_connections is greater than 1. - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: A File with properties, content, and metadata. - :rtype: :class:`~azure.storage.file.models.File` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - - stream = BytesIO() - file = self.get_file_to_stream( - share_name, - directory_name, - file_name, - stream, - start_range, - end_range, - validate_content, - progress_callback, - max_connections, - timeout, - snapshot) - - file.content = stream.getvalue() - return file - - def get_file_to_text( - self, share_name, directory_name, file_name, encoding='utf-8', - start_range=None, end_range=None, validate_content=False, - progress_callback=None, max_connections=2, timeout=None, snapshot=None): - ''' - Downloads a file as unicode text, with automatic chunking and progress - notifications. Returns an instance of :class:`~azure.storage.file.models.File` with properties, - metadata, and content. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param str encoding: - Python encoding to use when decoding the file data. - :param int start_range: - Start of byte range to use for downloading a section of the file. - If no end_range is given, all bytes after the start_range will be downloaded. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for downloading a section of the file. - If end_range is given, start_range must be provided. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param bool validate_content: - If set to true, validates an MD5 hash for each retrieved portion of - the file. This is primarily valuable for detecting bitflips on the wire - if using http instead of https as https (the default) will already - validate. Note that the service will only return transactional MD5s - for chunks 4MB or less so the first get request will be of size - self.MAX_CHUNK_GET_SIZE instead of self.MAX_SINGLE_GET_SIZE. If - self.MAX_CHUNK_GET_SIZE was set to greater than 4MB an error will be - thrown. As computing the MD5 takes processing time and more requests - will need to be done due to the reduced chunk size there may be some - increase in latency. - :param progress_callback: - Callback for progress with signature function(current, total) - where current is the number of bytes transfered so far, and total is - the size of the file if known. - :type progress_callback: func(current, total) - :param int max_connections: - If set to 2 or greater, an initial get will be done for the first - self.MAX_SINGLE_GET_SIZE bytes of the file. If this is the entire file, - the method returns at this point. If it is not, it will download the - remaining data parallel using the number of threads equal to - max_connections. Each chunk will be of size self.MAX_CHUNK_GET_SIZE. - If set to 1, a single large get request will be done. This is not - generally recommended but available if very few threads should be - used, network requests are very expensive, or a non-seekable stream - prevents parallel download. This may also be valuable if the file is - being concurrently modified to enforce atomicity or if many files are - expected to be empty as an extra request is required for empty files - if max_connections is greater than 1. - :param int timeout: - The timeout parameter is expressed in seconds. This method may make - multiple calls to the Azure service and the timeout will apply to - each call individually. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :return: A File with properties, content, and metadata. - :rtype: :class:`~azure.storage.file.models.File` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - _validate_not_none('encoding', encoding) - - file = self.get_file_to_bytes( - share_name, - directory_name, - file_name, - start_range, - end_range, - validate_content, - progress_callback, - max_connections, - timeout, - snapshot) - - file.content = file.content.decode(encoding) - return file - - def update_range(self, share_name, directory_name, file_name, data, - start_range, end_range, validate_content=False, timeout=None): - ''' - Writes the bytes specified by the request body into the specified range. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param bytes data: - Content of the range. - :param int start_range: - Start of byte range to use for updating a section of the file. - The range can be up to 4 MB in size. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for updating a section of the file. - The range can be up to 4 MB in size. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param bool validate_content: - If true, calculates an MD5 hash of the page content. The storage - service checks the hash of the content that has arrived - with the hash that was sent. This is primarily valuable for detecting - bitflips on the wire if using http instead of https as https (the default) - will already validate. Note that this MD5 hash is not stored with the - file. - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - request = self._get_basic_update_file_http_request( - share_name, directory_name, file_name, timeout=timeout) - - _validate_not_none('data', data) - _validate_and_format_range_headers(request, start_range, end_range) - request.body = _get_data_bytes_only('data', data) - - if validate_content: - computed_md5 = _get_content_md5(request.body) - request.headers['Content-MD5'] = _to_str(computed_md5) - - self._perform_request(request) - - def update_range_from_file_url(self, share_name, directory_name, file_name, start_range, end_range, source, - source_start_range, timeout=None): - ''' - Writes the bytes from one Azure File endpoint into the specified range of another Azure File endpoint. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int start_range: - Start of byte range to use for updating a section of the file. - The range can be up to 4 MB in size. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for updating a section of the file. - The range can be up to 4 MB in size. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param str source: - A URL of up to 2 KB in length that specifies an Azure file or blob. - The value should be URL-encoded as it would appear in a request URI. - If the source is in another account, the source must either be public - or must be authenticated via a shared access signature. If the source - is public, no authentication is required. - Examples: - https://myaccount.file.core.windows.net/myshare/mydir/myfile - https://otheraccount.file.core.windows.net/myshare/mydir/myfile?sastoken - :param int source_start_range: - Start of byte range to use for updating a section of the file. - The range can be up to 4 MB in size. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - - request = self._get_basic_update_file_http_request( - share_name, directory_name, file_name, timeout=timeout) - - _validate_not_none('source', source) - _validate_and_format_range_headers(request, start_range, end_range) - _validate_and_format_range_headers(request, - source_start_range, - source_start_range + - (end_range - start_range), - is_source=True) - - request.headers.update({ - 'x-ms-copy-source': _to_str(source), - 'Content-Length': _int_to_str(0) - }) - - self._perform_request(request) - - def _get_basic_update_file_http_request(self, share_name, directory_name, file_name, timeout=None): - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'range', - 'timeout': _int_to_str(timeout), - } - request.headers = { - 'x-ms-write': 'update', - } - - return request - - def clear_range(self, share_name, directory_name, file_name, start_range, - end_range, timeout=None): - ''' - Clears the specified range and releases the space used in storage for - that range. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int start_range: - Start of byte range to use for clearing a section of the file. - The range can be up to 4 MB in size. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - End of byte range to use for clearing a section of the file. - The range can be up to 4 MB in size. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int timeout: - The timeout parameter is expressed in seconds. - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'range', - 'timeout': _int_to_str(timeout), - } - request.headers = { - 'Content-Length': '0', - 'x-ms-write': 'clear', - } - _validate_and_format_range_headers( - request, start_range, end_range) - - self._perform_request(request) - - def list_ranges(self, share_name, directory_name, file_name, - start_range=None, end_range=None, timeout=None, snapshot=None): - ''' - Retrieves the valid ranges for a file. - - :param str share_name: - Name of existing share. - :param str directory_name: - The path to the directory. - :param str file_name: - Name of existing file. - :param int start_range: - Specifies the start offset of bytes over which to list ranges. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int end_range: - Specifies the end offset of bytes over which to list ranges. - The start_range and end_range params are inclusive. - Ex: start_range=0, end_range=511 will download first 512 bytes of file. - :param int timeout: - The timeout parameter is expressed in seconds. - :param str snapshot: - A string that represents the snapshot version, if applicable. - :returns: a list of valid ranges - :rtype: a list of :class:`~azure.storage.file.models.FileRange` - ''' - _validate_not_none('share_name', share_name) - _validate_not_none('file_name', file_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name, directory_name, file_name) - request.query = { - 'comp': 'rangelist', - 'timeout': _int_to_str(timeout), - 'sharesnapshot': _to_str(snapshot), - } - if start_range is not None: - _validate_and_format_range_headers( - request, - start_range, - end_range, - start_range_required=False, - end_range_required=False) - - return self._perform_request(request, _convert_xml_to_ranges) - - def create_permission_for_share(self, share_name, file_permission, timeout=None): - """ - Create a permission(a security descriptor) at the share level. - This 'permission' can be used for the files/directories in the share. - If a 'permission' already exists, it shall return the key of it, else - creates a new permission at the share level and return its key. - - :param share_name: - Name of share. - :param file_permission: - File permission, a Portable SDDL - :param timeout: - The timeout parameter is expressed in seconds. - :returns a file permission key - :rtype str - """ - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'PUT' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'filepermission', - 'timeout': _int_to_str(timeout), - } - request.body = file_permission - return self._perform_request(request, parser=_parse_permission_key) - - def get_permission_for_share(self, share_name, file_permission_key, timeout=None): - """ - Create a permission(a security descriptor) at the share level. - This 'permission' can be used for the files/directories in the share. - If a 'permission' already exists, it shall return the key of it, else - creates a new permission at the share level and return its key. - - :param share_name: - Name of share. - :param file_permission_key: - Key of the file permission to retrieve - :param timeout: - The timeout parameter is expressed in seconds. - :returns a file permission(a portable SDDL) - :rtype str - """ - _validate_not_none('share_name', share_name) - request = HTTPRequest() - request.method = 'GET' - request.host_locations = self._get_host_locations() - request.path = _get_path(share_name) - request.query = { - 'restype': 'share', - 'comp': 'filepermission', - 'timeout': _int_to_str(timeout), - } - request.headers = { - 'x-ms-file-permission-key': file_permission_key - } - request.body = None - - return self._perform_request(request, parser=_parse_permission) diff --git a/src/spring/azext_spring/azure_storage_file/models.py b/src/spring/azext_spring/azure_storage_file/models.py deleted file mode 100644 index a53904b2605..00000000000 --- a/src/spring/azext_spring/azure_storage_file/models.py +++ /dev/null @@ -1,600 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -# pylint: disable=too-few-public-methods, too-many-instance-attributes - -from azure.storage.common._common_conversion import _to_str - - -class Share(object): - ''' - File share class. - - :ivar str name: - The name of the share. - :ivar ShareProperties properties: - System properties for the share. - :ivar metadata: - A dict containing name-value pairs associated with the share as metadata. - This var is set to None unless the include=metadata param was included - for the list shares operation. If this parameter was specified but the - share has no metadata, metadata will be set to an empty dictionary. - :vartype metadata: dict(str, str) - :ivar str snapshot: - A DateTime value that uniquely identifies the snapshot. The value of - this header indicates the snapshot version, and may be used in - subsequent requests to access the snapshot. - ''' - - def __init__(self, name=None, props=None, metadata=None, snapshot=None): - self.name = name - self.properties = props or ShareProperties() - self.metadata = metadata - self.snapshot = snapshot - - -class ShareProperties(object): - ''' - File share's properties class. - - :ivar datetime last_modified: - A datetime object representing the last time the share was modified. - :ivar str etag: - The ETag contains a value that you can use to perform operations - conditionally. - :ivar int quote: - Returns the current share quota in GB. - ''' - - def __init__(self): - self.last_modified = None - self.etag = None - self.quota = None - - -class Directory(object): - ''' - Directory class. - - :ivar str name: - The name of the directory. - :ivar DirectoryProperties properties: - System properties for the directory. - :ivar metadata: - A dict containing name-value pairs associated with the directory as metadata. - This var is set to None unless the include=metadata param was included - for the list directory operation. If this parameter was specified but the - directory has no metadata, metadata will be set to an empty dictionary. - :vartype metadata: dict(str, str) - ''' - - def __init__(self, name=None, props=None, metadata=None): - self.name = name - self.properties = props or DirectoryProperties() - self.metadata = metadata - - -class DirectoryProperties(object): - ''' - File directory's properties class. - - :ivar datetime last_modified: - A datetime object representing the last time the directory was modified. - :ivar str etag: - The ETag contains a value that you can use to perform operations - conditionally. - :ivar bool server_encrypted: - Set to true if the directory metadata is encrypted on the server. - :ivar ~azure.storage.file.models.SMBProperties smb_properties: - SMB related file properties - ''' - - def __init__(self): - self.last_modified = None - self.etag = None - self.server_encrypted = None - self.smb_properties = SMBProperties() - - -class File(object): - ''' - File class. - - :ivar str name: - The name of the file. - :ivar content: - File content. - :vartype content: str or bytes - :ivar FileProperties properties: - System properties for the file. - :ivar metadata: - A dict containing name-value pairs associated with the file as metadata. - This var is set to None unless the include=metadata param was included - for the list file operation. If this parameter was specified but the - file has no metadata, metadata will be set to an empty dictionary. - :vartype metadata: dict(str, str) - ''' - - def __init__(self, name=None, content=None, props=None, metadata=None): - self.name = name - self.content = content - self.properties = props or FileProperties() - self.metadata = metadata - - -class FileProperties(object): - ''' - File Properties. - - :ivar datetime last_modified: - A datetime object representing the last time the file was modified. - :ivar str etag: - The ETag contains a value that you can use to perform operations - conditionally. - :ivar int content_length: - The length of the content returned. If the entire blob was requested, - the length of blob in bytes. If a subset of the blob was requested, the - length of the returned subset. - :ivar str content_range: - Indicates the range of bytes returned in the event that the client - requested a subset of the blob. - :ivar ~azure.storage.file.models.ContentSettings content_settings: - Stores all the content settings for the file. - :ivar ~azure.storage.file.models.CopyProperties copy: - Stores all the copy properties for the file. - :ivar bool server_encrypted: - Set to true if the file data and application metadata are completely encrypted. - :ivar ~azure.storage.file.models.SMBProperties smb_properties: - SMB related file properties - :ivar ~azure.storage.file.models.LeaseProperties lease: - Stores all the lease information for the file. - ''' - - def __init__(self): - self.last_modified = None - self.etag = None - self.content_length = None - self.content_range = None - self.content_settings = ContentSettings() - self.copy = CopyProperties() - self.server_encrypted = None - self.smb_properties = SMBProperties() - self.lease = LeaseProperties() - - -class SMBProperties(object): - """ - SMB related properties to get/set for for file/directory - - :ivar str or :class:`~azure.storage.file.models.NTFSAttributes` ntfs_attributes: - The file system attributes for files and directories. - If not set, indicates preservation of existing values. - Here is an example for when the var type is str: 'Temporary|Archive' - :ivar str or datetime creation_time: - When the File or Directory was created. - If it is a string type, time should have 7 decimal digits, eg. '2019-07-07T02:52:46.5540162Z' - :ivar str or datetime last_write_time: - When the File or Directory was last modified. eg. '2019-07-07T02:52:46.5540162Z' - If it is a string type, time should have 7 decimal digits, eg. '2019-07-07T02:52:46.5540162Z' - :ivar str permission_key: - The file's File Permission Key - :ivar str change_time: - When the File was last changed. This is what will be returned by service. Users don't need to specify. - :ivar str file_id: - The Id of this directory. This is what will be returned by service. Users don't need to specify. - :ivar str parent_id: - The Id of this directory's parent. This is what will be returned by service. Users don't need to specify. - """ - - def __init__(self, ntfs_attributes=None, creation_time=None, last_write_time=None, permission_key=None): - self.ntfs_attributes = ntfs_attributes - self.creation_time = creation_time - self.last_write_time = last_write_time - self.permission_key = permission_key - self.change_time = None - self.file_id = None - self.parent_id = None - - def _to_request_headers(self): - creation_time = self.creation_time if isinstance(self.creation_time, str) \ - else self.creation_time.isoformat() + '0Z' - last_write_time = self.last_write_time if isinstance(self.last_write_time, str) \ - else self.last_write_time.isoformat() + '0Z' - return { - 'x-ms-file-attributes': _to_str(self.ntfs_attributes), - 'x-ms-file-creation-time': creation_time, - 'x-ms-file-last-write-time': last_write_time, - 'x-ms-file-permission-key': _to_str(self.permission_key) - } - - -class LeaseProperties(object): - ''' - File Lease Properties. - - :ivar str status: - The lease status of the file. - Possible values: locked|unlocked - :ivar str state: - Lease state of the file. - Possible values: available|leased|expired|breaking|broken - :ivar str duration: - When a file is leased, specifies whether the lease is of infinite or fixed duration. - ''' - - def __init__(self): - self.status = None - self.state = None - self.duration = None - - -class Handle(object): - """ - Represents a file handle. - - :ivar str handle_id: - Used to identify handle. - :ivar str path: - Used to identify the name of the object for which the handle is open. - :ivar str file_id: - Uniquely identifies the file. - This is useful when renames are happening as the file ID does not change. - :ivar str parent_id: - Uniquely identifies the parent directory. - This is useful when renames are happening as the parent ID does not change. - :ivar str session_id: - Session ID in context of which the file handle was opened. - :ivar str client_ip: - Used to identify client that has opened the handle. - The field is included only if client IP is known by the service. - :ivar datetime open_time: - Used to decide if handle may have been leaked. - :ivar datetime last_reconnect_time: - Used to decide if handle was reopened after client/server disconnect due to networking or other faults. - The field is included only if disconnect event occurred and handle was reopened. - """ - - def __init__(self, handle_id=None, path=None, file_id=None, parent_id=None, session_id=None, - client_ip=None, open_time=None, last_reconnect_time=None): - self.handle_id = handle_id - self.path = path - self.file_id = file_id - self.parent_id = parent_id - self.session_id = session_id - self.client_ip = client_ip - self.open_time = open_time - self.last_reconnect_time = last_reconnect_time - - -class ContentSettings(object): - ''' - Used to store the content settings of a file. - - :ivar str content_type: - The content type specified for the file. If no content type was - specified, the default content type is application/octet-stream. - :ivar str content_encoding: - If content_encoding has previously been set - for the file, that value is stored. - :ivar str content_language: - If content_language has previously been set - for the file, that value is stored. - :ivar str content_disposition: - content_disposition conveys additional information about how to - process the response payload, and also can be used to attach - additional metadata. If content_disposition has previously been set - for the file, that value is stored. - :ivar str cache_control: - If cache_control has previously been set for - the file, that value is stored. - :ivar str content_md5: - If the content_md5 has been set for the file, this response - header is stored so that the client can check for message content - integrity. - ''' - - def __init__( - self, content_type=None, content_encoding=None, - content_language=None, content_disposition=None, - cache_control=None, content_md5=None): - self.content_type = content_type - self.content_encoding = content_encoding - self.content_language = content_language - self.content_disposition = content_disposition - self.cache_control = cache_control - self.content_md5 = content_md5 - - def _to_headers(self): - return { - 'x-ms-cache-control': _to_str(self.cache_control), - 'x-ms-content-type': _to_str(self.content_type), - 'x-ms-content-disposition': _to_str(self.content_disposition), - 'x-ms-content-md5': _to_str(self.content_md5), - 'x-ms-content-encoding': _to_str(self.content_encoding), - 'x-ms-content-language': _to_str(self.content_language), - } - - -class CopyProperties(object): - ''' - File Copy Properties. - - :ivar str id: - String identifier for the last attempted Copy File operation where this file - was the destination file. This header does not appear if this file has never - been the destination in a Copy File operation, or if this file has been - modified after a concluded Copy File operation using Set File Properties or - Put File. - :ivar str source: - URL up to 2 KB in length that specifies the source file used in the last attempted - Copy File operation where this file was the destination file. This header does not - appear if this file has never been the destination in a Copy File operation, or if - this file has been modified after a concluded Copy File operation using - Set File Properties or Put File. - :ivar str status: - State of the copy operation identified by Copy ID, with these values: - success: - Copy completed successfully. - pending: - Copy is in progress. Check copy_status_description if intermittent, - non-fatal errors impede copy progress but don't cause failure. - aborted: - Copy was ended by Abort Copy File. - failed: - Copy failed. See copy_status_description for failure details. - :ivar str progress: - Contains the number of bytes copied and the total bytes in the source in the last - attempted Copy File operation where this file was the destination file. Can show - between 0 and Content-Length bytes copied. - :ivar datetime completion_time: - Conclusion time of the last attempted Copy File operation where this file was the - destination file. This value can specify the time of a completed, aborted, or - failed copy attempt. - :ivar str status_description: - Only appears when x-ms-copy-status is failed or pending. Describes cause of fatal - or non-fatal copy operation failure. - ''' - - def __init__(self): - self.id = None - self.source = None - self.status = None - self.progress = None - self.completion_time = None - self.status_description = None - - -class FileRange(object): - ''' - File Range. - - :ivar int start: - Byte index for start of file range. - :ivar int end: - Byte index for end of file range. - ''' - - def __init__(self, start=None, end=None): - self.start = start - self.end = end - - -class DeleteSnapshot(object): - ''' - Required if the Share has associated snapshots. Specifies how to handle the snapshots. - ''' - - Include = 'include' - ''' - Delete the share and all of its snapshots. - ''' - - -class FilePermissions(object): - ''' - FilePermissions class to be used with - :func:`~azure.storage.file.fileservice.FileService.generate_file_shared_access_signature` API. - - :ivar FilePermissions FilePermissions.CREATE: - Create a new file or copy a file to a new file. - :ivar FilePermissions FilePermissions.DELETE: - Delete the file. - :ivar FilePermissions FilePermissions.READ: - Read the content, properties, metadata. Use the file as the source of a copy - operation. - :ivar FilePermissions FilePermissions.WRITE: - Create or write content, properties, metadata. Resize the file. Use the file - as the destination of a copy operation within the same account. - ''' - - def __init__(self, read=False, create=False, write=False, delete=False, - _str=None): - ''' - :param bool read: - Read the content, properties, metadata. Use the file as the source of a copy - operation. - :param bool create: - Create a new file or copy a file to a new file. - :param bool write: - Create or write content, properties, metadata. Resize the file. Use the file - as the destination of a copy operation within the same account. - :param bool delete: - Delete the file. - :param str _str: - A string representing the permissions. - ''' - - if not _str: - _str = '' - self.read = read or ('r' in _str) - self.create = create or ('c' in _str) - self.write = write or ('w' in _str) - self.delete = delete or ('d' in _str) - - def __or__(self, other): - return FilePermissions(_str=str(self) + str(other)) - - def __add__(self, other): - return FilePermissions(_str=str(self) + str(other)) - - def __str__(self): - return (('r' if self.read else '') + - ('c' if self.create else '') + - ('w' if self.write else '') + - ('d' if self.delete else '')) - - -FilePermissions.CREATE = FilePermissions(create=True) -FilePermissions.DELETE = FilePermissions(delete=True) -FilePermissions.READ = FilePermissions(read=True) -FilePermissions.WRITE = FilePermissions(write=True) - - -class SharePermissions(object): - ''' - SharePermissions class to be used with `azure.storage.file.FileService.generate_share_shared_access_signature` - method and for the AccessPolicies used with `azure.storage.file.FileService.set_share_acl`. - - :ivar SharePermissions FilePermissions.DELETE: - Delete any file in the share. - Note: You cannot grant permissions to delete a share with a service SAS. Use - an account SAS instead. - :ivar SharePermissions FilePermissions.LIST: - List files and directories in the share. - :ivar SharePermissions FilePermissions.READ: - Read the content, properties or metadata of any file in the share. Use any - file in the share as the source of a copy operation. - :ivar SharePermissions FilePermissions.WRITE: - For any file in the share, create or write content, properties or metadata. - Resize the file. Use the file as the destination of a copy operation within - the same account. - Note: You cannot grant permissions to read or write share properties or - metadata with a service SAS. Use an account SAS instead. - ''' - - # pylint: disable=redefined-builtin - def __init__(self, read=False, write=False, delete=False, list=False, - _str=None): - ''' - :param bool read: - Read the content, properties or metadata of any file in the share. Use any - file in the share as the source of a copy operation. - :param bool write: - For any file in the share, create or write content, properties or metadata. - Resize the file. Use the file as the destination of a copy operation within - the same account. - Note: You cannot grant permissions to read or write share properties or - metadata with a service SAS. Use an account SAS instead. - :param bool delete: - Delete any file in the share. - Note: You cannot grant permissions to delete a share with a service SAS. Use - an account SAS instead. - :param bool list: - List files and directories in the share. - :param str _str: - A string representing the permissions - ''' - - if not _str: - _str = '' - self.read = read or ('r' in _str) - self.write = write or ('w' in _str) - self.delete = delete or ('d' in _str) - self.list = list or ('l' in _str) - - def __or__(self, other): - return SharePermissions(_str=str(self) + str(other)) - - def __add__(self, other): - return SharePermissions(_str=str(self) + str(other)) - - def __str__(self): - return (('r' if self.read else '') + - ('w' if self.write else '') + - ('d' if self.delete else '') + - ('l' if self.list else '')) - - -SharePermissions.DELETE = SharePermissions(delete=True) -SharePermissions.LIST = SharePermissions(list=True) -SharePermissions.READ = SharePermissions(read=True) -SharePermissions.WRITE = SharePermissions(write=True) - - -class NTFSAttributes(object): - """ - Valid set of attributes to set for file or directory. - To set attribute for directory, 'Directory' should always be enabled except setting 'None' for directory. - - :ivar bool read_only: - Enable/disable 'ReadOnly' attribute for DIRECTORY or FILE - :ivar bool hidden: - Enable/disable 'Hidden' attribute for DIRECTORY or FILE - :ivar bool system: - Enable/disable 'System' attribute for DIRECTORY or FILE - :ivar bool none: - Enable/disable 'None' attribute for DIRECTORY or FILE to clear all attributes of FILE/DIRECTORY - :ivar bool directory: - Enable/disable 'Directory' attribute for DIRECTORY - :ivar bool archive: - Enable/disable 'Archive' attribute for DIRECTORY or FILE - :ivar bool temporary: - Enable/disable 'Temporary' attribute for FILE - :ivar bool offline: - Enable/disable 'Offline' attribute for DIRECTORY or FILE - :ivar bool not_content_indexed: - Enable/disable 'NotContentIndexed' attribute for DIRECTORY or FILE - :ivar bool no_scrub_data: - Enable/disable 'NoScrubData' attribute for DIRECTORY or FILE - """ - - def __init__(self, read_only=False, hidden=False, system=False, none=False, directory=False, archive=False, - temporary=False, offline=False, not_content_indexed=False, no_scrub_data=False, _str=None): - if not _str: - _str = '' - self.read_only = read_only or ('ReadOnly' in _str) - self.hidden = hidden or ('Hidden' in _str) - self.system = system or ('System' in _str) - self.none = none or ('None' in _str) - self.directory = directory or ('Directory' in _str) - self.archive = archive or ('Archive' in _str) - self.temporary = temporary or ('Temporary' in _str) - self.offline = offline or ('Offline' in _str) - self.not_content_indexed = not_content_indexed or ( - 'NotContentIndexed' in _str) - self.no_scrub_data = no_scrub_data or ('NoScrubData' in _str) - - def __or__(self, other): - return NTFSAttributes(_str=str(self) + str(other)) - - def __add__(self, other): - return NTFSAttributes(_str=str(self) + str(other)) - - def __str__(self): - concatenated_params = (('ReadOnly|' if self.read_only else '') + - ('Hidden|' if self.hidden else '') + - ('System|' if self.system else '') + - ('None|' if self.none else '') + - ('Directory|' if self.directory else '') + - ('Archive|' if self.archive else '') + - ('Temporary|' if self.temporary else '') + - ('Offline|' if self.offline else '') + - ('NotContentIndexed|' if self.not_content_indexed else '') + - ('NoScrubData|' if self.no_scrub_data else '')) - - return concatenated_params.strip('|') - - -NTFSAttributes.READ_ONLY = NTFSAttributes(read_only=True) -NTFSAttributes.HIDDEN = NTFSAttributes(hidden=True) -NTFSAttributes.SYSTEM = NTFSAttributes(system=True) -NTFSAttributes.NONE = NTFSAttributes(none=True) -NTFSAttributes.DIRECTORY = NTFSAttributes(directory=True) -NTFSAttributes.ARCHIVE = NTFSAttributes(archive=True) -NTFSAttributes.TEMPORARY = NTFSAttributes(temporary=True) -NTFSAttributes.OFFLINE = NTFSAttributes(offline=True) -NTFSAttributes.NOT_CONTENT_INDEXED = NTFSAttributes(not_content_indexed=True) -NTFSAttributes.NO_SCRUB_DATA = NTFSAttributes(no_scrub_data=True) diff --git a/src/spring/azext_spring/azure_storage_file/sharedaccesssignature.py b/src/spring/azext_spring/azure_storage_file/sharedaccesssignature.py deleted file mode 100644 index 67aaa7d191a..00000000000 --- a/src/spring/azext_spring/azure_storage_file/sharedaccesssignature.py +++ /dev/null @@ -1,237 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from azure.storage.common.sharedaccesssignature import ( - SharedAccessSignature, - _SharedAccessHelper, - _QueryStringConstants, - _sign_string, -) -from azure.storage.common._common_conversion import ( - _to_str, -) -from ._constants import X_MS_VERSION - - -class FileSharedAccessSignature(SharedAccessSignature): - ''' - Provides a factory for creating file and share access - signature tokens with a common account name and account key. Users can either - use the factory or can construct the appropriate service and use the - generate_*_shared_access_signature method directly. - ''' - - def __init__(self, account_name, account_key): - ''' - :param str account_name: - The storage account name used to generate the shared access signatures. - :param str account_key: - The access key to generate the shares access signatures. - ''' - super(FileSharedAccessSignature, self).__init__( - account_name, account_key, x_ms_version=X_MS_VERSION) - - # pylint: disable=redefined-builtin - def generate_file(self, share_name, directory_name=None, file_name=None, - permission=None, expiry=None, start=None, id=None, - ip=None, protocol=None, cache_control=None, - content_disposition=None, content_encoding=None, - content_language=None, content_type=None): - ''' - Generates a shared access signature for the file. - Use the returned signature with the sas_token parameter of FileService. - - :param str share_name: - Name of share. - :param str directory_name: - Name of directory. SAS tokens cannot be created for directories, so - this parameter should only be present if file_name is provided. - :param str file_name: - Name of file. - :param FilePermissions permission: - The permissions associated with the shared access signature. The - user is restricted to operations allowed by the permissions. - Permissions must be ordered read, create, write, delete, list. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has been - specified in an associated stored access policy. - :param expiry: - The time at which the shared access signature becomes invalid. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has - been specified in an associated stored access policy. Azure will always - convert values to UTC. If a date is passed in without timezone info, it - is assumed to be UTC. - :type expiry: datetime or str - :param start: - The time at which the shared access signature becomes valid. If - omitted, start time for this call is assumed to be the time when the - storage service receives the request. Azure will always convert values - to UTC. If a date is passed in without timezone info, it is assumed to - be UTC. - :type start: datetime or str - :param str id: - A unique value up to 64 characters in length that correlates to a - stored access policy. To create a stored access policy, use - set_file_service_properties. - :param str ip: - Specifies an IP address or a range of IP addresses from which to accept requests. - If the IP address from which the request originates does not match the IP address - or address range specified on the SAS token, the request is not authenticated. - For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS - restricts the request to those IP addresses. - :param str protocol: - Specifies the protocol permitted for a request made. The default value - is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. - :param str cache_control: - Response header value for Cache-Control when resource is accessed - using this shared access signature. - :param str content_disposition: - Response header value for Content-Disposition when resource is accessed - using this shared access signature. - :param str content_encoding: - Response header value for Content-Encoding when resource is accessed - using this shared access signature. - :param str content_language: - Response header value for Content-Language when resource is accessed - using this shared access signature. - :param str content_type: - Response header value for Content-Type when resource is accessed - using this shared access signature. - ''' - resource_path = share_name - if directory_name is not None: - resource_path += '/' + _to_str(directory_name) - resource_path += '/' + _to_str(file_name) - - sas = _FileSharedAccessHelper() - sas.add_base(permission, expiry, start, ip, - protocol, self.x_ms_version) - sas.add_id(id) - sas.add_resource('f') - sas.add_override_response_headers(cache_control, content_disposition, - content_encoding, content_language, - content_type) - sas.add_resource_signature( - self.account_name, self.account_key, resource_path) - - return sas.get_token() - - # pylint: disable=redefined-builtin - def generate_share(self, share_name, permission=None, expiry=None, - start=None, id=None, ip=None, protocol=None, - cache_control=None, content_disposition=None, - content_encoding=None, content_language=None, - content_type=None): - ''' - Generates a shared access signature for the share. - Use the returned signature with the sas_token parameter of FileService. - - :param str share_name: - Name of share. - :param SharePermissions permission: - The permissions associated with the shared access signature. The - user is restricted to operations allowed by the permissions. - Permissions must be ordered read, create, write, delete, list. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has been - specified in an associated stored access policy. - :param expiry: - The time at which the shared access signature becomes invalid. - Required unless an id is given referencing a stored access policy - which contains this field. This field must be omitted if it has - been specified in an associated stored access policy. Azure will always - convert values to UTC. If a date is passed in without timezone info, it - is assumed to be UTC. - :type expiry: datetime or str - :param start: - The time at which the shared access signature becomes valid. If - omitted, start time for this call is assumed to be the time when the - storage service receives the request. Azure will always convert values - to UTC. If a date is passed in without timezone info, it is assumed to - be UTC. - :type start: datetime or str - :param str id: - A unique value up to 64 characters in length that correlates to a - stored access policy. To create a stored access policy, use - set_file_service_properties. - :param str ip: - Specifies an IP address or a range of IP addresses from which to accept requests. - If the IP address from which the request originates does not match the IP address - or address range specified on the SAS token, the request is not authenticated. - For example, specifying sip=168.1.5.65 or sip=168.1.5.60-168.1.5.70 on the SAS - restricts the request to those IP addresses. - :param str protocol: - Specifies the protocol permitted for a request made. The default value - is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. - :param str cache_control: - Response header value for Cache-Control when resource is accessed - using this shared access signature. - :param str content_disposition: - Response header value for Content-Disposition when resource is accessed - using this shared access signature. - :param str content_encoding: - Response header value for Content-Encoding when resource is accessed - using this shared access signature. - :param str content_language: - Response header value for Content-Language when resource is accessed - using this shared access signature. - :param str content_type: - Response header value for Content-Type when resource is accessed - using this shared access signature. - ''' - sas = _FileSharedAccessHelper() - sas.add_base(permission, expiry, start, ip, - protocol, self.x_ms_version) - sas.add_id(id) - sas.add_resource('s') - sas.add_override_response_headers(cache_control, content_disposition, - content_encoding, content_language, - content_type) - sas.add_resource_signature( - self.account_name, self.account_key, share_name) - - return sas.get_token() - - -# pylint: disable=useless-super-delegation, too-few-public-methods -class _FileSharedAccessHelper(_SharedAccessHelper): - def __init__(self): - super(_FileSharedAccessHelper, self).__init__() - - def add_resource_signature(self, account_name, account_key, path): # pylint: disable=arguments-differ - def get_value_to_append(query): - return_value = self.query_dict.get(query) or '' - return return_value + '\n' - - if path[0] != '/': - path = '/' + path - - canonicalized_resource = '/file/' + account_name + path + '\n' - - # Form the string to sign from shared_access_policy and canonicalized - # resource. The order of values is important. - string_to_sign = \ - (get_value_to_append(_QueryStringConstants.SIGNED_PERMISSION) + - get_value_to_append(_QueryStringConstants.SIGNED_START) + - get_value_to_append(_QueryStringConstants.SIGNED_EXPIRY) + - canonicalized_resource + - get_value_to_append(_QueryStringConstants.SIGNED_IDENTIFIER) + - get_value_to_append(_QueryStringConstants.SIGNED_IP) + - get_value_to_append(_QueryStringConstants.SIGNED_PROTOCOL) + - get_value_to_append(_QueryStringConstants.SIGNED_VERSION) + - get_value_to_append(_QueryStringConstants.SIGNED_CACHE_CONTROL) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_DISPOSITION) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_ENCODING) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_LANGUAGE) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_TYPE)) - - # remove the trailing newline - if string_to_sign[-1] == '\n': - string_to_sign = string_to_sign[:-1] - - self._add_query(_QueryStringConstants.SIGNED_SIGNATURE, - _sign_string(account_key, string_to_sign)) diff --git a/src/spring/azext_spring/custom.py b/src/spring/azext_spring/custom.py index 28682f5cf3a..bca0a30a249 100644 --- a/src/spring/azext_spring/custom.py +++ b/src/spring/azext_spring/custom.py @@ -188,7 +188,7 @@ def _update_application_insights_asc_update(cmd, resource_group, name, location, def spring_delete(cmd, client, resource_group, name, no_wait=False): - logger.warning("Stop using Azure Spring Apps? We appreciate your feedback: https://aka.ms/springclouddeletesurvey") + logger.warning("Stop using Azure Spring Apps? We appreciate your feedback: https://aka.ms/asa_exitsurvey") return sdk_no_wait(no_wait, client.services.begin_delete, resource_group_name=resource_group, service_name=name)