From d9f869084071ed707364a0e7eaf525b25821eb11 Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 6 Feb 2023 12:46:13 -0600 Subject: [PATCH] feat: add compound file ordinal endpoints update --- sdk/diffgram/file/compound_file.py | 122 ++++++++++++++++++++++---- sdk/diffgram/file/file_constructor.py | 37 +++++--- 2 files changed, 131 insertions(+), 28 deletions(-) diff --git a/sdk/diffgram/file/compound_file.py b/sdk/diffgram/file/compound_file.py index 7bd64c7..716b22e 100644 --- a/sdk/diffgram/file/compound_file.py +++ b/sdk/diffgram/file/compound_file.py @@ -6,6 +6,7 @@ class CompoundChildFile: + id: int path: str url: str blob_path: str @@ -24,8 +25,10 @@ class CompoundChildFile: video_split_duration: int local_id: str ordinal: int + root_file: 'CompoundFile' def __init__(self, + root_file: 'CompoundFile', child_file_type: str, path: str = None, url: str = None, @@ -42,7 +45,9 @@ def __init__(self, assume_new_instances_machine_made: bool = None, convert_names_to_label_files: bool = None, video_split_duration: int = None, - ordinal: int = 0): + ordinal: int = 0, + id: int = None): + self.root_file = root_file self.child_file_type = child_file_type self.path = path self.url = url @@ -61,6 +66,38 @@ def __init__(self, self.video_split_duration = video_split_duration self.local_id = str(uuid4()) self.ordinal = ordinal + self.id = None + + def __str__(self): + return f'' + def refresh_from_data_dict(self, data: dict): + if not data: + return + for key in data: + setattr(self, key, data[key]) + + def update(self): + """ + Syncs child file data with backend API + :return: + """ + payload = { + 'ordinal': self.ordinal, + } + client = self.root_file.project + endpoint = f"/api/v1/project/{self.root_file.project.project_string_id}/file/{self.id}/update-metadata" + + response = client.session.put( + client.host + endpoint, + json = payload) + + client.handle_errors(response) + + data = response.json() + new_file_data = data['file'] + + self.refresh_from_data_dict(data = new_file_data) + return self def set_ordinal(self, value: int): self.ordinal = value @@ -69,13 +106,47 @@ def set_ordinal(self, value: int): class CompoundFile: project: Project parent_file_data: dict - child_files_to_upload: List[CompoundChildFile] + child_files: List[CompoundChildFile] def __init__(self, project: Project, name: str, directory_id: int): self.project = project self.name = name self.directory_id = directory_id - self.child_files_to_upload = [] + self.child_files = [] + + @staticmethod + def from_dict(project: Project, dir_id: int, dict_data: dict): + result = CompoundFile(project = project, name = dict_data.get('original_filename'), directory_id = dir_id) + result.__refresh_compound_file_from_data_dict(data = dict_data) + child_files = result.__fetch_child_files() + result.child_files = child_files + return result + + def __fetch_child_files(self) -> List[CompoundChildFile]: + client = self.project + endpoint = f"/api/v1/project/{self.project.project_string_id}/file/{self.id}/child-files" + + response = client.session.get(client.host + endpoint) + + client.handle_errors(response) + + data = response.json() + child_files_data = data['child_files'] + result = [] + for elm in child_files_data: + child_file = CompoundChildFile(root_file = self, child_file_type = elm.get('type')) + child_file.refresh_from_data_dict(data = elm) + result.append(child_file) + return result + + def update_all(self) -> bool: + """ + Syncs parent and child metadata with backend API. + :return: True/False + """ + for child in self.child_files: + child.update() + return True def __refresh_compound_file_from_data_dict(self, data: dict): if not data: @@ -83,9 +154,9 @@ def __refresh_compound_file_from_data_dict(self, data: dict): for key in data: setattr(self, key, data[key]) - def remove_compound_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]: - self.child_files_to_upload.remove(child_file) - return self.child_files_to_upload + def remove_child_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]: + self.child_files.remove(child_file) + return self.child_files def __create_compound_parent_file(self): url = f'/api/v1/project/{self.project.project_string_id}/file/new-compound' @@ -110,7 +181,8 @@ def __create_child_file(self, child_file: CompoundChildFile): frame_packet_map = child_file.frame_packet_map, assume_new_instances_machine_made = child_file.assume_new_instances_machine_made, convert_names_to_label_files = child_file.convert_names_to_label_files, - parent_file_id = self.parent_file_data.get('id') + parent_file_id = self.parent_file_data.get('id'), + ordinal = child_file.ordinal, ) elif child_file.child_file_type == 'from_url': return self.project.file.from_url( @@ -121,7 +193,8 @@ def __create_child_file(self, child_file: CompoundChildFile): video_split_duration = child_file.video_split_duration, instance_list = child_file.instance_list, frame_packet_map = child_file.frame_packet_map, - parent_file_id = self.parent_file_data.get('id') + parent_file_id = self.parent_file_data.get('id'), + ordinal = child_file.ordinal, ) elif child_file.child_file_type == 'from_blob_path': return self.project.file.from_blob_path( @@ -132,7 +205,8 @@ def __create_child_file(self, child_file: CompoundChildFile): instance_list = child_file.instance_list, file_name = child_file.file_name, frame_packet_map = child_file.frame_packet_map, - parent_file_id = self.parent_file_data.get('id') + parent_file_id = self.parent_file_data.get('id'), + ordinal = child_file.ordinal, ) def add_child_from_local(self, @@ -141,17 +215,21 @@ def add_child_from_local(self, frame_packet_map: dict = None, assume_new_instances_machine_made: bool = True, convert_names_to_label_files: bool = True, - ordinal: int = 0): + ordinal: int = None): + if ordinal is None: + ordinal = len(self.child_files) new_child_file = CompoundChildFile( + root_file = self, child_file_type = "from_local", path = path, directory_id = self.directory_id, instance_list = instance_list, frame_packet_map = frame_packet_map, assume_new_instances_machine_made = assume_new_instances_machine_made, - convert_names_to_label_files = convert_names_to_label_files + convert_names_to_label_files = convert_names_to_label_files, + ordinal = ordinal ) - self.child_files_to_upload.append(new_child_file) + self.child_files.append(new_child_file) return new_child_file def add_child_file_from_url(self, @@ -162,8 +240,11 @@ def add_child_file_from_url(self, video_split_duration: int = None, instance_list: list = None, frame_packet_map: dict = None, - ordinal: int = 0): + ordinal: int = None): + if ordinal is None: + ordinal = len(self.child_files) new_child_file = CompoundChildFile( + root_file = self, child_file_type = "from_url", url = url, media_type = media_type, @@ -173,8 +254,9 @@ def add_child_file_from_url(self, video_split_duration = video_split_duration, instance_list = instance_list, frame_packet_map = frame_packet_map, + ordinal = ordinal ) - self.child_files_to_upload.append(new_child_file) + self.child_files.append(new_child_file) return new_child_file def add_child_from_blob_path(self, @@ -185,9 +267,12 @@ def add_child_from_blob_path(self, instance_list: list = None, file_name: str = None, frame_packet_map: dict = None, - ordinal: int = 0 + ordinal: int = None ): + if ordinal is None: + ordinal = len(self.child_files) new_child_file = CompoundChildFile( + root_file = self, child_file_type = "from_blob_path", blob_path = blob_path, bucket_name = bucket_name, @@ -197,15 +282,16 @@ def add_child_from_blob_path(self, instance_list = instance_list, file_name = file_name, frame_packet_map = frame_packet_map, + ordinal = ordinal ) - self.child_files_to_upload.append(new_child_file) + self.child_files.append(new_child_file) return new_child_file def upload(self): - if len(self.child_files_to_upload) == 0: + if len(self.child_files) == 0: raise AssertionError('Need to add at least one child file to the compound file before calling upload()') parent_file_data: dict = self.__create_compound_parent_file() - for child_file in self.child_files_to_upload: + for child_file in self.child_files: self.__create_child_file(child_file) return parent_file_data diff --git a/sdk/diffgram/file/file_constructor.py b/sdk/diffgram/file/file_constructor.py index 6096483..1b14f33 100644 --- a/sdk/diffgram/file/file_constructor.py +++ b/sdk/diffgram/file/file_constructor.py @@ -44,7 +44,8 @@ def from_local( frame_packet_map: dict = None, assume_new_instances_machine_made: bool = True, convert_names_to_label_files: bool = True, - parent_file_id: int = None + parent_file_id: int = None, + ordinal: int = 0 ): """ Create a Project file from local path @@ -56,7 +57,9 @@ def from_local( files = {'file': (os.path.basename(path), open(path, 'rb'), 'application/octet-stream')} - json_payload = {} + json_payload = { + 'ordinal': ordinal + } if directory_id is None: directory_id = self.client.directory_id @@ -107,13 +110,15 @@ def __build_packet_payload(self, bucket_name: str = None, file_name: str = None, blob_path: str = None, - parent_file_id: int = None): + parent_file_id: int = None, + ordinal: int = 0): packet = {'media': {}} packet['media']['url'] = url packet['media']['type'] = media_type # Existing Instances packet['frame_packet_map'] = frame_packet_map + packet['ordinal'] = ordinal packet['type'] = type packet['connection_id'] = connection_id packet['parent_file_id'] = parent_file_id @@ -140,7 +145,8 @@ def from_blob_path(self, instance_list: list = None, file_name: str = None, frame_packet_map: dict = None, - parent_file_id: int = None): + parent_file_id: int = None, + ordinal: int = 0): """ Bind a blob path in the given connection ID into Diffgram :param blob_path: @@ -167,7 +173,8 @@ def from_blob_path(self, file_name = name, directory_id = directory_id, type = "from_blob_path", - parent_file_id=parent_file_id + parent_file_id=parent_file_id, + ordinal = ordinal ) self.from_packet(packet = packet) return True @@ -181,7 +188,8 @@ def from_url( video_split_duration: int = None, instance_list: list = None, # for Images frame_packet_map: dict = None, # for Video - parent_file_id: int = None + parent_file_id: int = None, + ordinal: int = 0 ): """ @@ -214,7 +222,8 @@ def from_url( video_split_duration = video_split_duration, instance_list = instance_list, frame_packet_map = frame_packet_map, - parent_file_id = parent_file_id + parent_file_id = parent_file_id, + ordinal = ordinal ) self.from_packet(packet = packet) @@ -554,7 +563,15 @@ def get_by_id(self, response_json = response.json() file_data = response_json.get(file_response_key) + if file_data.get('type') == 'compound': + from diffgram.file.compound_file import CompoundFile + return CompoundFile.from_dict( + project = self.client, + dir_id = self.client.directory_id, + dict_data = file_data + ) - return File.new( - client = self.client, - file_json = file_data) + else: + return File.new( + client = self.client, + file_json = file_data)