From 87682417b28e9c5059b86e98880bb2b1bcb97936 Mon Sep 17 00:00:00 2001 From: Martin Dobias Date: Thu, 15 Apr 2021 16:11:32 +0200 Subject: [PATCH 1/2] Fix order of how diffs are applied to basefile The client used version items from "history" from project info in the order as the server returns them... which is likely sorted, but using lexical sorting, so in some cases it would fail - e.g. when updating from v4 to v11 the order of diffs was v10, v11, v5, v6, v7, v8, v9. This makes sure that the versions are always correctly ordered. Fixes lutraconsulting/qgis-mergin-plugin#219 --- mergin/merginproject.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index f7000aed..2e104755 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -250,14 +250,21 @@ def get_pull_changes(self, server_files): diffs = [] diffs_size = 0 is_updated = False + + # get sorted list of the history (they may not be sorted or using lexical sorting - "v10", "v11", "v5", "v6", ...) + history_list = [] + for version_str, version_info in file['history'].items(): + history_list.append( (int_version(version_str), version_info) ) + history_list = sorted(history_list, key=lambda item: item[0]) # sort tuples based on version numbers + # need to track geodiff file history to see if there were any changes - for k, v in file['history'].items(): - if int_version(k) <= int_version(self.metadata['version']): + for version, version_info in history_list: + if version <= int_version(self.metadata['version']): continue # ignore history of no interest is_updated = True - if 'diff' in v: - diffs.append(v['diff']['path']) - diffs_size += v['diff']['size'] + if 'diff' in version_info: + diffs.append(version_info['diff']['path']) + diffs_size += version_info['diff']['size'] else: diffs = [] break # we found force update in history, does not make sense to download diffs From e10c1ae73af1066f6f83bb51b6379740668e6bf7 Mon Sep 17 00:00:00 2001 From: Martin Dobias Date: Thu, 15 Apr 2021 16:16:28 +0200 Subject: [PATCH 2/2] Add more logging when doing pull Diagnostic logs did not have the key error messages included when an error happened during pull - the error was only shown to the user. --- mergin/client_pull.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 88a71cb3..a2af3441 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -494,13 +494,20 @@ def pull_project_finalize(job): # make sure any exceptions from threads are not lost for future in job.futures: if future.exception() is not None: + job.mp.log.error("Error while downloading data: " + str(future.exception())) + job.mp.log.info("--- pull aborted") raise future.exception() job.mp.log.info("finalizing pull") # merge downloaded chunks - for file_to_merge in job.files_to_merge: - file_to_merge.merge() + try: + for file_to_merge in job.files_to_merge: + file_to_merge.merge() + except ClientError as err: + job.mp.log.error("Error merging chunks of downloaded file: " + str(err)) + job.mp.log.info("--- pull aborted") + raise # make sure we can update geodiff reference files (aka. basefiles) with diffs or # download their full versions so we have them up-to-date for applying changes @@ -517,6 +524,10 @@ def pull_project_finalize(job): # was also able to apply those diffs. It could be that someone modified # the basefile and we ended up in this inconsistent state. # let's remove the basefile and let the user retry - we should download clean version again + job.mp.log.error(f"Error patching basefile {basefile}") + job.mp.log.error("Diffs we were applying: " + str(diffs)) + job.mp.log.error("Removing basefile because it would be corrupted anyway...") + job.mp.log.info("--- pull aborted") os.remove(basefile) raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile))