From c92b1dc2f0d8d35b915d389f4655203cc481712d Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 28 Jan 2021 00:17:32 +0700 Subject: [PATCH 01/16] return dummy files on error and update in the background. Notify when background updating --- .../resources/lib/common/utils.py | 40 +++++++++++++++++++ .../resources/lib/menu.py | 2 +- .../resources/lib/refresh.py | 4 ++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index 540af1e9..9bf92abc 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -72,6 +72,11 @@ 'gainsboro', 'lightgray', 'silver', 'darkgray', 'gray', 'dimgray', 'lightslategray', 'slategray', 'darkslategray', 'black', # black 'cornsilk', 'blanchedalmond', 'bisque', 'navajowhite', 'wheat', 'burlywood', 'tan', 'rosybrown', 'sandybrown', 'goldenrod', 'peru', 'chocolate', 'saddlebrown', 'sienna', 'brown', 'maroon'] # brown + + + + + _startup_time = time.time() #TODO: could get reloaded so not accurate? def ft(seconds): @@ -459,6 +464,38 @@ def cache_files(path, widget_id): _, _, changed = cache_expiry(hash, widget_id, add=files) return (files,changed) +ERROR_FILE = { + "jsonrpc": "2.0", + "id": 1, + "result": { + "files": [ + { + "title": "Error", + "label": "Error", + "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=", + "art": get_art('alert'), + "filetype": "file", + } + ] + } +} + +UPDATING_FILE = { + "jsonrpc": "2.0", + "id": 1, + "result": { + "files": [ + { + "title": "Updating", + "label": "Updating", + "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=", + "art": get_art('alert'), + "filetype": "file", + } + ] + } +} + def cache_expiry(hash, widget_id, add=None, no_queue=False): # Currently just caches for 5 min so that the background refresh doesn't go in a loop. @@ -507,10 +544,13 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): else: if not os.path.exists(cache_path): result = "Empty" + contents = UPDATING_FILE + push_cache_queue(hash) else: contents = read_json(cache_path, log_file=True) if contents is None: result = "Invalid Read" + contents = ERROR_FILE else: # write any updated widget_ids so we know what to update when we dequeue # Also important as wwe use last modified of .history as accessed time diff --git a/plugin.program.autowidget/resources/lib/menu.py b/plugin.program.autowidget/resources/lib/menu.py index ef3d2dfc..e88209c8 100644 --- a/plugin.program.autowidget/resources/lib/menu.py +++ b/plugin.program.autowidget/resources/lib/menu.py @@ -108,7 +108,7 @@ def group_menu(group_id): 'group': group_id, 'path_id': path_def['id']}, info=path_def['file'], - art=path_def['file']['art'] or art, + art=path_def['file'].get('art') or art, cm=cm, isFolder=False) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 5eeac91c..8b66676d 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -276,6 +276,7 @@ def cache_and_update(widget_ids): or is expired and if so force it to be refreshed. When going through the queue this could mean we refresh paths that other widgets also use. These will then be skipped. """ + dialog = xbmcgui.Dialog() assert widget_ids effected_widgets = set() @@ -292,12 +293,15 @@ def cache_and_update(widget_ids): if isinstance(path, dict): _label = path['label'] path = path['file']['file'] + else: + _label = widget_def.get('group','') hash = hashlib.sha1(path).hexdigest() # TODO: we might be updating paths used by widgets that weren't initiall queued. # We need to return those and ensure they get refreshed also. effected_widgets = effected_widgets.union(utils.widgets_for_path(path)) if utils.is_cache_queue(hash): # we need to update this path regardless + dialog.notification('AutoWidget', "Updating widget {}".format(_label)) new_files, files_changed = utils.cache_files(path, widget_id) changed = changed or files_changed utils.remove_cache_queue(hash) From f37c3f38cd790f12d6b4168078b5045718916982 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 28 Jan 2021 00:36:28 +0700 Subject: [PATCH 02/16] handle unicode in notification --- plugin.program.autowidget/resources/lib/refresh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 8b66676d..65ed89b3 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -301,7 +301,7 @@ def cache_and_update(widget_ids): effected_widgets = effected_widgets.union(utils.widgets_for_path(path)) if utils.is_cache_queue(hash): # we need to update this path regardless - dialog.notification('AutoWidget', "Updating widget {}".format(_label)) + dialog.notification(u'AutoWidget', u"Updating widget {}".format(_label), sound=False) new_files, files_changed = utils.cache_files(path, widget_id) changed = changed or files_changed utils.remove_cache_queue(hash) From e9630963b45585b4e4f914fdab3ea3e688df50d0 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 28 Jan 2021 00:39:59 +0700 Subject: [PATCH 03/16] show when widgets have finished updating --- plugin.program.autowidget/resources/lib/refresh.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 65ed89b3..80494b4d 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -95,6 +95,9 @@ def _update_widgets(self): if not widget_def: continue _update_strings(widget_def) + dialog = xbmcgui.Dialog() + dialog.notification(u'AutoWidget', u"Finished Updating Widgets", sound=False) + if self.abortRequested(): break From 008eda5b72651f8c9429d084b81e1a3e3b83d245 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 28 Jan 2021 00:49:11 +0700 Subject: [PATCH 04/16] notification when backround updating ends --- plugin.program.autowidget/resources/lib/refresh.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 80494b4d..dfe2752a 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -77,9 +77,12 @@ def _update_widgets(self): while not self.abortRequested(): for _ in self.tick(15, 60*15): # TODO: somehow delay to all other plugins loaded? + updated = False unrefreshed_widgets = set() for hash, widget_ids in utils.next_cache_queue(): effected_widgets = cache_and_update(widget_ids) + if effected_widgets: + updated = True utils.remove_cache_queue(hash) # Just in queued path's widget defintion has changed and it didn't update this path unrefreshed_widgets = unrefreshed_widgets.union(effected_widgets).difference(set(widget_ids)) # # wait 5s or for the skin to reload the widget @@ -95,8 +98,9 @@ def _update_widgets(self): if not widget_def: continue _update_strings(widget_def) - dialog = xbmcgui.Dialog() - dialog.notification(u'AutoWidget', u"Finished Updating Widgets", sound=False) + if updated: + dialog = xbmcgui.Dialog() + dialog.notification(u'AutoWidget', u"Finished Updating Widgets", sound=False) if self.abortRequested(): From 144dafc59fde0aaa747612255d289849d6ac5153 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 28 Jan 2021 12:20:09 +0700 Subject: [PATCH 05/16] fix starting up with no cache --- .../resources/lib/common/utils.py | 19 +++++++++++++------ .../resources/lib/refresh.py | 1 + 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index 9bf92abc..cd0abd7b 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -486,8 +486,8 @@ def cache_files(path, widget_id): "result": { "files": [ { - "title": "Updating", - "label": "Updating", + "title": "Loading Content...", + "label": "Loading Content...", "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=", "art": get_art('alert'), "filetype": "file", @@ -529,6 +529,12 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): cache_json = json.dumps(add) if not add or not cache_json.strip(): result = "Invalid Write" + + elif 'error' in add or not add.get('result',{}).get('files'): + # In this case we don't want to cache a bad result + result = "Error" + # TODO: do we schedule a new update? or put dummy content up even if we have + # good cached content? else: write_json(cache_path, add) contents = add @@ -539,9 +545,12 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): write_json(history_path, cache_data) #expiry = history[-1][0] + DEFAULT_CACHE_TIME pred_dur = predict_update_frequency(history) - expiry = history[-1][0] + pred_dur/2.0 + expiry = history[-1][0] + pred_dur*0.75 # less than prediction to ensure pred keeps up to date result = "Wrote" else: + # write any updated widget_ids so we know what to update when we dequeue + # Also important as wwe use last modified of .history as accessed time + write_json(history_path, cache_data) if not os.path.exists(cache_path): result = "Empty" contents = UPDATING_FILE @@ -551,10 +560,8 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): if contents is None: result = "Invalid Read" contents = ERROR_FILE + push_cache_queue(hash) else: - # write any updated widget_ids so we know what to update when we dequeue - # Also important as wwe use last modified of .history as accessed time - write_json(history_path, cache_data) size = len(json.dumps(contents)) if history: expiry = history[-1][0] + predict_update_frequency(history) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index dfe2752a..126226cb 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -258,6 +258,7 @@ def get_files_list(path, titles=None, widget_id=None): _, files, _ = utils.cache_expiry(hash, widget_id) if files is None: # We had no old content so have to block and get it now + # TODO: will no longer happen because we return dummy file instead utils.log("Blocking cache path read: {}".format(hash[:5]), "info") files, changed = utils.cache_files(path, widget_id) From 4c2f49776c026ec329fd133cf607488fe15ce441 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Fri, 29 Jan 2021 00:10:59 +0700 Subject: [PATCH 06/16] don't do notifications duirng playback --- plugin.program.autowidget/resources/lib/refresh.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 126226cb..b2b0908c 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -80,7 +80,8 @@ def _update_widgets(self): updated = False unrefreshed_widgets = set() for hash, widget_ids in utils.next_cache_queue(): - effected_widgets = cache_and_update(widget_ids) + notify = self.refresh_enabled == 1 and not self.player.isPlayingVideo() + effected_widgets = cache_and_update(widget_ids, notify=notify) if effected_widgets: updated = True utils.remove_cache_queue(hash) # Just in queued path's widget defintion has changed and it didn't update this path @@ -279,7 +280,7 @@ def get_files_list(path, titles=None, widget_id=None): return new_files -def cache_and_update(widget_ids): +def cache_and_update(widget_ids, notify=True): """ a widget might have many paths. Ensure each path is either queued for an update or is expired and if so force it to be refreshed. When going through the queue this could mean we refresh paths that other widgets also use. These will then be skipped. @@ -309,7 +310,8 @@ def cache_and_update(widget_ids): effected_widgets = effected_widgets.union(utils.widgets_for_path(path)) if utils.is_cache_queue(hash): # we need to update this path regardless - dialog.notification(u'AutoWidget', u"Updating widget {}".format(_label), sound=False) + if notify: + dialog.notification(u'AutoWidget', u"Updating widget {}".format(_label), sound=False) new_files, files_changed = utils.cache_files(path, widget_id) changed = changed or files_changed utils.remove_cache_queue(hash) From 0217e6e192d0357f32bdee98906edad0165e0293 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Fri, 29 Jan 2021 00:13:18 +0700 Subject: [PATCH 07/16] and final notify --- plugin.program.autowidget/resources/lib/refresh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index b2b0908c..2dcd0a77 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -99,7 +99,7 @@ def _update_widgets(self): if not widget_def: continue _update_strings(widget_def) - if updated: + if updated and self.refresh_enabled == 1 and not self.player.isPlayingVideo(): dialog = xbmcgui.Dialog() dialog.notification(u'AutoWidget', u"Finished Updating Widgets", sound=False) From f09a0fd1402764b649bad54d596295085f5bf5db Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Mon, 1 Feb 2021 09:47:48 +0700 Subject: [PATCH 08/16] tidy update dummy path generation --- .../resources/lib/common/utils.py | 55 +++++++------------ 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index cd0abd7b..eca82d1e 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -73,12 +73,27 @@ 'cornsilk', 'blanchedalmond', 'bisque', 'navajowhite', 'wheat', 'burlywood', 'tan', 'rosybrown', 'sandybrown', 'goldenrod', 'peru', 'chocolate', 'saddlebrown', 'sienna', 'brown', 'maroon'] # brown +_startup_time = time.time() #TODO: could get reloaded so not accurate? +def make_holding_path(label, art): + return { + "jsonrpc": "2.0", + "id": 1, + "result": { + "files": [ + { + "title": label, + "label": label, + "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=", + "art": get_art(art), + "filetype": "file", + } + ] + } +} -_startup_time = time.time() #TODO: could get reloaded so not accurate? - def ft(seconds): return str(datetime.timedelta(seconds=int(seconds))) @@ -464,38 +479,6 @@ def cache_files(path, widget_id): _, _, changed = cache_expiry(hash, widget_id, add=files) return (files,changed) -ERROR_FILE = { - "jsonrpc": "2.0", - "id": 1, - "result": { - "files": [ - { - "title": "Error", - "label": "Error", - "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=", - "art": get_art('alert'), - "filetype": "file", - } - ] - } -} - -UPDATING_FILE = { - "jsonrpc": "2.0", - "id": 1, - "result": { - "files": [ - { - "title": "Loading Content...", - "label": "Loading Content...", - "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=", - "art": get_art('alert'), - "filetype": "file", - } - ] - } -} - def cache_expiry(hash, widget_id, add=None, no_queue=False): # Currently just caches for 5 min so that the background refresh doesn't go in a loop. @@ -553,13 +536,13 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): write_json(history_path, cache_data) if not os.path.exists(cache_path): result = "Empty" - contents = UPDATING_FILE + contents = make_holding_path(u"Loading Content...", "refresh") push_cache_queue(hash) else: contents = read_json(cache_path, log_file=True) if contents is None: result = "Invalid Read" - contents = ERROR_FILE + contents = make_holding_path("Error", "error") push_cache_queue(hash) else: size = len(json.dumps(contents)) From f9d215694e8220d039506845d69b0632aac47742 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Mon, 1 Feb 2021 10:40:22 +0700 Subject: [PATCH 09/16] ensure explode and clone get the path in the foreground so they can build the group properly on add --- .../resources/lib/add.py | 2 +- .../resources/lib/common/utils.py | 23 ++++++++++--------- .../resources/lib/refresh.py | 7 +++--- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/add.py b/plugin.program.autowidget/resources/lib/add.py index 66f8f6e7..0cfabf5c 100644 --- a/plugin.program.autowidget/resources/lib/add.py +++ b/plugin.program.autowidget/resources/lib/add.py @@ -208,7 +208,7 @@ def _copy_path(path_def): return group_def = manage.get_group_by_id(group_id) - files = refresh.get_files_list(path_def['file']['file']) + files = refresh.get_files_list(path_def['file']['file'], background=False) if not files: return diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index eca82d1e..11cef9fb 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -480,12 +480,11 @@ def cache_files(path, widget_id): return (files,changed) -def cache_expiry(hash, widget_id, add=None, no_queue=False): - # Currently just caches for 5 min so that the background refresh doesn't go in a loop. - # In the future it will cache for longer based on the history of how often in changed - # and when it changed in relation to events like events events. - # It should also manage the cache files to remove any too old. - # The cache expiry can also be used later to schedule a future background update. +def cache_expiry(hash, widget_id, add=None, background=True): + # Predict how long to cache for with a min of 5min so updates don't go in a loop + # TODO: find better way to prevents loops so that users trying to manually refresh can do so + # TODO: manage the cache files to remove any too old or no longer used + # TODO: update paths on autowidget refresh based on predicted update frequency cache_path = os.path.join(_addon_path, '{}.cache'.format(hash)) @@ -536,14 +535,16 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): write_json(history_path, cache_data) if not os.path.exists(cache_path): result = "Empty" - contents = make_holding_path(u"Loading Content...", "refresh") - push_cache_queue(hash) + if background: + contents = make_holding_path(u"Loading Content...", "refresh") + push_cache_queue(hash) else: contents = read_json(cache_path, log_file=True) if contents is None: result = "Invalid Read" - contents = make_holding_path("Error", "error") - push_cache_queue(hash) + if background: + contents = make_holding_path("Error", "error") + push_cache_queue(hash) else: size = len(json.dumps(contents)) if history: @@ -552,7 +553,7 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): # queue_len = len(list(iter_queue())) if expiry > time.time(): result = "Read" - elif no_queue: + elif not background: result = "Skip already updated" # elif queue_len > 3: # # Try to give system more breathing space by returning empty cache but ensuring refresh diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 2dcd0a77..66949ee2 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -251,15 +251,14 @@ def refresh_paths(notify=False, force=False): return True, 'AutoWidget' -def get_files_list(path, titles=None, widget_id=None): +def get_files_list(path, titles=None, widget_id=None, background=True): if not titles: titles = [] hash = hashlib.sha1(path).hexdigest() - _, files, _ = utils.cache_expiry(hash, widget_id) + _, files, _ = utils.cache_expiry(hash, widget_id, background=background) if files is None: - # We had no old content so have to block and get it now - # TODO: will no longer happen because we return dummy file instead + # Should only happen now when background is False utils.log("Blocking cache path read: {}".format(hash[:5]), "info") files, changed = utils.cache_files(path, widget_id) From 0dfd23407c132f73cef4c53c34adbef306c277ad Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 11 Feb 2021 01:13:50 +0700 Subject: [PATCH 10/16] use progress to show background updating --- .../resources/lib/common/utils.py | 4 +- .../resources/lib/menu.py | 2 +- .../resources/lib/refresh.py | 48 +++++++++++++------ 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index 11cef9fb..9fbb4477 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -436,7 +436,6 @@ def next_cache_queue(): path = os.path.join(_addon_path, '{}.history'.format(hash)) cache_data = read_json(path) if cache_data: - log("Dequeued cache update: {}".format(hash[:5]), 'notice') yield hash, cache_data.get('widgets',[]) @@ -484,7 +483,8 @@ def cache_expiry(hash, widget_id, add=None, background=True): # Predict how long to cache for with a min of 5min so updates don't go in a loop # TODO: find better way to prevents loops so that users trying to manually refresh can do so # TODO: manage the cache files to remove any too old or no longer used - # TODO: update paths on autowidget refresh based on predicted update frequency + # TODO: update paths on autowidget refresh based on predicted update frequency. e.g. plugins with random paths should + # update when autowidget updates. cache_path = os.path.join(_addon_path, '{}.cache'.format(hash)) diff --git a/plugin.program.autowidget/resources/lib/menu.py b/plugin.program.autowidget/resources/lib/menu.py index e88209c8..fca15e0f 100644 --- a/plugin.program.autowidget/resources/lib/menu.py +++ b/plugin.program.autowidget/resources/lib/menu.py @@ -314,7 +314,7 @@ def show_path(group_id, path_label, widget_id, path, idx=0, titles=None, num=1, directory.add_menu_item(title=title[0], path=file['file'], - art=file['art'], + art=file.get('art'), info=file, isFolder=file['filetype'] == 'directory', props=properties) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 66949ee2..36080b3b 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -17,12 +17,14 @@ _thread = None + class RefreshService(xbmc.Monitor): def __init__(self): """Starts all of the actions of AutoWidget's service.""" super(RefreshService, self).__init__() utils.log('+++++ STARTING AUTOWIDGET SERVICE +++++', 'info') + self.player = Player() utils.ensure_addon_data() self._update_properties() @@ -71,6 +73,8 @@ def tick(self, step, max, abort_check = lambda: False): i += step yield i + + def _update_widgets(self): self._refresh(True) @@ -79,9 +83,27 @@ def _update_widgets(self): # TODO: somehow delay to all other plugins loaded? updated = False unrefreshed_widgets = set() - for hash, widget_ids in utils.next_cache_queue(): - notify = self.refresh_enabled == 1 and not self.player.isPlayingVideo() - effected_widgets = cache_and_update(widget_ids, notify=notify) + queue = list(utils.next_cache_queue()) + class Progress(object): + dialog = None + service = self + done = set() + + def __call__(self, groupname, path): + if self.dialog is None: + self.dialog = xbmcgui.DialogProgressBG() + self.dialog.create(u"Updating Widgets") + if not self.service.player.isPlayingVideo(): + percent = len(self.done)/float(len(queue)+len(self.done)+1) * 100 + self.dialog.update(int(percent), message=groupname) + self.done.add(path) + progress = Progress() + + while queue: + hash, widget_ids = queue.pop(0) + utils.log("Dequeued cache update: {}".format(hash[:5]), 'notice') + + effected_widgets = cache_and_update(widget_ids, notify=progress) if effected_widgets: updated = True utils.remove_cache_queue(hash) # Just in queued path's widget defintion has changed and it didn't update this path @@ -94,14 +116,15 @@ def _update_widgets(self): # utils.log("paused queue until read {:.2} for {}".format(utils.last_read(hash)-before_update, hash[:5]), 'info') if self.abortRequested(): break + queue = list(utils.next_cache_queue()) for widget_id in unrefreshed_widgets: widget_def = manage.get_widget_by_id(widget_id) if not widget_def: continue _update_strings(widget_def) - if updated and self.refresh_enabled == 1 and not self.player.isPlayingVideo(): - dialog = xbmcgui.Dialog() - dialog.notification(u'AutoWidget', u"Finished Updating Widgets", sound=False) + if progress.dialog is not None: + progress.dialog.update(100, "") + progress.dialog.close() if self.abortRequested(): @@ -264,10 +287,9 @@ def get_files_list(path, titles=None, widget_id=None, background=True): new_files = [] if 'error' not in files: - files = files.get('result').get('files') + files = files.get('result').get('files', []) if not files: - utils.log('No items found for {}'.format(path)) - return + utils.log('No items found for {}'.format(path), 'warning') filtered_files = [x for x in files if x['title'] not in titles] for file in filtered_files: @@ -279,13 +301,11 @@ def get_files_list(path, titles=None, widget_id=None, background=True): return new_files -def cache_and_update(widget_ids, notify=True): +def cache_and_update(widget_ids, notify=None): """ a widget might have many paths. Ensure each path is either queued for an update or is expired and if so force it to be refreshed. When going through the queue this could mean we refresh paths that other widgets also use. These will then be skipped. """ - dialog = xbmcgui.Dialog() - assert widget_ids effected_widgets = set() for widget_id in widget_ids: @@ -309,8 +329,8 @@ def cache_and_update(widget_ids, notify=True): effected_widgets = effected_widgets.union(utils.widgets_for_path(path)) if utils.is_cache_queue(hash): # we need to update this path regardless - if notify: - dialog.notification(u'AutoWidget', u"Updating widget {}".format(_label), sound=False) + if notify is not None: + notify(_label, path) new_files, files_changed = utils.cache_files(path, widget_id) changed = changed or files_changed utils.remove_cache_queue(hash) From 21bdc9858a2fa4f4526a821acc053a25f4f13572 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 11 Feb 2021 02:39:14 +0700 Subject: [PATCH 11/16] add progress when exploding --- plugin.program.autowidget/resources/lib/add.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugin.program.autowidget/resources/lib/add.py b/plugin.program.autowidget/resources/lib/add.py index 0cfabf5c..9e7fbe45 100644 --- a/plugin.program.autowidget/resources/lib/add.py +++ b/plugin.program.autowidget/resources/lib/add.py @@ -206,17 +206,25 @@ def _copy_path(path_def): group_id = add_group(path_def['target'], path_def['label']) if not group_id: return + + progress = xbmcgui.DialogProgress() + progress.create(u"Exploding") group_def = manage.get_group_by_id(group_id) files = refresh.get_files_list(path_def['file']['file'], background=False) if not files: + progress.close() return + done = 0 for file in files: + done += 1 if file['type'] in ['movie', 'episode', 'musicvideo', 'song']: continue + progress.update(int(done/float(len(files))*100), file.get('label')) labels = build_labels('json', file, path_def['target']) _add_path(group_def, labels, over=True) + progress.close() dialog.notification('AutoWidget', utils.get_string(32131) .format(len(files), group_def['label'])) From 6b93433a32d032e6792b0585fb171bc7c15985bc Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Thu, 11 Feb 2021 02:42:15 +0700 Subject: [PATCH 12/16] copying instead of exploding --- plugin.program.autowidget/resources/lib/add.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin.program.autowidget/resources/lib/add.py b/plugin.program.autowidget/resources/lib/add.py index 9e7fbe45..f5a4a341 100644 --- a/plugin.program.autowidget/resources/lib/add.py +++ b/plugin.program.autowidget/resources/lib/add.py @@ -208,7 +208,8 @@ def _copy_path(path_def): return progress = xbmcgui.DialogProgress() - progress.create(u"Exploding") + progress.create(u"Copying") + progress.update(1, u"Retrieving") group_def = manage.get_group_by_id(group_id) files = refresh.get_files_list(path_def['file']['file'], background=False) From d6b87640774ad69670366b60c480dbbb7c4f18ae Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Mon, 22 Feb 2021 10:20:52 +0700 Subject: [PATCH 13/16] ensure paths are bytes before getting hash. not sure why paths are unicode sometimes --- plugin.program.autowidget/resources/lib/common/utils.py | 7 +++++-- plugin.program.autowidget/resources/lib/refresh.py | 7 ++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index 9fbb4477..da520030 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -454,8 +454,11 @@ def remove_cache_queue(hash): queue_path = os.path.join(_addon_path, '{}.queue'.format(hash)) remove_file(queue_path) +def path2hash(path): + return hashlib.sha1(six.ensure_binary(path, "utf8")).hexdigest() + def widgets_for_path(path): - hash = hashlib.sha1(path).hexdigest() + hash = path2hash(path) history_path = os.path.join(_addon_path, '{}.history'.format(hash)) cache_data = read_json(history_path) if os.path.exists(history_path) else None if cache_data is None: @@ -465,7 +468,7 @@ def widgets_for_path(path): def cache_files(path, widget_id): - hash = hashlib.sha1(six.text_type(path)).hexdigest() + hash = path2hash(path) version = _get_json_version() props = version == (10, 3, 1) or (version[0] >= 11 and version[1] >= 12) props_info = info_types + ['customproperties'] diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 36080b3b..8c200475 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -3,9 +3,6 @@ import random import time -import hashlib -import json -import os import threading from resources.lib import manage @@ -278,7 +275,7 @@ def get_files_list(path, titles=None, widget_id=None, background=True): if not titles: titles = [] - hash = hashlib.sha1(path).hexdigest() + hash = utils.path2hash(path) _, files, _ = utils.cache_expiry(hash, widget_id, background=background) if files is None: # Should only happen now when background is False @@ -323,7 +320,7 @@ def cache_and_update(widget_ids, notify=None): path = path['file']['file'] else: _label = widget_def.get('group','') - hash = hashlib.sha1(path).hexdigest() + hash = utils.path2hash(path) # TODO: we might be updating paths used by widgets that weren't initiall queued. # We need to return those and ensure they get refreshed also. effected_widgets = effected_widgets.union(utils.widgets_for_path(path)) From ffb80cb463333f447ad6ff76aa70c345c339185c Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Mon, 22 Feb 2021 14:46:53 +0700 Subject: [PATCH 14/16] reduce wait for queue processing to 1s. --- plugin.program.autowidget/resources/lib/menu.py | 5 ++++- plugin.program.autowidget/resources/lib/refresh.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/menu.py b/plugin.program.autowidget/resources/lib/menu.py index fca15e0f..d550eabe 100644 --- a/plugin.program.autowidget/resources/lib/menu.py +++ b/plugin.program.autowidget/resources/lib/menu.py @@ -295,13 +295,16 @@ def show_path(group_id, path_label, widget_id, path, idx=0, titles=None, num=1, 'path': file['file'], 'target': 'next'} + next_path = file['file'] if not paged_widgets or merged else None directory.add_menu_item(title=label, params=update_params if paged_widgets and not merged else None, - path=file['file'] if not paged_widgets or merged else None, + path=next_path, art=utils.get_art('next_page', color), info=file, isFolder=not paged_widgets or merged, props=properties) + # Ensure we precache next page for faster access + utils.push_cache_queue(utils.path2hash(next_path)) else: dupe = False title = (file['label'], file.get('imdbnumber')) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 8c200475..2a8a4ad0 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -76,8 +76,8 @@ def _update_widgets(self): self._refresh(True) while not self.abortRequested(): - for _ in self.tick(15, 60*15): - # TODO: somehow delay to all other plugins loaded? + for _ in self.tick(step=1, max=60*15): + # TODO: somehow delay till all other plugins loaded? updated = False unrefreshed_widgets = set() queue = list(utils.next_cache_queue()) From 14feb7543d3f0db7373004336534996d51d2a1da Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Mon, 22 Feb 2021 14:47:50 +0700 Subject: [PATCH 15/16] precaching next page on paged widgets. but doesn't seem to be working yet --- .../resources/lib/common/directory.py | 1 + plugin.program.autowidget/resources/lib/menu.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/common/directory.py b/plugin.program.autowidget/resources/lib/common/directory.py index 20edf705..c17866af 100644 --- a/plugin.program.autowidget/resources/lib/common/directory.py +++ b/plugin.program.autowidget/resources/lib/common/directory.py @@ -125,6 +125,7 @@ def add_menu_item(title, params=None, path=None, info=None, cm=None, art=None, xbmcplugin.addDirectoryItem(handle=_handle, url=_plugin, listitem=item, isFolder=isFolder) + return _plugin def finish_directory(handle, category, type): diff --git a/plugin.program.autowidget/resources/lib/menu.py b/plugin.program.autowidget/resources/lib/menu.py index d550eabe..8bd304cc 100644 --- a/plugin.program.autowidget/resources/lib/menu.py +++ b/plugin.program.autowidget/resources/lib/menu.py @@ -295,16 +295,15 @@ def show_path(group_id, path_label, widget_id, path, idx=0, titles=None, num=1, 'path': file['file'], 'target': 'next'} - next_path = file['file'] if not paged_widgets or merged else None - directory.add_menu_item(title=label, + next_path = directory.add_menu_item(title=label, params=update_params if paged_widgets and not merged else None, - path=next_path, + path=file['file'] if not paged_widgets or merged else None, art=utils.get_art('next_page', color), info=file, isFolder=not paged_widgets or merged, props=properties) # Ensure we precache next page for faster access - utils.push_cache_queue(utils.path2hash(next_path)) + utils.cache_expiry(utils.path2hash(next_path), widget_id) else: dupe = False title = (file['label'], file.get('imdbnumber')) From 336d8c36c7fcec725bcbf32b1a22138294f62ba2 Mon Sep 17 00:00:00 2001 From: Dylan Jay Date: Mon, 22 Feb 2021 14:48:50 +0700 Subject: [PATCH 16/16] ensure we have history when queuing updates --- .../resources/lib/common/utils.py | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index da520030..9d2698ec 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -420,6 +420,22 @@ def iter_queue(): for path in sorted(queued, key=os.path.getmtime): yield path +def read_history(hash, create_if_missing=True): + history_path = os.path.join(_addon_path, '{}.history'.format(hash)) + if not os.path.exists(history_path): + if create_if_missing: + cache_data = {} + history = cache_data.setdefault('history', []) + widgets = cache_data.setdefault('widgets', []) + write_json(history_path, cache_data) + else: + cache_data = None + else: + cache_data = read_json(history_path) + return cache_data + + + def next_cache_queue(): # Simple queue by creating a .queue file # TODO: use watchdog to use less resources @@ -433,14 +449,18 @@ def next_cache_queue(): # TODO: need to workout if a blocking write is happen while it was queued or right now. # probably need a .lock file to ensure foreground calls can get priority. hash = hash_from_cache_path(path) - path = os.path.join(_addon_path, '{}.history'.format(hash)) - cache_data = read_json(path) - if cache_data: - yield hash, cache_data.get('widgets',[]) + cache_data = read_history(hash, create_if_missing=True) + yield hash, cache_data.get('widgets',[]) -def push_cache_queue(hash): +def push_cache_queue(hash, widget_id=None): queue_path = os.path.join(_addon_path, '{}.queue'.format(hash)) + history = read_history(hash, create_if_missing=True) # Ensure its created + if widget_id is not None and widget_id not in history['widgets']: + history_path = os.path.join(_addon_path, '{}.history'.format(hash)) + history['widgets'].append(widget_id) + write_json(history_path, history) + if os.path.exists(queue_path): pass # Leave original modification date so item is higher priority else: @@ -455,7 +475,10 @@ def remove_cache_queue(hash): remove_file(queue_path) def path2hash(path): - return hashlib.sha1(six.ensure_binary(path, "utf8")).hexdigest() + if path is not None: + return hashlib.sha1(six.ensure_binary(path, "utf8")).hexdigest() + else: + return None def widgets_for_path(path): hash = path2hash(path)