diff --git a/plugin.program.autowidget/resources/lib/add.py b/plugin.program.autowidget/resources/lib/add.py index 66f8f6e7..f5a4a341 100644 --- a/plugin.program.autowidget/resources/lib/add.py +++ b/plugin.program.autowidget/resources/lib/add.py @@ -206,17 +206,26 @@ def _copy_path(path_def): group_id = add_group(path_def['target'], path_def['label']) if not group_id: return + + progress = xbmcgui.DialogProgress() + progress.create(u"Copying") + progress.update(1, u"Retrieving") group_def = manage.get_group_by_id(group_id) - files = refresh.get_files_list(path_def['file']['file']) + files = refresh.get_files_list(path_def['file']['file'], background=False) if not files: + progress.close() return + done = 0 for file in files: + done += 1 if file['type'] in ['movie', 'episode', 'musicvideo', 'song']: continue + progress.update(int(done/float(len(files))*100), file.get('label')) labels = build_labels('json', file, path_def['target']) _add_path(group_def, labels, over=True) + progress.close() dialog.notification('AutoWidget', utils.get_string(32131) .format(len(files), group_def['label'])) diff --git a/plugin.program.autowidget/resources/lib/common/directory.py b/plugin.program.autowidget/resources/lib/common/directory.py index 20edf705..c17866af 100644 --- a/plugin.program.autowidget/resources/lib/common/directory.py +++ b/plugin.program.autowidget/resources/lib/common/directory.py @@ -125,6 +125,7 @@ def add_menu_item(title, params=None, path=None, info=None, cm=None, art=None, xbmcplugin.addDirectoryItem(handle=_handle, url=_plugin, listitem=item, isFolder=isFolder) + return _plugin def finish_directory(handle, category, type): diff --git a/plugin.program.autowidget/resources/lib/common/utils.py b/plugin.program.autowidget/resources/lib/common/utils.py index 540af1e9..9d2698ec 100644 --- a/plugin.program.autowidget/resources/lib/common/utils.py +++ b/plugin.program.autowidget/resources/lib/common/utils.py @@ -72,8 +72,28 @@ 'gainsboro', 'lightgray', 'silver', 'darkgray', 'gray', 'dimgray', 'lightslategray', 'slategray', 'darkslategray', 'black', # black 'cornsilk', 'blanchedalmond', 'bisque', 'navajowhite', 'wheat', 'burlywood', 'tan', 'rosybrown', 'sandybrown', 'goldenrod', 'peru', 'chocolate', 'saddlebrown', 'sienna', 'brown', 'maroon'] # brown + _startup_time = time.time() #TODO: could get reloaded so not accurate? + +def make_holding_path(label, art): + return { + "jsonrpc": "2.0", + "id": 1, + "result": { + "files": [ + { + "title": label, + "label": label, + "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=", + "art": get_art(art), + "filetype": "file", + } + ] + } +} + + def ft(seconds): return str(datetime.timedelta(seconds=int(seconds))) @@ -400,6 +420,22 @@ def iter_queue(): for path in sorted(queued, key=os.path.getmtime): yield path +def read_history(hash, create_if_missing=True): + history_path = os.path.join(_addon_path, '{}.history'.format(hash)) + if not os.path.exists(history_path): + if create_if_missing: + cache_data = {} + history = cache_data.setdefault('history', []) + widgets = cache_data.setdefault('widgets', []) + write_json(history_path, cache_data) + else: + cache_data = None + else: + cache_data = read_json(history_path) + return cache_data + + + def next_cache_queue(): # Simple queue by creating a .queue file # TODO: use watchdog to use less resources @@ -413,15 +449,18 @@ def next_cache_queue(): # TODO: need to workout if a blocking write is happen while it was queued or right now. # probably need a .lock file to ensure foreground calls can get priority. hash = hash_from_cache_path(path) - path = os.path.join(_addon_path, '{}.history'.format(hash)) - cache_data = read_json(path) - if cache_data: - log("Dequeued cache update: {}".format(hash[:5]), 'notice') - yield hash, cache_data.get('widgets',[]) + cache_data = read_history(hash, create_if_missing=True) + yield hash, cache_data.get('widgets',[]) -def push_cache_queue(hash): +def push_cache_queue(hash, widget_id=None): queue_path = os.path.join(_addon_path, '{}.queue'.format(hash)) + history = read_history(hash, create_if_missing=True) # Ensure its created + if widget_id is not None and widget_id not in history['widgets']: + history_path = os.path.join(_addon_path, '{}.history'.format(hash)) + history['widgets'].append(widget_id) + write_json(history_path, history) + if os.path.exists(queue_path): pass # Leave original modification date so item is higher priority else: @@ -435,8 +474,14 @@ def remove_cache_queue(hash): queue_path = os.path.join(_addon_path, '{}.queue'.format(hash)) remove_file(queue_path) +def path2hash(path): + if path is not None: + return hashlib.sha1(six.ensure_binary(path, "utf8")).hexdigest() + else: + return None + def widgets_for_path(path): - hash = hashlib.sha1(path).hexdigest() + hash = path2hash(path) history_path = os.path.join(_addon_path, '{}.history'.format(hash)) cache_data = read_json(history_path) if os.path.exists(history_path) else None if cache_data is None: @@ -446,7 +491,7 @@ def widgets_for_path(path): def cache_files(path, widget_id): - hash = hashlib.sha1(six.text_type(path)).hexdigest() + hash = path2hash(path) version = _get_json_version() props = version == (10, 3, 1) or (version[0] >= 11 and version[1] >= 12) props_info = info_types + ['customproperties'] @@ -460,12 +505,12 @@ def cache_files(path, widget_id): return (files,changed) -def cache_expiry(hash, widget_id, add=None, no_queue=False): - # Currently just caches for 5 min so that the background refresh doesn't go in a loop. - # In the future it will cache for longer based on the history of how often in changed - # and when it changed in relation to events like events events. - # It should also manage the cache files to remove any too old. - # The cache expiry can also be used later to schedule a future background update. +def cache_expiry(hash, widget_id, add=None, background=True): + # Predict how long to cache for with a min of 5min so updates don't go in a loop + # TODO: find better way to prevents loops so that users trying to manually refresh can do so + # TODO: manage the cache files to remove any too old or no longer used + # TODO: update paths on autowidget refresh based on predicted update frequency. e.g. plugins with random paths should + # update when autowidget updates. cache_path = os.path.join(_addon_path, '{}.cache'.format(hash)) @@ -492,6 +537,12 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): cache_json = json.dumps(add) if not add or not cache_json.strip(): result = "Invalid Write" + + elif 'error' in add or not add.get('result',{}).get('files'): + # In this case we don't want to cache a bad result + result = "Error" + # TODO: do we schedule a new update? or put dummy content up even if we have + # good cached content? else: write_json(cache_path, add) contents = add @@ -502,19 +553,25 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): write_json(history_path, cache_data) #expiry = history[-1][0] + DEFAULT_CACHE_TIME pred_dur = predict_update_frequency(history) - expiry = history[-1][0] + pred_dur/2.0 + expiry = history[-1][0] + pred_dur*0.75 # less than prediction to ensure pred keeps up to date result = "Wrote" else: + # write any updated widget_ids so we know what to update when we dequeue + # Also important as wwe use last modified of .history as accessed time + write_json(history_path, cache_data) if not os.path.exists(cache_path): result = "Empty" + if background: + contents = make_holding_path(u"Loading Content...", "refresh") + push_cache_queue(hash) else: contents = read_json(cache_path, log_file=True) if contents is None: result = "Invalid Read" + if background: + contents = make_holding_path("Error", "error") + push_cache_queue(hash) else: - # write any updated widget_ids so we know what to update when we dequeue - # Also important as wwe use last modified of .history as accessed time - write_json(history_path, cache_data) size = len(json.dumps(contents)) if history: expiry = history[-1][0] + predict_update_frequency(history) @@ -522,7 +579,7 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False): # queue_len = len(list(iter_queue())) if expiry > time.time(): result = "Read" - elif no_queue: + elif not background: result = "Skip already updated" # elif queue_len > 3: # # Try to give system more breathing space by returning empty cache but ensuring refresh diff --git a/plugin.program.autowidget/resources/lib/menu.py b/plugin.program.autowidget/resources/lib/menu.py index ef3d2dfc..8bd304cc 100644 --- a/plugin.program.autowidget/resources/lib/menu.py +++ b/plugin.program.autowidget/resources/lib/menu.py @@ -108,7 +108,7 @@ def group_menu(group_id): 'group': group_id, 'path_id': path_def['id']}, info=path_def['file'], - art=path_def['file']['art'] or art, + art=path_def['file'].get('art') or art, cm=cm, isFolder=False) @@ -295,13 +295,15 @@ def show_path(group_id, path_label, widget_id, path, idx=0, titles=None, num=1, 'path': file['file'], 'target': 'next'} - directory.add_menu_item(title=label, + next_path = directory.add_menu_item(title=label, params=update_params if paged_widgets and not merged else None, path=file['file'] if not paged_widgets or merged else None, art=utils.get_art('next_page', color), info=file, isFolder=not paged_widgets or merged, props=properties) + # Ensure we precache next page for faster access + utils.cache_expiry(utils.path2hash(next_path), widget_id) else: dupe = False title = (file['label'], file.get('imdbnumber')) @@ -314,7 +316,7 @@ def show_path(group_id, path_label, widget_id, path, idx=0, titles=None, num=1, directory.add_menu_item(title=title[0], path=file['file'], - art=file['art'], + art=file.get('art'), info=file, isFolder=file['filetype'] == 'directory', props=properties) diff --git a/plugin.program.autowidget/resources/lib/refresh.py b/plugin.program.autowidget/resources/lib/refresh.py index 5eeac91c..2a8a4ad0 100644 --- a/plugin.program.autowidget/resources/lib/refresh.py +++ b/plugin.program.autowidget/resources/lib/refresh.py @@ -3,9 +3,6 @@ import random import time -import hashlib -import json -import os import threading from resources.lib import manage @@ -17,12 +14,14 @@ _thread = None + class RefreshService(xbmc.Monitor): def __init__(self): """Starts all of the actions of AutoWidget's service.""" super(RefreshService, self).__init__() utils.log('+++++ STARTING AUTOWIDGET SERVICE +++++', 'info') + self.player = Player() utils.ensure_addon_data() self._update_properties() @@ -71,15 +70,39 @@ def tick(self, step, max, abort_check = lambda: False): i += step yield i + + def _update_widgets(self): self._refresh(True) while not self.abortRequested(): - for _ in self.tick(15, 60*15): - # TODO: somehow delay to all other plugins loaded? + for _ in self.tick(step=1, max=60*15): + # TODO: somehow delay till all other plugins loaded? + updated = False unrefreshed_widgets = set() - for hash, widget_ids in utils.next_cache_queue(): - effected_widgets = cache_and_update(widget_ids) + queue = list(utils.next_cache_queue()) + class Progress(object): + dialog = None + service = self + done = set() + + def __call__(self, groupname, path): + if self.dialog is None: + self.dialog = xbmcgui.DialogProgressBG() + self.dialog.create(u"Updating Widgets") + if not self.service.player.isPlayingVideo(): + percent = len(self.done)/float(len(queue)+len(self.done)+1) * 100 + self.dialog.update(int(percent), message=groupname) + self.done.add(path) + progress = Progress() + + while queue: + hash, widget_ids = queue.pop(0) + utils.log("Dequeued cache update: {}".format(hash[:5]), 'notice') + + effected_widgets = cache_and_update(widget_ids, notify=progress) + if effected_widgets: + updated = True utils.remove_cache_queue(hash) # Just in queued path's widget defintion has changed and it didn't update this path unrefreshed_widgets = unrefreshed_widgets.union(effected_widgets).difference(set(widget_ids)) # # wait 5s or for the skin to reload the widget @@ -90,11 +113,16 @@ def _update_widgets(self): # utils.log("paused queue until read {:.2} for {}".format(utils.last_read(hash)-before_update, hash[:5]), 'info') if self.abortRequested(): break + queue = list(utils.next_cache_queue()) for widget_id in unrefreshed_widgets: widget_def = manage.get_widget_by_id(widget_id) if not widget_def: continue _update_strings(widget_def) + if progress.dialog is not None: + progress.dialog.update(100, "") + progress.dialog.close() + if self.abortRequested(): break @@ -243,23 +271,22 @@ def refresh_paths(notify=False, force=False): return True, 'AutoWidget' -def get_files_list(path, titles=None, widget_id=None): +def get_files_list(path, titles=None, widget_id=None, background=True): if not titles: titles = [] - hash = hashlib.sha1(path).hexdigest() - _, files, _ = utils.cache_expiry(hash, widget_id) + hash = utils.path2hash(path) + _, files, _ = utils.cache_expiry(hash, widget_id, background=background) if files is None: - # We had no old content so have to block and get it now + # Should only happen now when background is False utils.log("Blocking cache path read: {}".format(hash[:5]), "info") files, changed = utils.cache_files(path, widget_id) new_files = [] if 'error' not in files: - files = files.get('result').get('files') + files = files.get('result').get('files', []) if not files: - utils.log('No items found for {}'.format(path)) - return + utils.log('No items found for {}'.format(path), 'warning') filtered_files = [x for x in files if x['title'] not in titles] for file in filtered_files: @@ -271,12 +298,11 @@ def get_files_list(path, titles=None, widget_id=None): return new_files -def cache_and_update(widget_ids): +def cache_and_update(widget_ids, notify=None): """ a widget might have many paths. Ensure each path is either queued for an update or is expired and if so force it to be refreshed. When going through the queue this could mean we refresh paths that other widgets also use. These will then be skipped. """ - assert widget_ids effected_widgets = set() for widget_id in widget_ids: @@ -292,12 +318,16 @@ def cache_and_update(widget_ids): if isinstance(path, dict): _label = path['label'] path = path['file']['file'] - hash = hashlib.sha1(path).hexdigest() + else: + _label = widget_def.get('group','') + hash = utils.path2hash(path) # TODO: we might be updating paths used by widgets that weren't initiall queued. # We need to return those and ensure they get refreshed also. effected_widgets = effected_widgets.union(utils.widgets_for_path(path)) if utils.is_cache_queue(hash): # we need to update this path regardless + if notify is not None: + notify(_label, path) new_files, files_changed = utils.cache_files(path, widget_id) changed = changed or files_changed utils.remove_cache_queue(hash)