return dummy files on error and update in the background.

plugin.program.autowidget/resources/lib/add.py

@@ -206,17 +206,26 @@ def _copy_path(path_def):
     group_id = add_group(path_def['target'], path_def['label'])
     if not group_id:
         return
+
+    progress = xbmcgui.DialogProgress()
+    progress.create(u"Copying")
+    progress.update(1, u"Retrieving")
 
     group_def = manage.get_group_by_id(group_id)
-    files = refresh.get_files_list(path_def['file']['file'])
+    files = refresh.get_files_list(path_def['file']['file'], background=False)
     if not files:
+        progress.close()
         return
 
+    done = 0
     for file in files:
+        done += 1
         if file['type'] in ['movie', 'episode', 'musicvideo', 'song']:
             continue
+        progress.update(int(done/float(len(files))*100), file.get('label'))
 
         labels = build_labels('json', file, path_def['target'])
         _add_path(group_def, labels, over=True)
+    progress.close()
     dialog.notification('AutoWidget', utils.get_string(32131)
                                       .format(len(files), group_def['label']))

plugin.program.autowidget/resources/lib/common/directory.py

@@ -125,6 +125,7 @@ def add_menu_item(title, params=None, path=None, info=None, cm=None, art=None,
 
     xbmcplugin.addDirectoryItem(handle=_handle, url=_plugin, listitem=item,
                                 isFolder=isFolder)
+    return _plugin
 
 
 def finish_directory(handle, category, type):

plugin.program.autowidget/resources/lib/common/utils.py

@@ -72,8 +72,28 @@
           'gainsboro', 'lightgray', 'silver', 'darkgray', 'gray', 'dimgray', 'lightslategray', 'slategray', 'darkslategray', 'black',  # black
           'cornsilk', 'blanchedalmond', 'bisque', 'navajowhite', 'wheat', 'burlywood', 'tan', 'rosybrown', 'sandybrown', 'goldenrod', 'peru', 'chocolate', 'saddlebrown', 'sienna', 'brown', 'maroon']  # brown
 
+
 _startup_time = time.time() #TODO: could get reloaded so not accurate?
 
+
+def make_holding_path(label, art):
+    return {
+        "jsonrpc": "2.0", 
+        "id": 1, 
+        "result": {
+            "files": [
+                {
+                    "title": label,
+                    "label": label,
+                    "file": "plugin://plugin.program.autowidget/?mode=force&refresh=&reload=",
+                    "art": get_art(art),
+                    "filetype": "file", 
+                }
+            ]
+        }
+}
+
+
 def ft(seconds):
     return str(datetime.timedelta(seconds=int(seconds)))
 
@@ -400,6 +420,22 @@ def iter_queue():
     for path in sorted(queued, key=os.path.getmtime):
         yield path
 
+def read_history(hash, create_if_missing=True):
+    history_path = os.path.join(_addon_path, '{}.history'.format(hash))
+    if not os.path.exists(history_path):
+        if create_if_missing:
+            cache_data = {}
+            history = cache_data.setdefault('history', [])
+            widgets = cache_data.setdefault('widgets', [])
+            write_json(history_path, cache_data) 
+        else:
+            cache_data = None
+    else:
+        cache_data = read_json(history_path)
+    return cache_data
+
+
+
 def next_cache_queue():
     # Simple queue by creating a .queue file
     # TODO: use watchdog to use less resources
@@ -413,15 +449,18 @@ def next_cache_queue():
         # TODO: need to workout if a blocking write is happen while it was queued or right now.
         # probably need a .lock file to ensure foreground calls can get priority.
         hash = hash_from_cache_path(path)
-        path = os.path.join(_addon_path, '{}.history'.format(hash))
-        cache_data = read_json(path)
-        if cache_data:
-            log("Dequeued cache update: {}".format(hash[:5]), 'notice')
-            yield hash, cache_data.get('widgets',[])
+        cache_data = read_history(hash, create_if_missing=True)
+        yield hash, cache_data.get('widgets',[])
 
 
-def push_cache_queue(hash):
+def push_cache_queue(hash, widget_id=None):
     queue_path = os.path.join(_addon_path, '{}.queue'.format(hash))
+    history = read_history(hash, create_if_missing=True) # Ensure its created
+    if widget_id is not None and widget_id not in history['widgets']:
+        history_path = os.path.join(_addon_path, '{}.history'.format(hash))
+        history['widgets'].append(widget_id)
+        write_json(history_path, history)
+
     if os.path.exists(queue_path):
         pass # Leave original modification date so item is higher priority
     else:
@@ -435,8 +474,14 @@ def remove_cache_queue(hash):
     queue_path = os.path.join(_addon_path, '{}.queue'.format(hash))
     remove_file(queue_path)
 
+def path2hash(path):
+    if path is not None:
+        return hashlib.sha1(six.ensure_binary(path, "utf8")).hexdigest()
+    else:
+        return None
+
 def widgets_for_path(path):
-    hash = hashlib.sha1(path).hexdigest()
+    hash = path2hash(path)
     history_path = os.path.join(_addon_path, '{}.history'.format(hash))
     cache_data = read_json(history_path) if os.path.exists(history_path) else None
     if cache_data is None:
@@ -446,7 +491,7 @@ def widgets_for_path(path):
 
 
 def cache_files(path, widget_id):
-    hash = hashlib.sha1(six.text_type(path)).hexdigest()
+    hash = path2hash(path)
     version = _get_json_version()
     props = version == (10, 3, 1) or (version[0] >= 11 and version[1] >= 12)
     props_info = info_types + ['customproperties']
@@ -460,12 +505,12 @@ def cache_files(path, widget_id):
     return (files,changed)
 
 
-def cache_expiry(hash, widget_id, add=None, no_queue=False):
-    # Currently just caches for 5 min so that the background refresh doesn't go in a loop.
-    # In the future it will cache for longer based on the history of how often in changed
-    # and when it changed in relation to events like events events.
-    # It should also manage the cache files to remove any too old.
-    # The cache expiry can also be used later to schedule a future background update.
+def cache_expiry(hash, widget_id, add=None, background=True):
+    # Predict how long to cache for with a min of 5min so updates don't go in a loop
+    # TODO: find better way to prevents loops so that users trying to manually refresh can do so
+    # TODO: manage the cache files to remove any too old or no longer used
+    # TODO: update paths on autowidget refresh based on predicted update frequency. e.g. plugins with random paths should
+    # update when autowidget updates.
 
     cache_path = os.path.join(_addon_path, '{}.cache'.format(hash))
 
@@ -492,6 +537,12 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False):
         cache_json = json.dumps(add)
         if not add or not cache_json.strip():
             result = "Invalid Write"
+
+        elif 'error' in add or not add.get('result',{}).get('files'):
+            # In this case we don't want to cache a bad result
+            result = "Error"
+            # TODO: do we schedule a new update? or put dummy content up even if we have
+            # good cached content?
         else:
             write_json(cache_path, add)
             contents = add
@@ -502,27 +553,33 @@ def cache_expiry(hash, widget_id, add=None, no_queue=False):
             write_json(history_path, cache_data)
             #expiry = history[-1][0] + DEFAULT_CACHE_TIME
             pred_dur = predict_update_frequency(history)
-            expiry = history[-1][0] + pred_dur/2.0
+            expiry = history[-1][0] + pred_dur*0.75 # less than prediction to ensure pred keeps up to date
             result = "Wrote"
     else:
+        # write any updated widget_ids so we know what to update when we dequeue
+        # Also important as wwe use last modified of .history as accessed time
+        write_json(history_path, cache_data) 
         if not os.path.exists(cache_path):
             result = "Empty"
+            if background:
+                contents = make_holding_path(u"Loading Content...", "refresh")
+                push_cache_queue(hash)
         else:
             contents = read_json(cache_path, log_file=True)
             if contents is None:
                 result = "Invalid Read"
+                if background:
+                    contents = make_holding_path("Error", "error")
+                    push_cache_queue(hash)
             else:
-                # write any updated widget_ids so we know what to update when we dequeue
-                # Also important as wwe use last modified of .history as accessed time
-                write_json(history_path, cache_data) 
                 size = len(json.dumps(contents))
                 if history:
                     expiry = history[-1][0] + predict_update_frequency(history)
 
 #                queue_len = len(list(iter_queue()))
                 if expiry > time.time():
                     result = "Read"
-                elif no_queue:
+                elif not background:
                     result = "Skip already updated"
                 # elif queue_len > 3:
                 #     # Try to give system more breathing space by returning empty cache but ensuring refresh

plugin.program.autowidget/resources/lib/menu.py

@@ -108,7 +108,7 @@ def group_menu(group_id):
                                             'group': group_id,
                                             'path_id': path_def['id']},
                                     info=path_def['file'],
-                                    art=path_def['file']['art'] or art,
+                                    art=path_def['file'].get('art') or art,
                                     cm=cm,
                                     isFolder=False)
 
@@ -295,13 +295,15 @@ def show_path(group_id, path_label, widget_id, path, idx=0, titles=None, num=1,
                              'path': file['file'],
                              'target': 'next'}
 
-            directory.add_menu_item(title=label,
+            next_path = directory.add_menu_item(title=label,
                                     params=update_params if paged_widgets and not merged else None,
                                     path=file['file'] if not paged_widgets or merged else None,
                                     art=utils.get_art('next_page', color),
                                     info=file,
                                     isFolder=not paged_widgets or merged,
                                     props=properties)
+            # Ensure we precache next page for faster access
+            utils.cache_expiry(utils.path2hash(next_path), widget_id)
         else:
             dupe = False
             title = (file['label'], file.get('imdbnumber'))
@@ -314,7 +316,7 @@ def show_path(group_id, path_label, widget_id, path, idx=0, titles=None, num=1,
 
             directory.add_menu_item(title=title[0],
                                     path=file['file'],
-                                    art=file['art'],
+                                    art=file.get('art'),
                                     info=file,
                                     isFolder=file['filetype'] == 'directory',
                                     props=properties)

plugin.program.autowidget/resources/lib/refresh.py

@@ -3,9 +3,6 @@
 
 import random
 import time
-import hashlib
-import json
-import os
 import threading
 
 from resources.lib import manage
@@ -17,12 +14,14 @@
 _thread = None
 
 
+
 class RefreshService(xbmc.Monitor):
 
     def __init__(self):
         """Starts all of the actions of AutoWidget's service."""
         super(RefreshService, self).__init__()
         utils.log('+++++ STARTING AUTOWIDGET SERVICE +++++', 'info')
+
         self.player = Player()
         utils.ensure_addon_data()
         self._update_properties()
@@ -71,15 +70,39 @@ def tick(self, step, max, abort_check = lambda: False):
             i += step
             yield i
 
+
+
     def _update_widgets(self):
         self._refresh(True)
 
         while not self.abortRequested():
-            for _ in self.tick(15, 60*15):
-                # TODO: somehow delay to all other plugins loaded?
+            for _ in self.tick(step=1, max=60*15):
+                # TODO: somehow delay till all other plugins loaded?
+                updated = False
                 unrefreshed_widgets = set()
-                for hash, widget_ids in utils.next_cache_queue():
-                    effected_widgets = cache_and_update(widget_ids)
+                queue = list(utils.next_cache_queue())
+                class Progress(object):
+                    dialog = None
+                    service = self
+                    done = set()
+
+                    def __call__(self, groupname, path):
+                        if self.dialog is None:
+                            self.dialog = xbmcgui.DialogProgressBG()
+                            self.dialog.create(u"Updating Widgets")
+                        if not self.service.player.isPlayingVideo():
+                            percent = len(self.done)/float(len(queue)+len(self.done)+1) * 100
+                            self.dialog.update(int(percent), message=groupname)
+                        self.done.add(path)
+                progress = Progress()
+
+                while queue:
+                    hash, widget_ids = queue.pop(0)
+                    utils.log("Dequeued cache update: {}".format(hash[:5]), 'notice')
+
+                    effected_widgets = cache_and_update(widget_ids, notify=progress)
+                    if effected_widgets:
+                        updated = True
                     utils.remove_cache_queue(hash) # Just in queued path's widget defintion has changed and it didn't update this path
                     unrefreshed_widgets = unrefreshed_widgets.union(effected_widgets).difference(set(widget_ids))
                     # # wait 5s or for the skin to reload the widget
@@ -90,11 +113,16 @@ def _update_widgets(self):
                     # utils.log("paused queue until read {:.2} for {}".format(utils.last_read(hash)-before_update, hash[:5]), 'info') 
                     if self.abortRequested():
                         break
+                    queue = list(utils.next_cache_queue())
                 for widget_id in unrefreshed_widgets:
                     widget_def = manage.get_widget_by_id(widget_id)
                     if not widget_def:
                         continue
                     _update_strings(widget_def)
+                if progress.dialog is not None:
+                    progress.dialog.update(100, "")
+                    progress.dialog.close()
+
 
             if self.abortRequested():
                 break
@@ -243,23 +271,22 @@ def refresh_paths(notify=False, force=False):
     return True, 'AutoWidget'
 
 
-def get_files_list(path, titles=None, widget_id=None):
+def get_files_list(path, titles=None, widget_id=None, background=True):
     if not titles:
         titles = []
 
-    hash = hashlib.sha1(path).hexdigest()
-    _, files, _ = utils.cache_expiry(hash, widget_id)
+    hash = utils.path2hash(path)
+    _, files, _ = utils.cache_expiry(hash, widget_id, background=background)
     if files is None:
-        # We had no old content so have to block and get it now
+        # Should only happen now when background is False
         utils.log("Blocking cache path read: {}".format(hash[:5]), "info")
         files, changed = utils.cache_files(path, widget_id)
 
     new_files = []
     if 'error' not in files:
-        files = files.get('result').get('files')
+        files = files.get('result').get('files', [])
         if not files:
-            utils.log('No items found for {}'.format(path))
-            return
+            utils.log('No items found for {}'.format(path), 'warning')
 
         filtered_files = [x for x in files if x['title'] not in titles]
         for file in filtered_files:
@@ -271,12 +298,11 @@ def get_files_list(path, titles=None, widget_id=None):
 
         return new_files
 
-def cache_and_update(widget_ids):
+def cache_and_update(widget_ids, notify=None):
     """ a widget might have many paths. Ensure each path is either queued for an update
     or is expired and if so force it to be refreshed. When going through the queue this 
     could mean we refresh paths that other widgets also use. These will then be skipped.
     """
-
     assert widget_ids
     effected_widgets = set()
     for widget_id in widget_ids:
@@ -292,12 +318,16 @@ def cache_and_update(widget_ids):
             if isinstance(path, dict):
                 _label = path['label']
                 path = path['file']['file']
-            hash = hashlib.sha1(path).hexdigest()
+            else:
+                _label = widget_def.get('group','')
+            hash = utils.path2hash(path)
             # TODO: we might be updating paths used by widgets that weren't initiall queued.
             # We need to return those and ensure they get refreshed also.
             effected_widgets = effected_widgets.union(utils.widgets_for_path(path))
             if utils.is_cache_queue(hash):
                 # we need to update this path regardless
+                if notify is not None:
+                    notify(_label, path)
                 new_files, files_changed = utils.cache_files(path, widget_id)
                 changed = changed or files_changed
                 utils.remove_cache_queue(hash)