Refactor weather app to use basic DataManager and Config objects

ruins/components/topic_select.py

@@ -6,43 +6,43 @@
 from typing import List
 import streamlit as st
 
+from ruins.core import Config
 
-def topic_selector(topic_list: List[str], force_topic_select: bool = True, container=st, **kwargs) -> str:
+
+def topic_selector(config: Config, container=st, config_expander=st) -> str:
     """
-    Select a topic from a list of topics. The selected topic is returned and
-    additionally published to the session cache.
-
-    Parameters
-    ----------
-    topic_list : List[str]
-        List of topics to select from.
-    force_topic_select : bool
-        If False, the dropdown will not be shown if a topic was already
-        selected and is present in the streamlit session cache.
-        Default: True
-    container : streamlit.st.container
-        Container to use for the dropdown. Defaults to main streamlit.
-    **kwargs
-        These keyword arguments are only accepted to directly inject
-        :class:`Config <ruins.config.Config>` objects.
+    TODO: Alex will das dokumentieren....
     """
-    # check if a topic is already present
-    if kwargs.get('no_cache', False):
-        current_topic = kwargs.get('current_topic', None)
-    else:  # pragma: no cover
-        current_topic = st.session_state.get('topic', kwargs.get('current_topic', None))
-    if current_topic is not None and not force_topic_select:
-        return current_topic
+    current_topic = config.get('current_topic')
+
+    # get topic list
+    topic_list = config['topic_list']
+
+    # get the policy
+    policy = config.get_control_policy('topic_selector')
+
+    # create the control
+    if current_topic is not None:
+        topic = container.selectbox('Select a topic', topic_list)
 
-    # otherwise print select
-    topic = st.selectbox(
-        'Select a topic',
-        topic_list, 
-        index=0 if current_topic is None else topic_list.index(current_topic)
-    )
-
-    # store topic in session cache
-    if current_topic != topic and not kwargs.get('no_cache', False): # pragma: no cover
-        st.session_state['topic'] = topic
+    elif policy == 'show': # pragma: no cover
+        topic = container.selectbox(
+            'Select a topic',
+            topic_list,
+            #index=topic_list.index(config['current_topic'])
+        )
 
-    return topic
+    elif policy == 'hide': # pragma: no cover
+        topic = config_expander.selectbox(
+            'Select a topic',
+            topic_list,
+            #index=topic_list.index(config['current_topic'])
+        )
+
+    else:
+        topic = current_topic
+
+    # set the new topic
+    if current_topic != topic:
+        st.session_state.current_topic = topic
+

ruins/core/__init__.py

@@ -1,2 +1,3 @@
 from .config import Config
-from .data_manager import DataManager
+from .data_manager import DataManager
+from .build import build_config

ruins/core/build.py

@@ -2,16 +2,27 @@
 Build a :class:`Config <ruins.core.Config>` and a 
 :class:`DataManager <ruins.core.DataManager>` from a kwargs dict.
 """
-from types import Union, Tuple
+from typing import Union, Tuple, Dict, List
+import streamlit as st
 
 from .config import Config
 from .data_manager import DataManager
 
 
+st.experimental_singleton
+def contextualized_data_manager(**kwargs) -> DataManager:
+    return DataManager(**kwargs)
 
-def build_config(omit_dataManager: bool = False, **kwargs) -> Tuple[Config, Union[None, DataManager]]:
+
+def build_config(omit_dataManager: bool = False, url_params: Dict[str, List[str]] = {}, **kwargs) -> Tuple[Config, Union[None, DataManager]]:
     """
     """
+    # prepare the url params, if any
+    # url params are always a list: https://docs.streamlit.io/library/api-reference/utilities/st.experimental_get_query_params
+    # TODO: This should be sanitzed to avoid injection attacks!
+    ukwargs = {k: v[0] if len(v) == 1 else v for k, v in url_params.items()}
+    kwargs.update(ukwargs)
+
     # extract the DataManager, if it was already instantiated
     if 'dataManager' in kwargs:
         dataManager = kwargs.pop('dataManager')
@@ -22,8 +33,8 @@ def build_config(omit_dataManager: bool = False, **kwargs) -> Tuple[Config, Unio
     config = Config(**kwargs)
 
     if omit_dataManager:
-        return config
+        return config,  None
     else:
         if dataManager is None:
-            dataManager = DataManager(**config)
+            dataManager = contextualized_data_manager(**config)
         return config, dataManager

ruins/core/cache.py

@@ -0,0 +1,33 @@
+from typing import Callable, List
+from functools import wraps
+import hashlib
+
+
+LOCAL = dict()
+
+def _hashargs(fname, argnames):
+    h = f'{fname};' + ','.join(argnames)
+    digest = hashlib.sha256(h.encode()).hexdigest()
+
+    return digest
+
+def partial_memoize(hash_names: List[str], store: str = 'local'):
+    def func_decorator(f: Callable):
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            argnames = [str(a) for a in args if a in hash_names]
+            argnames.extend([str(v) for k, v in kwargs.items() if k in hash_names])
+
+            # get the parameter hash
+            h = _hashargs(f.__name__, argnames)
+
+            # check if result exists
+            if h in LOCAL:
+                return LOCAL.get(h)
+            else:
+                # process
+                result = f(*args, **kwargs)
+                LOCAL[h] = result
+                return result
+        return wrapper
+    return func_decorator

ruins/core/config.py

@@ -1,8 +1,16 @@
+from streamlit import session_state
+import streamlit as st
+
 import os
 from os.path import join as pjoin
 import json
 from collections.abc import Mapping
 
+
+# check if streamlit is running
+if not st._is_running_with_streamlit:
+    session_state = dict()
+
 class Config(Mapping):
     """
     Streamlit app Config object.
@@ -29,6 +37,7 @@ def __init__(self, path: str = None, **kwargs) -> None:
         # path 
         self.basepath = os.path.abspath(pjoin(os.path.dirname(__file__), '..', '..'))
         self.datapath = pjoin(self.basepath, 'data')
+        self.hot_load = kwargs.get('hot_load', False)
 
         # mime readers
         self.default_sources = {
@@ -45,8 +54,14 @@ def __init__(self, path: str = None, **kwargs) -> None:
         }
         self.sources_args.update(kwargs.get('include_args', {}))
 
+        # app management
+        self.layout = 'centered'
+
+        # app content
+        self.topic_list = ['Warming', 'Weather Indices', 'Drought/Flood', 'Agriculture', 'Extreme Events', 'Wind Energy']
+
         # store the keys
-        self._keys = ['debug', 'basepath', 'datapath', 'default_sources', 'sources_args']
+        self._keys = ['debug', 'basepath', 'datapath', 'hot_load', 'default_sources', 'sources_args', 'layout', 'topic_list']
 
         # check if a path was provided
         conf_args = self.from_json(path) if path else {}
@@ -71,8 +86,35 @@ def _update(self, new_settings: dict) -> None:
             if k not in self._keys:
                 self._keys.append(k)
 
+    def get_control_policy(self, control_name: str) -> str:
+        """
+        Get the control policy for the given control name.
+
+        allowed policies are:
+            - show: always show the control on the main container
+            - hide: hide the control on the main container, but move to the expander
+            - ignore: don't show anything
+
+        """
+        if self.has_key(f'{control_name}_policy'):
+            return self.get(f'{control_name}_policy')
+        elif self.has_key('controls_policy'):
+            return self.get('controls_policy')
+        else:
+            # TODO: discuss with conrad to change this
+            return 'show'
+
+
     def get(self, key: str, default = None):
-        return getattr(self, key, default)
+        if hasattr(self, key):
+            return getattr(self, key)
+        elif key in session_state:
+            return session_state[key]
+        else:
+            return default
+
+    def has_key(self, key) -> bool:
+        return hasattr(self, key) or hasattr(session_state, key) or key in session_state
 
     def __len__(self) -> int:
         return len(self._keys)
@@ -82,4 +124,14 @@ def __iter__(self):
             yield k
 
     def __getitem__(self, key: str):
-        return getattr(self, key)
+        if hasattr(self, key):
+            return getattr(self, key)
+        elif key in session_state:
+            return session_state[key]
+        else:
+            raise KeyError(f"Key {key} not found")
+
+    def __setitem__(self, key: str, value):
+        setattr(self, key, value)
+        if key not in self._keys:
+            self._keys.append(key)

ruins/core/data_manager.py

@@ -67,10 +67,15 @@ class FileSource(DataSource, abc.ABC):
     Abstract base class for file sources. This provides the common interface
     for every data source that is based on a file.
     """
-    def __init__(self, path: str, cache: bool = True, **kwargs):
+    def __init__(self, path: str, cache: bool = True, hot_load = False, **kwargs):
         super().__init__(**kwargs)
         self.path = path
         self.cache = cache
+
+        # check if the dataset should be pre-loaded
+        if hot_load:
+            self.cache = True
+            self.data = self._load_source()
 
     @abc.abstractmethod
     def _load_source(self):
@@ -94,8 +99,11 @@ class HDF5Source(FileSource):
     """
     HDF5 file sources. This class is used to load HDF5 files.
     """
-    def _load_source(self):
+    def _load_source(self) -> xr.Dataset:
         return xr.open_dataset(self.path)
+
+    def read(self) -> xr.Dataset:
+        return super(HDF5Source, self).read()
 
 
 class CSVSource(FileSource):
@@ -139,7 +147,7 @@ class DataManager(Mapping):
         The include_mimes can be overwritten by passing filenames directly.
 
     """
-    def __init__(self, datapath: str = None, cache: bool = True, debug: bool = False, **kwargs) -> None:
+    def __init__(self, datapath: str = None, cache: bool = True, hot_load = False, debug: bool = False, **kwargs) -> None:
         """
         You can pass in a Config as kwargs.
         """
@@ -148,16 +156,17 @@ def __init__(self, datapath: str = None, cache: bool = True, debug: bool = False
             from ruins.core import Config
             self.from_config(**Config(**kwargs))
         else:
-            self.from_config(datapath=datapath, cache=cache, debug=debug, **kwargs)
+            self.from_config(datapath=datapath, cache=cache, hot_load=hot_load, debug=debug, **kwargs)
 
-    def from_config(self, datapath: str = None, cache: bool = True, debug: bool = False, **kwargs) -> None:
+    def from_config(self, datapath: str = None, cache: bool = True, hot_load: bool = False, debug: bool = False, **kwargs) -> None:
         """
         Initialize the DataManager from a :class:`Config <ruins.core.Config>` object.
         """
         # store the main settings
         self._config = kwargs
         self._datapath = datapath
         self.cache = cache
+        self.hot_load = hot_load
         self.debug = debug
 
         # file settings
@@ -225,7 +234,7 @@ def add_source(self, path: str, not_exists: str = 'raise') -> None:
 
             # add the source
 #            args = self._config.get(basename, {})
-            args.update({'path': path, 'cache': self.cache})
+            args.update({'path': path, 'cache': self.cache, 'hot_load': self.hot_load})
             self._data_sources[basename] = BaseClass(**args)
         else:
             if not_exists == 'raise':
@@ -255,7 +264,7 @@ def __iter__(self):
         for name in self._data_sources.keys():
             yield name
 
-    def __getitem__(self, key: str):
+    def __getitem__(self, key: str) -> DataSource:
         """Return the requested datasource"""
         return self._data_sources[key]
 

ruins/plotting/kde.py

@@ -122,4 +122,4 @@ def kde(data, cmdata='none', split_ts=1, cplot=True, eq_period=True):
 
     ax.set_ylabel('Occurrence (KDE)')
 
-    return ax
+    return fig, ax