From 8e48e8addf2980795ae90b9d94ef9cb352fce849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Sat, 12 Mar 2022 08:49:59 +0100 Subject: [PATCH 01/25] fix build --- ruins/core/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruins/core/build.py b/ruins/core/build.py index 6247315..efa68cc 100644 --- a/ruins/core/build.py +++ b/ruins/core/build.py @@ -2,7 +2,7 @@ Build a :class:`Config ` and a :class:`DataManager ` from a kwargs dict. """ -from types import Union, Tuple +from typing import Union, Tuple from .config import Config from .data_manager import DataManager From dfeaedeab7007c550833ae0060faff39ef521f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Sat, 12 Mar 2022 08:50:10 +0100 Subject: [PATCH 02/25] extend config --- ruins/core/__init__.py | 3 ++- ruins/core/config.py | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ruins/core/__init__.py b/ruins/core/__init__.py index d1db6a5..ec72ca5 100644 --- a/ruins/core/__init__.py +++ b/ruins/core/__init__.py @@ -1,2 +1,3 @@ from .config import Config -from .data_manager import DataManager \ No newline at end of file +from .data_manager import DataManager +from .build import build_config \ No newline at end of file diff --git a/ruins/core/config.py b/ruins/core/config.py index d9d412c..3e48984 100644 --- a/ruins/core/config.py +++ b/ruins/core/config.py @@ -45,8 +45,14 @@ def __init__(self, path: str = None, **kwargs) -> None: } self.sources_args.update(kwargs.get('include_args', {})) + # app management + self.layout = 'centered' + + # app content + self.topic_list = ['Warming', 'Weather Indices', 'Drought/Flood', 'Agriculture', 'Extreme Events', 'Wind Energy'] + # store the keys - self._keys = ['debug', 'basepath', 'datapath', 'default_sources', 'sources_args'] + self._keys = ['debug', 'basepath', 'datapath', 'default_sources', 'sources_args', 'layout', 'topic_list'] # check if a path was provided conf_args = self.from_json(path) if path else {} From 97b527fbfd3352f0165488506f387454ffad83de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Sat, 12 Mar 2022 08:50:24 +0100 Subject: [PATCH 03/25] building dataManager into weather app --- ruins/apps/weather.py | 51 +++++++++++++++++------------------------- ruins/plotting/maps.py | 2 +- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 2c0b9b9..140aba4 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -6,26 +6,12 @@ from ruins.plotting import plt_map, kde, yrplot_hm from ruins import components +from ruins.core import build_config, DataManager, Config #### # OLD STUFF -# -# TODO: replace with DataManager -def load_alldata(): - weather = xr.load_dataset('data/weather.nc') - climate = xr.load_dataset('data/cordex_coast.nc') - - # WARNING - bug fix for now: - # 'HadGEM2-ES' model runs are problematic and will be removed for now - # The issue is with the timestamp and requires revision of the ESGF reading routines - kys = [s for s in list(climate.keys()) if 'HadGEM2-ES' not in s] #remove all entries of HadGEM2-ES (6 entries) - climate = climate[kys] - - return weather, climate - - def applySDM(wdata, data, meth='rel', cdf_threshold=0.9999999, lower_limit=0.1): '''apply structured distribution mapping to climate data and return unbiased version of dataset''' from sdm import SDM @@ -144,14 +130,11 @@ def climate_indices(weather: xr.Dataset, climate: xr.Dataset, stati='coast', cli return -def weather_explorer(w_topic: str): - weather, climate = load_alldata() - #weather = load_data('Weather') - - #aspects = ['Annual', 'Monthly', 'Season'] - #w_aspect = st.sidebar.selectbox('Temporal aggegate:', aspects) - - #cliproj = st.sidebar.checkbox('add climate projections',False) +def weather_explorer(w_topic: str, config: Config, dataManager: DataManager): + """ + """ + # load data + weather = dataManager['weather'].read() statios = list(weather.keys()) stat1 = st.selectbox('Select station/group (see map in sidebar for location):', statios) @@ -303,19 +286,27 @@ def weather_explorer(w_topic: str): unsafe_allow_html=True) -def main_app(): +def main_app(**kwargs): + """Describe the params in kwargs here + + The main weather explorer app accepts all + """ + # build the config and dataManager from kwargs + config, dataManager = build_config(**kwargs) + + st.set_page_config(page_title='Weather Explorer', layout=config.layout) + + # build the app st.header('Weather Data Explorer') st.markdown('''In this section we provide visualisations to explore changes in observed weather data. Based on different variables and climate indices it is possible to investigate how climate change manifests itself in different variables, at different stations and with different temporal aggregation.''',unsafe_allow_html=True) - # TODO: refactor this - topics = ['Warming', 'Weather Indices', 'Drought/Flood', 'Agriculture', 'Extreme Events', 'Wind Energy'] - # topic selector - topic = components.topic_selector(topic_list=topics, container=st.sidebar) + topic = components.topic_selector(topic_list=config.topic_list, container=st.sidebar) # TODO refactor this - weather_explorer(topic) + weather_explorer(topic, config, dataManager) if __name__ == '__main__': - main_app() + import fire + fire.Fire(main_app) diff --git a/ruins/plotting/maps.py b/ruins/plotting/maps.py index cdbcd1a..2268270 100644 --- a/ruins/plotting/maps.py +++ b/ruins/plotting/maps.py @@ -4,7 +4,7 @@ import numpy as np -def plt_map(sel='all',cm='none'): +def plt_map(sel='all', cm='none'): # TODO remove this part import xarray as xr import pandas as pd From 5bf6f86e2b64ecd6cac13a43bbda3f1c0154f2b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Sat, 12 Mar 2022 08:57:40 +0100 Subject: [PATCH 04/25] accept url params --- ruins/apps/weather.py | 3 ++- ruins/core/build.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 140aba4..7ebb0a1 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -292,7 +292,8 @@ def main_app(**kwargs): The main weather explorer app accepts all """ # build the config and dataManager from kwargs - config, dataManager = build_config(**kwargs) + url_params = st.experimental_get_query_params() + config, dataManager = build_config(url_params=url_params, **kwargs) st.set_page_config(page_title='Weather Explorer', layout=config.layout) diff --git a/ruins/core/build.py b/ruins/core/build.py index efa68cc..bcbbfc5 100644 --- a/ruins/core/build.py +++ b/ruins/core/build.py @@ -2,16 +2,22 @@ Build a :class:`Config ` and a :class:`DataManager ` from a kwargs dict. """ -from typing import Union, Tuple +from typing import Union, Tuple, Dict, List from .config import Config from .data_manager import DataManager -def build_config(omit_dataManager: bool = False, **kwargs) -> Tuple[Config, Union[None, DataManager]]: +def build_config(omit_dataManager: bool = False, url_params: Dict[str, List[str]] = {}, **kwargs) -> Tuple[Config, Union[None, DataManager]]: """ """ + # prepare the url params, if any + # url params are always a list: https://docs.streamlit.io/library/api-reference/utilities/st.experimental_get_query_params + # TODO: This should be sanitzed to avoid injection attacks! + ukwargs = {k: v[0] if len(v) == 1 else v for k, v in url_params.items()} + kwargs.update(ukwargs) + # extract the DataManager, if it was already instantiated if 'dataManager' in kwargs: dataManager = kwargs.pop('dataManager') From 2ae529f416a3f20b44db2ee8c5e3a92a0cccb0c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Mon, 14 Mar 2022 16:54:52 +0100 Subject: [PATCH 05/25] refactoring weather app parts --- ruins/apps/weather.py | 304 +++++++++++++++++++++++++----------------- 1 file changed, 182 insertions(+), 122 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 7ebb0a1..896f108 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -1,3 +1,4 @@ +from typing import List import streamlit as st import xarray as xr # TODO: these references should be moved to DataManager import pandas as pd # TODO: these references should be moved to DataManager @@ -7,6 +8,7 @@ from ruins.plotting import plt_map, kde, yrplot_hm from ruins import components from ruins.core import build_config, DataManager, Config +from ruins.core.cache import partial_memoize @@ -61,7 +63,15 @@ def climate_indi(ts, indi='Summer days (Tmax ≥ 25°C)'): # TODO: document + signature # TODO: extract plotting -def climate_indices(weather: xr.Dataset, climate: xr.Dataset, stati='coast', cliproj=True): +def climate_indices(dataManager: DataManager, config: Config): + # get data + weather = dataManager['weather'].read() + climate = dataManager['codex_coast'].read() + + # get the relevant settings + stati = config.get('selected_station', 'coast') + + cindi = ['Ice days (Tmax < 0°C)', 'Frost days (Tmin < 0°C)', 'Summer days (Tmax ≥ 25°C)', 'Hot days (Tmax ≥ 30°C)','Tropic nights (Tmin ≥ 20°C)', 'Rainy days (Precip ≥ 1mm)'] ci_topic = st.selectbox('Select Index:', cindi) @@ -83,7 +93,7 @@ def climate_indices(weather: xr.Dataset, climate: xr.Dataset, stati='coast', cli wi.plot(style='.', color='steelblue', label='Coast weather') wi.rolling(10, center=True).mean().plot(color='steelblue', label='Rolling mean\n(10 years)') - if cliproj: + if config['include_climate']: c1 = climate.sel(vars=vari).to_dataframe() c1 = c1[c1.columns[c1.columns != 'vars']] c2 = applySDM(w1[vari], c1, meth=meth) @@ -130,157 +140,209 @@ def climate_indices(weather: xr.Dataset, climate: xr.Dataset, stati='coast', cli return -def weather_explorer(w_topic: str, config: Config, dataManager: DataManager): - """ +def data_select(dataManager: DataManager, config: Config, container=st) -> Config: + """Create the user interface to control the data view """ - # load data + # get a station list weather = dataManager['weather'].read() + station_list = list(weather.keys()) + selected_station = container.selectbox('Select station/group (see map in sidebar for location):', station_list) - statios = list(weather.keys()) - stat1 = st.selectbox('Select station/group (see map in sidebar for location):', statios) + # select a temporal aggregation + aggregations = config.get('temporal_aggregations', ['Annual', 'Monthly']) + temp_agg = container.selectbox('Select temporal aggregation:', aggregations) - aspects = ['Annual', 'Monthly'] # , 'Season'] - w_aspect = st.selectbox('Select temporal aggegate:', aspects) + # include climate projections + include_climate = container.checkbox('Include climate projections (for coastal region)', value=False) - cliproj = st.checkbox('add climate projections (for coastal region)',False) - if cliproj: - plt_map(stat1, 'CORDEX') - st.sidebar.markdown( - '''Map with available stations (blue dots) and selected reference station (magenta highlight). The climate model grid is given in orange with the selected references as filled dots.''', - unsafe_allow_html=True) - else: - plt_map(stat1) - st.sidebar.markdown( - '''Map with available stations (blue dots) and selected reference station (magenta highlight).''', - unsafe_allow_html=True) + # add settings + config['selected_station'] = selected_station + config['temp_agg'] = temp_agg + config['include_climate'] = include_climate - if w_topic == 'Warming': + return config - navi_vars = ['Maximum Air Temperature', 'Mean Air Temperature', 'Minimum Air Temperature'] - navi_var = st.sidebar.radio("Select variable:", options=navi_vars) - if navi_var[:4] == 'Mini': - vari = 'Tmin' - afu = np.min - ag = 'min' - elif navi_var[:4] == 'Maxi': - vari = 'Tmax' - afu = np.max - ag = 'max' - else: - vari = 'T' - afu = np.mean - ag = 'mean' - - if w_aspect == 'Annual': - wdata = weather[stat1].sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe()[stat1] - wdata = wdata[~np.isnan(wdata)] - allw = weather.sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe().iloc[:, 1:] - - dataLq = float(np.floor(allw.min().quantile(0.22))) - datamin = float(np.min([dataLq, np.round(allw.min().min(), 1)])) - if cliproj: - rcps = ['rcp26', 'rcp45', 'rcp85'] - rcp = st.selectbox( - 'RCP (Mean over all projections will be shown. For more details go to section "Climate Projections"):', - rcps) - - data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe() - data = data[data.columns[data.columns != 'vars']] - data_ub = applySDM(wdata, data, meth='abs') - dataUq = float(np.ceil(data_ub.max().quantile(0.76))) - datamax = float(np.max([dataUq, np.round(data_ub.max().max(), 1)])) - else: - dataUq = float(np.ceil(allw.max().quantile(0.76))) - datamax = float(np.max([dataUq,np.round(allw.max().max(), 1)])) +@partial_memoize(hash_names=['reducer', 'station', 'variable', 'time']) +def _reduce_weather_data(dataManager: DataManager, reducer: Callable, station: str, variable: str, time: str) -> Tuple(pd.DataFrame, pd.DataFrame): + # get weather data + weather = dataManager['weather'].read() - datarng = st.slider('Adjust data range on x-axis of plot:', min_value=datamin, max_value=datamax, value=(dataLq, dataUq), step=0.1, key='drangew') + # reduce to station and variable + reduced = weather[station].sel(vars=variable).resample(time=time).apply(reducer) - if cliproj: - ax = kde(wdata, data_ub.mean(axis=1), split_ts=3) - else: - ax = kde(wdata, split_ts=3) + # change to dataframe + wdata = reduced.to_dataframe()[station] + wdata = wdata[~np.isnan(wdata)] + + # reduce for all stations + reduced = weather.sel(vars=variable).resample(time=time).apply(reducer) + reference = reduced.to_dataframe().iloc[:, 1:] - ax.set_title(stat1 + ' Annual ' + navi_var) - ax.set_xlabel('T (°C)') - ax.set_xlim(datarng[0],datarng[1]) - st.pyplot() + return wdata, reference - sndstat = st.checkbox('Show second station for comparison') - if sndstat: - stat2 = st.selectbox('Select second station:', [x for x in statios if x != stat1]) - wdata2 = weather[stat2].sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe()[stat2] +def warming_data_plotter(dataManager: DataManager, config: Config): + # TODO refactor in data-aggregator and data-plotter for different time frames - ax2 = kde(wdata2, split_ts=3) - ax2.set_title(stat2 + ' Annual ' + navi_var) - ax2.set_xlabel('T (°C)') - ax2.set_xlim(datarng[0],datarng[1]) - st.pyplot() + # ---- + # data-aggregator controls + navi_vars = ['Maximum Air Temperature', 'Mean Air Temperature', 'Minimum Air Temperature'] + navi_var = st.sidebar.radio("Select variable:", options=navi_vars) + if navi_var[:4] == 'Mini': + vari = 'Tmin' + afu = np.min + ag = 'min' + elif navi_var[:4] == 'Maxi': + vari = 'Tmax' + afu = np.max + ag = 'max' + else: + vari = 'T' + afu = np.mean + ag = 'mean' + + # controls end + # ---- + + # TODO: this produces a slider but also needs some data caching + if config['temp_agg'] == 'Annual': + wdata, allw = _reduce_weather_data(dataManager, afu, config['selected_station'], vari, '1Y') + + dataLq = float(np.floor(allw.min().quantile(0.22))) + datamin = float(np.min([dataLq, np.round(allw.min().min(), 1)])) + + if config['include_climate']: + rcps = ['rcp26', 'rcp45', 'rcp85'] + rcp = st.selectbox( + 'RCP (Mean over all projections will be shown. For more details go to section "Climate Projections"):', + rcps) + + data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe() + data = data[data.columns[data.columns != 'vars']] + data_ub = applySDM(wdata, data, meth='abs') + + dataUq = float(np.ceil(data_ub.max().quantile(0.76))) + datamax = float(np.max([dataUq, np.round(data_ub.max().max(), 1)])) + else: + dataUq = float(np.ceil(allw.max().quantile(0.76))) + datamax = float(np.max([dataUq,np.round(allw.max().max(), 1)])) + + datarng = st.slider('Adjust data range on x-axis of plot:', min_value=datamin, max_value=datamax, value=(dataLq, dataUq), step=0.1, key='drangew') - # Re-implement this as a application wide service - # expl_md = read_markdown_file('explainer/stripes.md') - # st.markdown(expl_md, unsafe_allow_html=True) + # ------------------- + # start plotting plot + if config['include_climate']: + ax = kde(wdata, data_ub.mean(axis=1), split_ts=3) + else: + ax = kde(wdata, split_ts=3) + + ax.set_title(stat1 + ' Annual ' + navi_var) + ax.set_xlabel('T (°C)') + ax.set_xlim(datarng[0],datarng[1]) + st.pyplot() - elif w_aspect == 'Monthly': - wdata = weather[stat1].sel(vars=vari).resample(time='1M').apply(afu).to_dataframe()[stat1] - wdata = wdata[~np.isnan(wdata)] - ref_yr = st.slider('Reference period for anomaly calculation:', min_value=int(wdata.index.year.min()), max_value=2020,value=(max(1980, int(wdata.index.year.min())), 2000)) + sndstat = st.checkbox('Show second station for comparison') - if cliproj: - rcps = ['rcp26', 'rcp45', 'rcp85'] - rcp = st.selectbox('RCP (Mean over all projections will be shown. For more details go to section "Climate Projections"):', rcps) + if sndstat: + stat2 = st.selectbox('Select second station:', [x for x in statios if x != stat1]) + wdata2 = weather[stat2].sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe()[stat2] - data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1M').apply(afu).to_dataframe() - data = data[data.columns[data.columns != 'vars']] + ax2 = kde(wdata2, split_ts=3) + ax2.set_title(stat2 + ' Annual ' + navi_var) + ax2.set_xlabel('T (°C)') + ax2.set_xlim(datarng[0],datarng[1]) + st.pyplot() - #ub = st.sidebar.checkbox('Apply SDM bias correction',True) - ub = True # simplify here and automatically apply bias correction + # Re-implement this as a application wide service + # expl_md = read_markdown_file('explainer/stripes.md') + # st.markdown(expl_md, unsafe_allow_html=True) - if ub: - data_ub = applySDM(wdata, data, meth='abs') - yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data_ub.mean(axis=1)]),ref_yr, ag, li=2006) - else: - yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data.mean(axis=1)]), ref_yr, ag, li=2006) + elif config['temp_agg'] == 'Monthly': + wdata = weather[stat1].sel(vars=vari).resample(time='1M').apply(afu).to_dataframe()[stat1] + wdata = wdata[~np.isnan(wdata)] + ref_yr = st.slider('Reference period for anomaly calculation:', min_value=int(wdata.index.year.min()), max_value=2020,value=(max(1980, int(wdata.index.year.min())), 2000)) - plt.title(stat1 + ' ' + navi_var + ' anomaly to ' + str(ref_yr[0]) + '-' + str(ref_yr[1])) - st.pyplot() + if config['include_climate']: + rcps = ['rcp26', 'rcp45', 'rcp85'] + rcp = st.selectbox('RCP (Mean over all projections will be shown. For more details go to section "Climate Projections"):', rcps) + data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1M').apply(afu).to_dataframe() + data = data[data.columns[data.columns != 'vars']] + #ub = st.sidebar.checkbox('Apply SDM bias correction',True) + ub = True # simplify here and automatically apply bias correction + if ub: + data_ub = applySDM(wdata, data, meth='abs') + yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data_ub.mean(axis=1)]),ref_yr, ag, li=2006) else: - yrplot_hm(wdata,ref_yr,ag) - plt.title(stat1 + ' ' + navi_var + ' anomaly to ' + str(ref_yr[0]) + '-' + str(ref_yr[1])) - st.pyplot() + yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data.mean(axis=1)]), ref_yr, ag, li=2006) - sndstat = st.checkbox('Compare to a second station?') + plt.title(stat1 + ' ' + navi_var + ' anomaly to ' + str(ref_yr[0]) + '-' + str(ref_yr[1])) + st.pyplot() - if sndstat: - stat2 = st.selectbox('Select second station:', [x for x in statios if x != stat1]) - data2 = weather[stat2].sel(vars=vari).resample(time='1M').apply(afu).to_dataframe()[stat2] - data2 = data2[~np.isnan(data2)] - ref_yr2 = list(ref_yr) - if ref_yr2[1]blue dots) and selected reference station (magenta highlight). The climate model grid is given in orange with the selected references as filled dots.''', + unsafe_allow_html=True) + else: + fig = plt_map(dataManager, sel=config['selected_station']) + st.sidebar.plotly_chart(fig) + st.sidebar.markdown( + '''Map with available stations (blue dots) and selected reference station (magenta highlight).''', + unsafe_allow_html=True) + + # switch the topic + if w_topic == 'Warming': + warming_data_plotter(dataManager, config) + elif w_topic == 'Weather Indices': - climate_indices(stat1,cliproj) + climate_indices(config) - if cliproj: + if config['include_climate']: st.markdown( '''RCPs are scenarios about possible greenhouse gas concentrations by the year 2100. RCP2.6 is a world in which little further greenhouse gasses are emitted -- similar to the Paris climate agreement from 2015. RCP8.5 was intendent to explore a rather risky, more worst-case future with further increased emissions. RCP4.5 is one candidate of a more moderate greenhouse gas projection, which might be more likely to resemble a realistic situation. It is important to note that the very limited differentiation between RCP scenarios have been under debate for several years. One outcome is the definition of Shared Socioeconomic Pathways (SSPs) for which today, however, there are not very many model runs awailable. For more information, please check with the [Climatescenario Primer](https://climatescenarios.org/primer/), [CarbonBrief](https://www.carbonbrief.org/explainer-how-shared-socioeconomic-pathways-explore-future-climate-change) and this [NatureComment](https://www.nature.com/articles/d41586-020-00177-3)''', unsafe_allow_html=True) @@ -288,8 +350,6 @@ def weather_explorer(w_topic: str, config: Config, dataManager: DataManager): def main_app(**kwargs): """Describe the params in kwargs here - - The main weather explorer app accepts all """ # build the config and dataManager from kwargs url_params = st.experimental_get_query_params() From d02caff07fdb9a9bab45cd8056dbb76948002b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Mon, 14 Mar 2022 16:55:05 +0100 Subject: [PATCH 06/25] add custom memoizing --- ruins/core/cache.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 ruins/core/cache.py diff --git a/ruins/core/cache.py b/ruins/core/cache.py new file mode 100644 index 0000000..a76efa0 --- /dev/null +++ b/ruins/core/cache.py @@ -0,0 +1,32 @@ +from typing import Callable, List +from functools import wraps +import hashlib + + +LOCAL = dict() + +def _hashargs(fname, argnames): + h = f'{fname};' + ','.join(argnames) + digest = hashlib.sha256(h.encode()).hexdigest() + + return digest + +def partial_memoize(hash_names: List[str], store: str = 'local'): + def func_decorator(f: Callable): + @wraps(f) + def wrapper(*args, **kwargs): + argnames = [a for a in args if a in hash_names] + argnames.extend([k for k in kwargs.values() if k in hash_names]) + # get the parameter hash + h = _hashargs(f.__name__, argnames) + + # check if result exists + if h in LOCAL: + return LOCAL.get(h) + else: + # process + result = f(*args, **kwargs) + LOCAL[h] = result + return result + return wrapper + return func_decorator From 22194337a57f156c68eae946044bc4d819758b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Mon, 14 Mar 2022 16:55:21 +0100 Subject: [PATCH 07/25] add singleton DataManager --- ruins/core/build.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ruins/core/build.py b/ruins/core/build.py index bcbbfc5..62eef1d 100644 --- a/ruins/core/build.py +++ b/ruins/core/build.py @@ -3,11 +3,16 @@ :class:`DataManager ` from a kwargs dict. """ from typing import Union, Tuple, Dict, List +import streamlit as st from .config import Config from .data_manager import DataManager +st.experimental_singleton +def contextualized_data_manager(**kwargs) -> DataManager: + return DataManager(**kwargs) + def build_config(omit_dataManager: bool = False, url_params: Dict[str, List[str]] = {}, **kwargs) -> Tuple[Config, Union[None, DataManager]]: """ @@ -28,8 +33,8 @@ def build_config(omit_dataManager: bool = False, url_params: Dict[str, List[str] config = Config(**kwargs) if omit_dataManager: - return config + return config, None else: if dataManager is None: - dataManager = DataManager(**config) + dataManager = contextualized_data_manager(**config) return config, dataManager From fe7ff762dc162a602d1c449f9fdf09aafe1b29ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Mon, 14 Mar 2022 16:55:35 +0100 Subject: [PATCH 08/25] use dataManager for plots --- ruins/plotting/maps.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/ruins/plotting/maps.py b/ruins/plotting/maps.py index 2268270..9a18df3 100644 --- a/ruins/plotting/maps.py +++ b/ruins/plotting/maps.py @@ -1,23 +1,29 @@ import streamlit as st import plotly.graph_objs as go import plotly.express as px +import pandas as pd import numpy as np +from ruins.core import DataManager +from ruins.core.cache import partial_memoize + + +@partial_memoize(hash_names=['sel', 'cm']) +def plt_map(dataManager: DataManager, sel='all', cm='none') -> go.Figure: + # cordex_grid = xr.open_dataset('data/CORDEXgrid.nc') + # cimp_grid = xr.open_dataset('data/CMIP5grid.nc') + # stats = pd.read_csv('data/stats.csv', index_col=0) + cordex_grid = dataManager['CORDEXgrid'].read() + cimp_grid = dataManager['CMIP5grid'].read() + stats = dataManager['stats'].read() -def plt_map(sel='all', cm='none'): - # TODO remove this part - import xarray as xr - import pandas as pd - dummy = xr.open_dataset('data/CORDEXgrid.nc') - dummy5 = xr.open_dataset('data/CMIP5grid.nc') - stats = pd.read_csv('data/stats.csv', index_col=0) stats['ms'] = 15. stats['color'] = 'gray' mapbox_access_token = 'pk.eyJ1IjoiY29qYWNrIiwiYSI6IkRTNjV1T2MifQ.EWzL4Qk-VvQoaeJBfE6VSA' px.set_mapbox_access_token(mapbox_access_token) - nodexy = pd.DataFrame([dummy.lon.values.ravel(), dummy.lat.values.ravel()]).T + nodexy = pd.DataFrame([cordex_grid.lon.values.ravel(), cordex_grid.lat.values.ravel()]).T nodexy.columns = ['lon', 'lat'] nodexy['hov'] = 'CORDEX grid' @@ -127,8 +133,8 @@ def add_cmpx(cm): [13, 13], [14, 13], [15, 13], [16, 13], [17, 13], [18, 13], [19, 14], [20, 14], [21, 13]] for cc in maskcordex_coast: fig.add_trace(go.Scattermapbox( - lat=[dummy.lat.values[tuple(cc)]], - lon=[dummy.lon.values[tuple(cc)]], + lat=[cordex_grid.lat.values[tuple(cc)]], + lon=[cordex_grid.lon.values[tuple(cc)]], mode='markers', marker=go.scattermapbox.Marker( size=8, @@ -143,11 +149,11 @@ def add_cmpx(cm): #fig = px.scatter_mapbox(nodexy, lat='lat', lon='lon', center={'lat': 53.0, 'lon': 8.3}, zoom=5, opacity=0.1, hover_data=['hov']) fig = px.scatter_mapbox(stats, lat='lat', lon='lon', center={'lat': 53.0, 'lon': 8.6}, zoom=5, size='ms', opacity=0.8, color='color', hover_data=['Station name', 'lat', 'lon'], size_max=10) if cm != 'none': - fig = lin_grid(fig, dummy) - fig = lin_grid(fig, dummy5, '#2c7fb8') + fig = lin_grid(fig, cordex_grid) + fig = lin_grid(fig, cimp_grid, '#2c7fb8') fig = add_cmpx(cm) fig = add_stats(sel) fig.update_layout(showlegend=False,width=300, height=350,margin=dict(l=10, r=10, b=10, t=10)) # ,center={'lat':54.0,'lon':8.3}, zoom=7) - st.sidebar.plotly_chart(fig) - return + # st.sidebar.plotly_chart(fig) + return fig From fc192d71a5dc29f8053f402b4d10b216d7c621fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Mon, 14 Mar 2022 16:55:44 +0100 Subject: [PATCH 09/25] enable hot-load --- ruins/core/config.py | 8 +++++++- ruins/core/data_manager.py | 16 +++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ruins/core/config.py b/ruins/core/config.py index 3e48984..202242d 100644 --- a/ruins/core/config.py +++ b/ruins/core/config.py @@ -29,6 +29,7 @@ def __init__(self, path: str = None, **kwargs) -> None: # path self.basepath = os.path.abspath(pjoin(os.path.dirname(__file__), '..', '..')) self.datapath = pjoin(self.basepath, 'data') + self.hot_load = kwargs.get('hot_load', False) # mime readers self.default_sources = { @@ -52,7 +53,7 @@ def __init__(self, path: str = None, **kwargs) -> None: self.topic_list = ['Warming', 'Weather Indices', 'Drought/Flood', 'Agriculture', 'Extreme Events', 'Wind Energy'] # store the keys - self._keys = ['debug', 'basepath', 'datapath', 'default_sources', 'sources_args', 'layout', 'topic_list'] + self._keys = ['debug', 'basepath', 'datapath', 'hot_load', 'default_sources', 'sources_args', 'layout', 'topic_list'] # check if a path was provided conf_args = self.from_json(path) if path else {} @@ -89,3 +90,8 @@ def __iter__(self): def __getitem__(self, key: str): return getattr(self, key) + + def __setitem__(self, key: str, value): + setattr(self, key, value) + if key not in self._keys: + self._keys.append(key) diff --git a/ruins/core/data_manager.py b/ruins/core/data_manager.py index 5f95e3f..bc8bf0d 100644 --- a/ruins/core/data_manager.py +++ b/ruins/core/data_manager.py @@ -67,10 +67,15 @@ class FileSource(DataSource, abc.ABC): Abstract base class for file sources. This provides the common interface for every data source that is based on a file. """ - def __init__(self, path: str, cache: bool = True, **kwargs): + def __init__(self, path: str, cache: bool = True, hot_load = False, **kwargs): super().__init__(**kwargs) self.path = path self.cache = cache + + # check if the dataset should be pre-loaded + if hot_load: + self.cache = True + self.data = self._load_source() @abc.abstractmethod def _load_source(self): @@ -139,7 +144,7 @@ class DataManager(Mapping): The include_mimes can be overwritten by passing filenames directly. """ - def __init__(self, datapath: str = None, cache: bool = True, debug: bool = False, **kwargs) -> None: + def __init__(self, datapath: str = None, cache: bool = True, hot_load = False, debug: bool = False, **kwargs) -> None: """ You can pass in a Config as kwargs. """ @@ -148,9 +153,9 @@ def __init__(self, datapath: str = None, cache: bool = True, debug: bool = False from ruins.core import Config self.from_config(**Config(**kwargs)) else: - self.from_config(datapath=datapath, cache=cache, debug=debug, **kwargs) + self.from_config(datapath=datapath, cache=cache, hot_load=hot_load, debug=debug, **kwargs) - def from_config(self, datapath: str = None, cache: bool = True, debug: bool = False, **kwargs) -> None: + def from_config(self, datapath: str = None, cache: bool = True, hot_load: bool = False, debug: bool = False, **kwargs) -> None: """ Initialize the DataManager from a :class:`Config ` object. """ @@ -158,6 +163,7 @@ def from_config(self, datapath: str = None, cache: bool = True, debug: bool = Fa self._config = kwargs self._datapath = datapath self.cache = cache + self.hot_load = hot_load self.debug = debug # file settings @@ -225,7 +231,7 @@ def add_source(self, path: str, not_exists: str = 'raise') -> None: # add the source # args = self._config.get(basename, {}) - args.update({'path': path, 'cache': self.cache}) + args.update({'path': path, 'cache': self.cache, 'hot_load': self.hot_load}) self._data_sources[basename] = BaseClass(**args) else: if not_exists == 'raise': From e7c4806e0470785cf4128a433009c4b1ad8cab25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 07:36:06 +0100 Subject: [PATCH 10/25] add missing callable typedef --- ruins/apps/weather.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 896f108..c2f5a13 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Callable import streamlit as st import xarray as xr # TODO: these references should be moved to DataManager import pandas as pd # TODO: these references should be moved to DataManager From a7c505649dd2e56874a429c6f36d618a90237a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 07:38:17 +0100 Subject: [PATCH 11/25] fixed Tuple typedef usage --- ruins/apps/weather.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index c2f5a13..64e1e49 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -1,4 +1,4 @@ -from typing import List, Callable +from typing import List, Callable, Tuple import streamlit as st import xarray as xr # TODO: these references should be moved to DataManager import pandas as pd # TODO: these references should be moved to DataManager @@ -164,7 +164,7 @@ def data_select(dataManager: DataManager, config: Config, container=st) -> Confi @partial_memoize(hash_names=['reducer', 'station', 'variable', 'time']) -def _reduce_weather_data(dataManager: DataManager, reducer: Callable, station: str, variable: str, time: str) -> Tuple(pd.DataFrame, pd.DataFrame): +def _reduce_weather_data(dataManager: DataManager, reducer: Callable, station: str, variable: str, time: str) -> Tuple[pd.DataFrame, pd.DataFrame]: # get weather data weather = dataManager['weather'].read() From 579e0ee1c8a91cd1177029c2fa088710a87ae6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 07:46:01 +0100 Subject: [PATCH 12/25] some type hints --- ruins/core/data_manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ruins/core/data_manager.py b/ruins/core/data_manager.py index bc8bf0d..2c686d4 100644 --- a/ruins/core/data_manager.py +++ b/ruins/core/data_manager.py @@ -99,8 +99,11 @@ class HDF5Source(FileSource): """ HDF5 file sources. This class is used to load HDF5 files. """ - def _load_source(self): + def _load_source(self) -> xr.Dataset: return xr.open_dataset(self.path) + + def read(self) -> xr.Dataset: + return super(HDF5Source, self).read() class CSVSource(FileSource): @@ -261,7 +264,7 @@ def __iter__(self): for name in self._data_sources.keys(): yield name - def __getitem__(self, key: str): + def __getitem__(self, key: str) -> DataSource: """Return the requested datasource""" return self._data_sources[key] From d349567d60e491672cb3950309555ddc6a64167a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 15:27:40 +0100 Subject: [PATCH 13/25] updated topic select --- ruins/components/topic_select.py | 70 ++++++++++++++++---------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/ruins/components/topic_select.py b/ruins/components/topic_select.py index 8289115..472037e 100644 --- a/ruins/components/topic_select.py +++ b/ruins/components/topic_select.py @@ -6,43 +6,43 @@ from typing import List import streamlit as st +from ruins.core import Config -def topic_selector(topic_list: List[str], force_topic_select: bool = True, container=st, **kwargs) -> str: + +def topic_selector(config: Config, container=st, config_expander=st) -> str: """ - Select a topic from a list of topics. The selected topic is returned and - additionally published to the session cache. - - Parameters - ---------- - topic_list : List[str] - List of topics to select from. - force_topic_select : bool - If False, the dropdown will not be shown if a topic was already - selected and is present in the streamlit session cache. - Default: True - container : streamlit.st.container - Container to use for the dropdown. Defaults to main streamlit. - **kwargs - These keyword arguments are only accepted to directly inject - :class:`Config ` objects. + TODO: Alex will das dokumentieren.... """ - # check if a topic is already present - if kwargs.get('no_cache', False): - current_topic = kwargs.get('current_topic', None) - else: # pragma: no cover - current_topic = st.session_state.get('topic', kwargs.get('current_topic', None)) - if current_topic is not None and not force_topic_select: - return current_topic + current_topic = config.get('current_topic') + + # get topic list + topic_list = config['topic_list'] + + # get the policy + policy = config.get_control_policy('topic_selector') + + # create the control + if current_topic is not None: + topic = container.selectbox('Select a topic', topic_list) - # otherwise print select - topic = st.selectbox( - 'Select a topic', - topic_list, - index=0 if current_topic is None else topic_list.index(current_topic) - ) - - # store topic in session cache - if current_topic != topic and not kwargs.get('no_cache', False): # pragma: no cover - st.session_state['topic'] = topic + elif policy == 'show': + topic = container.selectbox( + 'Select a topic', + topic_list, + #index=topic_list.index(config['current_topic']) + ) - return topic + elif policy == 'hide': + topic = config_expander.selectbox( + 'Select a topic', + topic_list, + #index=topic_list.index(config['current_topic']) + ) + + else: + topic = current_topic + + # set the new topic + if current_topic != topic: + st.session_state.current_topic = topic + From d9fa6c0703c2111a50565bda236b303cb770e866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 15:27:53 +0100 Subject: [PATCH 14/25] changed weather app and config for new controls --- ruins/apps/weather.py | 36 ++++++++++++++++++++---------------- ruins/core/config.py | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 64e1e49..432e800 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -66,7 +66,7 @@ def climate_indi(ts, indi='Summer days (Tmax ≥ 25°C)'): def climate_indices(dataManager: DataManager, config: Config): # get data weather = dataManager['weather'].read() - climate = dataManager['codex_coast'].read() + climate = dataManager['cordex_coast'].read() # get the relevant settings stati = config.get('selected_station', 'coast') @@ -140,7 +140,7 @@ def climate_indices(dataManager: DataManager, config: Config): return -def data_select(dataManager: DataManager, config: Config, container=st) -> Config: +def data_select(dataManager: DataManager, config: Config, container=st) -> None: """Create the user interface to control the data view """ # get a station list @@ -156,11 +156,9 @@ def data_select(dataManager: DataManager, config: Config, container=st) -> Confi include_climate = container.checkbox('Include climate projections (for coastal region)', value=False) # add settings - config['selected_station'] = selected_station - config['temp_agg'] = temp_agg - config['include_climate'] = include_climate - - return config + st.session_state.selected_station = selected_station + st.session_state.temp_agg = temp_agg + st.session_state.include_climate = include_climate @partial_memoize(hash_names=['reducer', 'station', 'variable', 'time']) @@ -183,6 +181,11 @@ def _reduce_weather_data(dataManager: DataManager, reducer: Callable, station: s def warming_data_plotter(dataManager: DataManager, config: Config): + weather: xr.Dataset = dataManager['weather'].read() + climate = dataManager['cordex_coast'].read() + statios = dataManager['stats'].read() + stat1 = config['selected_station'] + # TODO refactor in data-aggregator and data-plotter for different time frames # ---- @@ -247,7 +250,6 @@ def warming_data_plotter(dataManager: DataManager, config: Config): if sndstat: stat2 = st.selectbox('Select second station:', [x for x in statios if x != stat1]) wdata2 = weather[stat2].sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe()[stat2] - ax2 = kde(wdata2, split_ts=3) ax2.set_title(stat2 + ' Annual ' + navi_var) ax2.set_xlabel('T (°C)') @@ -314,12 +316,12 @@ def warming_data_plotter(dataManager: DataManager, config: Config): # st.markdown(expl_md, unsafe_allow_html=True) -def weather_explorer(w_topic: str, config: Config, dataManager: DataManager): +def weather_explorer(config: Config, dataManager: DataManager): """ TODO: refactor this whole app into the main_app """ - # update config with current data settings - config = data_select(dataManager, config, container=st) + # update session state with current data settings + data_select(dataManager, config, container=st) # check config if config['include_climate']: @@ -336,11 +338,12 @@ def weather_explorer(w_topic: str, config: Config, dataManager: DataManager): unsafe_allow_html=True) # switch the topic - if w_topic == 'Warming': + topic = config['current_topic'] + if topic == 'Warming': warming_data_plotter(dataManager, config) - elif w_topic == 'Weather Indices': - climate_indices(config) + elif topic == 'Weather Indices': + climate_indices(dataManager, config) if config['include_climate']: st.markdown( @@ -362,10 +365,11 @@ def main_app(**kwargs): st.markdown('''In this section we provide visualisations to explore changes in observed weather data. Based on different variables and climate indices it is possible to investigate how climate change manifests itself in different variables, at different stations and with different temporal aggregation.''',unsafe_allow_html=True) # topic selector - topic = components.topic_selector(topic_list=config.topic_list, container=st.sidebar) + exp = st.expander('CONFIG', expanded=False) + components.topic_selector(config=config, container=st.sidebar, config_expander=exp) # TODO refactor this - weather_explorer(topic, config, dataManager) + weather_explorer(config, dataManager) if __name__ == '__main__': diff --git a/ruins/core/config.py b/ruins/core/config.py index 202242d..590b632 100644 --- a/ruins/core/config.py +++ b/ruins/core/config.py @@ -1,3 +1,5 @@ +from streamlit import session_state + import os from os.path import join as pjoin import json @@ -78,8 +80,35 @@ def _update(self, new_settings: dict) -> None: if k not in self._keys: self._keys.append(k) + def get_control_policy(self, control_name: str) -> str: + """ + Get the control policy for the given control name. + + allowed policies are: + - show: always show the control on the main container + - hide: hide the control on the main container, but move to the expander + - ignore: don't show anything + + """ + if self.has_key(f'{control_name}_policy'): + return self.get(f'{control_name}_policy') + elif self.has_key('controls_policy'): + return self.get('controls_policy') + else: + # TODO: discuss with conrad to change this + return 'show' + + def get(self, key: str, default = None): - return getattr(self, key, default) + if hasattr(self, key): + return getattr(self, key) + elif key in session_state: + return session_state[key] + else: + return default + + def has_key(self, key) -> bool: + return hasattr(self, key) or hasattr(session_state, key) def __len__(self) -> int: return len(self._keys) @@ -89,7 +118,12 @@ def __iter__(self): yield k def __getitem__(self, key: str): - return getattr(self, key) + if hasattr(self, key): + return getattr(self, key) + elif key in session_state: + return session_state[key] + else: + raise KeyError(f"Key {key} not found") def __setitem__(self, key: str, value): setattr(self, key, value) From 968a11fabe85d00640191120239a0f9d75bb0792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 15:37:45 +0100 Subject: [PATCH 15/25] changed config_expander location --- ruins/apps/weather.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 432e800..4262316 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -360,12 +360,14 @@ def main_app(**kwargs): st.set_page_config(page_title='Weather Explorer', layout=config.layout) + # config expander + exp = st.sidebar.expander('Data select and config', expanded=False) + # build the app st.header('Weather Data Explorer') st.markdown('''In this section we provide visualisations to explore changes in observed weather data. Based on different variables and climate indices it is possible to investigate how climate change manifests itself in different variables, at different stations and with different temporal aggregation.''',unsafe_allow_html=True) # topic selector - exp = st.expander('CONFIG', expanded=False) components.topic_selector(config=config, container=st.sidebar, config_expander=exp) # TODO refactor this From 48dac2c3b95f6822bbf630bdf411ddce6e8aa75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 16:07:09 +0100 Subject: [PATCH 16/25] fix in caching hash --- ruins/core/cache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ruins/core/cache.py b/ruins/core/cache.py index a76efa0..6ad5460 100644 --- a/ruins/core/cache.py +++ b/ruins/core/cache.py @@ -16,7 +16,8 @@ def func_decorator(f: Callable): @wraps(f) def wrapper(*args, **kwargs): argnames = [a for a in args if a in hash_names] - argnames.extend([k for k in kwargs.values() if k in hash_names]) + argnames.extend([v for k, v in kwargs.items() if k in hash_names]) + # get the parameter hash h = _hashargs(f.__name__, argnames) From d4987ab3433a964ac09ef6434bfcaa4cb057c58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 16:07:22 +0100 Subject: [PATCH 17/25] refactoring weather data reducer --- ruins/apps/weather.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 4262316..a41dc22 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -161,23 +161,31 @@ def data_select(dataManager: DataManager, config: Config, container=st) -> None: st.session_state.include_climate = include_climate -@partial_memoize(hash_names=['reducer', 'station', 'variable', 'time']) -def _reduce_weather_data(dataManager: DataManager, reducer: Callable, station: str, variable: str, time: str) -> Tuple[pd.DataFrame, pd.DataFrame]: +@partial_memoize(hash_names=['station', 'variable', 'time']) +def _reduce_weather_data(dataManager: DataManager, variable: str, time: str, station: str = None ) -> pd.DataFrame: # get weather data weather = dataManager['weather'].read() + if station is None: + base = weather + else: + base = weather[station] + # reduce to station and variable - reduced = weather[station].sel(vars=variable).resample(time=time).apply(reducer) + reduced = base.sel(vars=variable).resample(time=time) - # change to dataframe - wdata = reduced.to_dataframe()[station] - wdata = wdata[~np.isnan(wdata)] + if variable == 'Tmax': + df = reduced.max(dim='time').to_dataframe() + elif variable == 'Tmin': + df = reduced.min(dim='time').to_dataframe() + else: + df = reduced.mean(dim='time').to_dataframe() - # reduce for all stations - reduced = weather.sel(vars=variable).resample(time=time).apply(reducer) - reference = reduced.to_dataframe().iloc[:, 1:] - - return wdata, reference + if station is None: + return df.iloc[:, 1:] + else: + return df[station] + def warming_data_plotter(dataManager: DataManager, config: Config): @@ -194,15 +202,12 @@ def warming_data_plotter(dataManager: DataManager, config: Config): navi_var = st.sidebar.radio("Select variable:", options=navi_vars) if navi_var[:4] == 'Mini': vari = 'Tmin' - afu = np.min ag = 'min' elif navi_var[:4] == 'Maxi': vari = 'Tmax' - afu = np.max ag = 'max' else: vari = 'T' - afu = np.mean ag = 'mean' # controls end @@ -210,8 +215,9 @@ def warming_data_plotter(dataManager: DataManager, config: Config): # TODO: this produces a slider but also needs some data caching if config['temp_agg'] == 'Annual': - wdata, allw = _reduce_weather_data(dataManager, afu, config['selected_station'], vari, '1Y') - + wdata = _reduce_weather_data(dataManger=dataManager, station=config['selected_station'], variable=vari, time='1Y') + allw = _reduce_weather_data(dataManager=dataManager, variable=vari, time='1Y') + dataLq = float(np.floor(allw.min().quantile(0.22))) datamin = float(np.min([dataLq, np.round(allw.min().min(), 1)])) From 3521c6d0cb6b7d0121d678a74c981a0f2f078f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Tue, 15 Mar 2022 16:34:00 +0100 Subject: [PATCH 18/25] update reducer to handle weather and climate data --- ruins/apps/weather.py | 27 +++++++++++++++------------ ruins/core/cache.py | 4 ++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index a41dc22..270e3fa 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -161,15 +161,18 @@ def data_select(dataManager: DataManager, config: Config, container=st) -> None: st.session_state.include_climate = include_climate -@partial_memoize(hash_names=['station', 'variable', 'time']) -def _reduce_weather_data(dataManager: DataManager, variable: str, time: str, station: str = None ) -> pd.DataFrame: +@partial_memoize(hash_names=['name', 'station', 'variable', 'time', '_filter']) +def _reduce_weather_data(dataManager: DataManager, name: str, variable: str, time: str, station: str = None, _filter: dict = None) -> pd.DataFrame: # get weather data - weather = dataManager['weather'].read() + arr: xr.Dataset = dataManager[name].read() + + if _filter is not None: + arr = arr.filter_by_attrs(**_filter) if station is None: - base = weather + base = arr else: - base = weather[station] + base = arr[station] # reduce to station and variable reduced = base.sel(vars=variable).resample(time=time) @@ -182,10 +185,9 @@ def _reduce_weather_data(dataManager: DataManager, variable: str, time: str, sta df = reduced.mean(dim='time').to_dataframe() if station is None: - return df.iloc[:, 1:] + return df.loc[:, df.columns != 'vars'] else: - return df[station] - + return df[station] def warming_data_plotter(dataManager: DataManager, config: Config): @@ -215,8 +217,8 @@ def warming_data_plotter(dataManager: DataManager, config: Config): # TODO: this produces a slider but also needs some data caching if config['temp_agg'] == 'Annual': - wdata = _reduce_weather_data(dataManger=dataManager, station=config['selected_station'], variable=vari, time='1Y') - allw = _reduce_weather_data(dataManager=dataManager, variable=vari, time='1Y') + wdata = _reduce_weather_data(dataManager, name='weather', station=config['selected_station'], variable=vari, time='1Y') + allw = _reduce_weather_data(dataManager, name='weather', variable=vari, time='1Y') dataLq = float(np.floor(allw.min().quantile(0.22))) datamin = float(np.min([dataLq, np.round(allw.min().min(), 1)])) @@ -227,8 +229,9 @@ def warming_data_plotter(dataManager: DataManager, config: Config): 'RCP (Mean over all projections will be shown. For more details go to section "Climate Projections"):', rcps) - data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe() - data = data[data.columns[data.columns != 'vars']] + #data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe() + #data = data[data.columns[data.columns != 'vars']] + data = _reduce_weather_data(dataManager, name='cordex_coast', variable=vari, time='1Y', _filter=dict(RCP=rcp)) data_ub = applySDM(wdata, data, meth='abs') dataUq = float(np.ceil(data_ub.max().quantile(0.76))) diff --git a/ruins/core/cache.py b/ruins/core/cache.py index 6ad5460..50e14d0 100644 --- a/ruins/core/cache.py +++ b/ruins/core/cache.py @@ -15,8 +15,8 @@ def partial_memoize(hash_names: List[str], store: str = 'local'): def func_decorator(f: Callable): @wraps(f) def wrapper(*args, **kwargs): - argnames = [a for a in args if a in hash_names] - argnames.extend([v for k, v in kwargs.items() if k in hash_names]) + argnames = [str(a) for a in args if a in hash_names] + argnames.extend([str(v) for k, v in kwargs.items() if k in hash_names]) # get the parameter hash h = _hashargs(f.__name__, argnames) From dd63a62fc9821b365d5fc6b06bd50558814b84ee Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Wed, 16 Mar 2022 10:05:38 +0100 Subject: [PATCH 19/25] import seaborn for heatmap --- ruins/plotting/weather_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ruins/plotting/weather_data.py b/ruins/plotting/weather_data.py index 5a52116..a3cd1fe 100644 --- a/ruins/plotting/weather_data.py +++ b/ruins/plotting/weather_data.py @@ -1,6 +1,7 @@ import numpy as np import pandas as pd import matplotlib.pyplot as plt +import seaborn as sns def yrplot_hm(sr, ref=[1980, 2000], ag='sum', qa=0.95, cbar_title='Temperature anomaly (K)', cmx='coolwarm', cmxeq=True, li=False): From d672083767f8418b5b0bca6ed00b278ba4dea255 Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Wed, 16 Mar 2022 10:36:48 +0100 Subject: [PATCH 20/25] _reduce_weather_data() for monthly aggregation and fix comparison to second station --- ruins/apps/weather.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 270e3fa..2644ed2 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -145,7 +145,7 @@ def data_select(dataManager: DataManager, config: Config, container=st) -> None: """ # get a station list weather = dataManager['weather'].read() - station_list = list(weather.keys()) + station_list = list(weather.keys()) # TODO station names krummhoern, coast, inland, niedersachsen? selected_station = container.selectbox('Select station/group (see map in sidebar for location):', station_list) # select a temporal aggregation @@ -193,7 +193,7 @@ def _reduce_weather_data(dataManager: DataManager, name: str, variable: str, tim def warming_data_plotter(dataManager: DataManager, config: Config): weather: xr.Dataset = dataManager['weather'].read() climate = dataManager['cordex_coast'].read() - statios = dataManager['stats'].read() + statios = dataManager['stats'].read().index # TODO weather also has 'station names' krummhoern, coast, inland, niedersachsen? stat1 = config['selected_station'] # TODO refactor in data-aggregator and data-plotter for different time frames @@ -229,8 +229,6 @@ def warming_data_plotter(dataManager: DataManager, config: Config): 'RCP (Mean over all projections will be shown. For more details go to section "Climate Projections"):', rcps) - #data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe() - #data = data[data.columns[data.columns != 'vars']] data = _reduce_weather_data(dataManager, name='cordex_coast', variable=vari, time='1Y', _filter=dict(RCP=rcp)) data_ub = applySDM(wdata, data, meth='abs') @@ -257,8 +255,9 @@ def warming_data_plotter(dataManager: DataManager, config: Config): sndstat = st.checkbox('Show second station for comparison') if sndstat: - stat2 = st.selectbox('Select second station:', [x for x in statios if x != stat1]) - wdata2 = weather[stat2].sel(vars=vari).resample(time='1Y').apply(afu).to_dataframe()[stat2] + stat2 = st.selectbox('Select second station:', [x for x in statios if x != config['selected_station']]) + wdata2 = _reduce_weather_data(dataManager, name='weather', station=stat2, variable=vari, time='1Y') + ax2 = kde(wdata2, split_ts=3) ax2.set_title(stat2 + ' Annual ' + navi_var) ax2.set_xlabel('T (°C)') @@ -270,16 +269,15 @@ def warming_data_plotter(dataManager: DataManager, config: Config): # st.markdown(expl_md, unsafe_allow_html=True) elif config['temp_agg'] == 'Monthly': - wdata = weather[stat1].sel(vars=vari).resample(time='1M').apply(afu).to_dataframe()[stat1] - wdata = wdata[~np.isnan(wdata)] + wdata = _reduce_weather_data(dataManager, name='weather', station=config['selected_station'], variable=vari, time='1M') + ref_yr = st.slider('Reference period for anomaly calculation:', min_value=int(wdata.index.year.min()), max_value=2020,value=(max(1980, int(wdata.index.year.min())), 2000)) if config['include_climate']: rcps = ['rcp26', 'rcp45', 'rcp85'] rcp = st.selectbox('RCP (Mean over all projections will be shown. For more details go to section "Climate Projections"):', rcps) - data = climate.filter_by_attrs(RCP=rcp).sel(vars=vari).resample(time='1M').apply(afu).to_dataframe() - data = data[data.columns[data.columns != 'vars']] + data = _reduce_weather_data(dataManager, name='cordex_coast', variable=vari, time='1M', _filter=dict(RCP=rcp)) #ub = st.sidebar.checkbox('Apply SDM bias correction',True) ub = True # simplify here and automatically apply bias correction @@ -304,8 +302,7 @@ def warming_data_plotter(dataManager: DataManager, config: Config): if sndstat: stat2 = st.selectbox('Select second station:', [x for x in statios if x != stat1]) - data2 = weather[stat2].sel(vars=vari).resample(time='1M').apply(afu).to_dataframe()[stat2] - data2 = data2[~np.isnan(data2)] + data2 = _reduce_weather_data(dataManager, name='weather', station=stat2, variable=vari, time='1M') ref_yr2 = list(ref_yr) if ref_yr2[1] Date: Wed, 16 Mar 2022 11:03:25 +0100 Subject: [PATCH 21/25] plot to fig and call st.pyplot(fig) --- ruins/apps/weather.py | 28 ++++++++++++++-------------- ruins/plotting/kde.py | 2 +- ruins/plotting/weather_data.py | 5 +++-- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 2644ed2..60b013a 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -88,7 +88,7 @@ def climate_indices(dataManager: DataManager, config: Config): w1 = weather[stati].sel(vars=vari).to_dataframe().dropna() w1.columns = ['bla', vari] - plt.figure(figsize=(10,2.5)) + fig = plt.figure(figsize=(10,2.5)) wi = climate_indi(w1, ci_topic).astype(int) wi.plot(style='.', color='steelblue', label='Coast weather') wi.rolling(10, center=True).mean().plot(color='steelblue', label='Rolling mean\n(10 years)') @@ -123,7 +123,7 @@ def climate_indices(dataManager: DataManager, config: Config): plt.legend(ncol=2) plt.ylabel('Number of days') plt.title(ci_topic) - st.pyplot() + st.pyplot(fig) if ci_topic == 'Ice days (Tmax < 0°C)': st.markdown('''Number of days in one year which persistently remain below 0°C air temperature.''') @@ -243,14 +243,14 @@ def warming_data_plotter(dataManager: DataManager, config: Config): # ------------------- # start plotting plot if config['include_climate']: - ax = kde(wdata, data_ub.mean(axis=1), split_ts=3) + fig, ax = kde(wdata, data_ub.mean(axis=1), split_ts=3) else: - ax = kde(wdata, split_ts=3) + fig, ax = kde(wdata, split_ts=3) ax.set_title(stat1 + ' Annual ' + navi_var) ax.set_xlabel('T (°C)') ax.set_xlim(datarng[0],datarng[1]) - st.pyplot() + st.pyplot(fig) sndstat = st.checkbox('Show second station for comparison') @@ -258,11 +258,11 @@ def warming_data_plotter(dataManager: DataManager, config: Config): stat2 = st.selectbox('Select second station:', [x for x in statios if x != config['selected_station']]) wdata2 = _reduce_weather_data(dataManager, name='weather', station=stat2, variable=vari, time='1Y') - ax2 = kde(wdata2, split_ts=3) + fig, ax2 = kde(wdata2, split_ts=3) ax2.set_title(stat2 + ' Annual ' + navi_var) ax2.set_xlabel('T (°C)') ax2.set_xlim(datarng[0],datarng[1]) - st.pyplot() + st.pyplot(fig) # Re-implement this as a application wide service # expl_md = read_markdown_file('explainer/stripes.md') @@ -284,19 +284,19 @@ def warming_data_plotter(dataManager: DataManager, config: Config): if ub: data_ub = applySDM(wdata, data, meth='abs') - yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data_ub.mean(axis=1)]),ref_yr, ag, li=2006) + fig = yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data_ub.mean(axis=1)]),ref_yr, ag, li=2006) else: - yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data.mean(axis=1)]), ref_yr, ag, li=2006) + fig = yrplot_hm(pd.concat([wdata.loc[wdata.index[0]:data.index[0] - pd.Timedelta('1M')], data.mean(axis=1)]), ref_yr, ag, li=2006) plt.title(stat1 + ' ' + navi_var + ' anomaly to ' + str(ref_yr[0]) + '-' + str(ref_yr[1])) - st.pyplot() + st.pyplot(fig) # TODO: break up this as well else: - yrplot_hm(wdata,ref_yr,ag) + fig = yrplot_hm(wdata,ref_yr,ag) plt.title(stat1 + ' ' + navi_var + ' anomaly to ' + str(ref_yr[0]) + '-' + str(ref_yr[1])) - st.pyplot() + st.pyplot(fig) sndstat = st.checkbox('Compare to a second station?') @@ -313,9 +313,9 @@ def warming_data_plotter(dataManager: DataManager, config: Config): if ref_yr2[1] - ref_yr2[0] < 10: ref_yr2[1] = ref_yr2[0] + 10 - yrplot_hm(data2, ref_yr2, ag) + fig = yrplot_hm(data2, ref_yr2, ag) plt.title(stat2 + ' ' + navi_var + ' anomaly to ' + str(ref_yr2[0]) + '-' + str(ref_yr2[1])) - st.pyplot() + st.pyplot(fig) # Re-implement this as a application wide service # expl_md = read_markdown_file('explainer/stripes_m.md') diff --git a/ruins/plotting/kde.py b/ruins/plotting/kde.py index 18ad7fb..ff96635 100644 --- a/ruins/plotting/kde.py +++ b/ruins/plotting/kde.py @@ -122,4 +122,4 @@ def kde(data, cmdata='none', split_ts=1, cplot=True, eq_period=True): ax.set_ylabel('Occurrence (KDE)') - return ax + return fig, ax diff --git a/ruins/plotting/weather_data.py b/ruins/plotting/weather_data.py index a3cd1fe..77cfea0 100644 --- a/ruins/plotting/weather_data.py +++ b/ruins/plotting/weather_data.py @@ -41,7 +41,7 @@ def yrplot_hm(sr, ref=[1980, 2000], ag='sum', qa=0.95, cbar_title='Temperature a if ag == 'sum': dummy.iloc[:, 13] = dummy.iloc[:, 13] / 12 - plt.figure(figsize=(8,len(dummy)/15.)) + fig = plt.figure(figsize=(8,len(dummy)/15.)) ax = sns.heatmap(dummy, cmap=cmx, vmin=vxL, vmax=vxU, cbar_kws={'label': cbar_title}) if ref == None: @@ -66,7 +66,8 @@ def yrplot_hm(sr, ref=[1980, 2000], ag='sum', qa=0.95, cbar_title='Temperature a ax.set_ylabel('Year') ax.set_xlabel('Month ') - return + + return fig def monthlyx(dy, dyx=1, ylab='T (°C)', clab1='Monthly Mean in Year', clab2='Monthly Max in Year', pls='cividis_r'): From 4baecbbec1868fcd3b8e3c4177aba70d0360fa8c Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Wed, 16 Mar 2022 15:37:40 +0100 Subject: [PATCH 22/25] station list always from weather.keys() --- ruins/apps/weather.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruins/apps/weather.py b/ruins/apps/weather.py index 60b013a..56b395e 100644 --- a/ruins/apps/weather.py +++ b/ruins/apps/weather.py @@ -193,7 +193,7 @@ def _reduce_weather_data(dataManager: DataManager, name: str, variable: str, tim def warming_data_plotter(dataManager: DataManager, config: Config): weather: xr.Dataset = dataManager['weather'].read() climate = dataManager['cordex_coast'].read() - statios = dataManager['stats'].read().index # TODO weather also has 'station names' krummhoern, coast, inland, niedersachsen? + statios = list(weather.keys()) stat1 = config['selected_station'] # TODO refactor in data-aggregator and data-plotter for different time frames From 1d5c71553c898707e04dcd5ed252a4033cadf1a2 Mon Sep 17 00:00:00 2001 From: AlexDo1 Date: Wed, 16 Mar 2022 15:38:24 +0100 Subject: [PATCH 23/25] fix test_weather --- ruins/tests/test_weather.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ruins/tests/test_weather.py b/ruins/tests/test_weather.py index 64fe7c7..fc8eee1 100644 --- a/ruins/tests/test_weather.py +++ b/ruins/tests/test_weather.py @@ -1,7 +1,7 @@ from ruins.apps import weather from ruins.tests.util import get_test_config -from ruins.core import DataManager +from ruins.core import DataManager, Config # TODO use the config and inject the dedub config here @@ -14,14 +14,11 @@ def test_climate_indices(): """Test only climate indices """ - conf = get_test_config() - dm = DataManager(**conf) - - w = dm['weather'].read() - c = dm['cordex_coast'].read() + config = Config(include_climate=True) + dm = DataManager() # run - weather.climate_indices(w, c) + weather.climate_indices(dataManager=dm, config=config) def test_climate_indi(): From b4101a99b7854f869f616d30efec2668c22ab4cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Wed, 16 Mar 2022 16:12:39 +0100 Subject: [PATCH 24/25] remove topic select unittests --- ruins/components/topic_select.py | 4 ++-- ruins/core/config.py | 8 +++++++- ruins/tests/test_topic_selector.py | 23 ----------------------- 3 files changed, 9 insertions(+), 26 deletions(-) delete mode 100644 ruins/tests/test_topic_selector.py diff --git a/ruins/components/topic_select.py b/ruins/components/topic_select.py index 472037e..4d7b5b9 100644 --- a/ruins/components/topic_select.py +++ b/ruins/components/topic_select.py @@ -25,14 +25,14 @@ def topic_selector(config: Config, container=st, config_expander=st) -> str: if current_topic is not None: topic = container.selectbox('Select a topic', topic_list) - elif policy == 'show': + elif policy == 'show': # pragma: no cover topic = container.selectbox( 'Select a topic', topic_list, #index=topic_list.index(config['current_topic']) ) - elif policy == 'hide': + elif policy == 'hide': # pragma: no cover topic = config_expander.selectbox( 'Select a topic', topic_list, diff --git a/ruins/core/config.py b/ruins/core/config.py index 590b632..00d6157 100644 --- a/ruins/core/config.py +++ b/ruins/core/config.py @@ -1,10 +1,16 @@ from streamlit import session_state +import streamlit as st import os from os.path import join as pjoin import json from collections.abc import Mapping + +# check if streamlit is running +if not st._is_running_with_streamlit: + session_state = dict() + class Config(Mapping): """ Streamlit app Config object. @@ -108,7 +114,7 @@ def get(self, key: str, default = None): return default def has_key(self, key) -> bool: - return hasattr(self, key) or hasattr(session_state, key) + return hasattr(self, key) or hasattr(session_state, key) or key in session_state def __len__(self) -> int: return len(self._keys) diff --git a/ruins/tests/test_topic_selector.py b/ruins/tests/test_topic_selector.py deleted file mode 100644 index 0a091c8..0000000 --- a/ruins/tests/test_topic_selector.py +++ /dev/null @@ -1,23 +0,0 @@ -from ruins import components - - -def test_default(): - """Test default behavior""" - assert components.topic_selector(['a', 'b', 'c'], no_cache=True) == 'a' - - -def test_current_preset(): - """Test with current topic set""" - topic = components.topic_selector(['a', 'b', 'c'], current_topic='b', no_cache=True) - - assert topic == 'b' - - -def test_no_force_render(): - """Test with rendering disabled""" - # default - assert components.topic_selector(['a', 'b', 'c'], force_topic_select=False, no_cache=True) == 'a' - - # with current topic set - assert components.topic_selector(['a', 'b', 'c'], current_topic='b', force_topic_select=False, no_cache=True) == 'b' - \ No newline at end of file From 9903dad7eaaf78ede0619fa9536240381f065415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Wed, 16 Mar 2022 16:42:30 +0100 Subject: [PATCH 25/25] fix include_climate key --- ruins/tests/test_config.py | 9 +++++++++ ruins/tests/test_weather.py | 14 +++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ruins/tests/test_config.py b/ruins/tests/test_config.py index 548047b..75deb4d 100644 --- a/ruins/tests/test_config.py +++ b/ruins/tests/test_config.py @@ -3,6 +3,7 @@ """ import os import json +import pytest from ruins import core @@ -62,3 +63,11 @@ def test_config_as_dict(): # check default get behavior assert c.get('doesNotExist') is None assert c.get('doesNotExists', 'foobar') == 'foobar' + + +def test_config_key_error(): + """An unknown key should throw a key error""" + c = core.Config() + + with pytest.raises(KeyError): + c['doesNotExist'] diff --git a/ruins/tests/test_weather.py b/ruins/tests/test_weather.py index fc8eee1..968c003 100644 --- a/ruins/tests/test_weather.py +++ b/ruins/tests/test_weather.py @@ -3,19 +3,15 @@ from ruins.core import DataManager, Config - -# TODO use the config and inject the dedub config here - -# TODO only run this test when the Value Error is solved -#def test_run_app(): -# """Make sure the appp runs without failing""" -# weather.main_app() +# create the test config +config = get_test_config() def test_climate_indices(): """Test only climate indices """ - config = Config(include_climate=True) - dm = DataManager() + # add the include_climate config + config['include_climate'] = True + dm = DataManager(**config) # run weather.climate_indices(dataManager=dm, config=config)