From f9a4b880deb2b1bf79da7d7f30b7beb0c9de23ff Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 4 Nov 2019 13:53:18 +0100 Subject: [PATCH 1/6] [Issue228] Almost done, one bug remaining --- tofu/data/_core.py | 4 +-- tofu/geom/_core.py | 6 ++-- tofu/utils.py | 79 ++++++++++++++++++++++++++++++---------------- 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/tofu/data/_core.py b/tofu/data/_core.py index e7290bbdb..e5bdae09e 100644 --- a/tofu/data/_core.py +++ b/tofu/data/_core.py @@ -170,7 +170,7 @@ def __init__(self, data=None, t=None, X=None, lamb=None, dchans=None, dlabels=None, dX12='geom', Id=None, Name=None, Exp=None, shot=None, Diag=None, dextra=None, lCam=None, config=None, - fromdict=None, SavePath=os.path.abspath('./'), + fromdict=None, sep=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude): # To replace __init_subclass__ for Python 2 @@ -2278,7 +2278,7 @@ def __init_subclass__(cls, **kwdargs): def __init__(self, dtime=None, dradius=None, d0d=None, d1d=None, d2d=None, dmesh=None, config=None, Id=None, Name=None, Exp=None, shot=None, - fromdict=None, SavePath=os.path.abspath('./'), + fromdict=None, sep=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude): # To replace __init_subclass__ for Python 2 diff --git a/tofu/geom/_core.py b/tofu/geom/_core.py index d29b4f61d..7614c7756 100644 --- a/tofu/geom/_core.py +++ b/tofu/geom/_core.py @@ -163,7 +163,7 @@ def __init__(self, Poly=None, Type=None, Id=None, Name=None, Exp=None, shot=None, sino_RefPt=None, sino_nP=_def.TorNP, Clock=False, arrayorder='C', fromdict=None, - SavePath=os.path.abspath('./'), + sep=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude, color=None): # To replace __init_subclass__ for Python 2 @@ -1686,7 +1686,7 @@ def __init__(self, lStruct=None, Lim=None, dextraprop=None, Id=None, Name=None, Exp=None, shot=None, Type=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude, - fromdict=None): + fromdict=None, sep=None): # To replace __init_subclass__ for Python 2 if sys.version[0]=='2': @@ -2958,7 +2958,7 @@ def _set_color_ddef(cls, color): def __init__(self, dgeom=None, lOptics=None, Etendues=None, Surfaces=None, config=None, dchans=None, dX12='geom', Id=None, Name=None, Exp=None, shot=None, Diag=None, - sino_RefPt=None, fromdict=None, method='optimized', + sino_RefPt=None, fromdict=None, sep=None, method='optimized', SavePath=os.path.abspath('./'), color=None, plotdebug=True): # To replace __init_subclass__ for Python 2 diff --git a/tofu/utils.py b/tofu/utils.py index 7679c6864..ef45b4887 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -22,7 +22,7 @@ from tofu import __version__ import tofu.pathfile as tfpf -_sep = '_' +_SEP = '.' _dict_lexcept_key = [] _pyv = int(sys.version[0]) @@ -135,9 +135,12 @@ def get_figuresize(fs, fsdef=(12,6), ############################################# -def flatten_dict(d, parent_key='', sep=_sep, deep='ref', +def flatten_dict(d, parent_key='', sep=None, deep='ref', lexcept_key=_dict_lexcept_key): + if sep is None: + sep = _SEP + items = [] lexcept_key = [] if lexcept_key is None else lexcept_key for k, v in d.items(): @@ -155,7 +158,9 @@ def flatten_dict(d, parent_key='', sep=_sep, deep='ref', items.append((new_key, v)) return dict(items) -def _reshape_dict(ss, vv, dinit={}, sep=_sep): +def _reshape_dict(ss, vv, dinit={}, sep=None): + if sep is None: + sep = _SEP ls = ss.split(sep) k = ss if len(ls)==1 else ls[0] if len(ls) == 2: @@ -172,7 +177,10 @@ def _reshape_dict(ss, vv, dinit={}, sep=_sep): assert k not in dinit.keys() dinit[k] = vv -def reshape_dict(d, sep=_sep, lcls=[]): +def reshape_dict(d, sep=None, lcls=[]): + if sep is None: + sep = _SEP + # Get all individual keys out = {} for ss, vv in d.items(): @@ -218,12 +226,12 @@ def __dir__(self): # Miscellaneous ############################################# -def _set_arrayorder(obj, arrayorder='C'): +def _set_arrayorder(obj, arrayorder='C', sep=None): """ Set the memory order of all np.ndarrays in a tofu object """ msg = "Arg arrayorder must be in ['C','F']" assert arrayorder in ['C','F'], msg - d = obj.to_dict(strip=-1) + d = obj.to_dict(strip=-1, sep=sep) account = {'Success':[], 'Failed':[]} for k, v in d.items(): if type(v) is np.array and v.ndim>1: @@ -245,7 +253,7 @@ def _set_arrayorder(obj, arrayorder='C'): # save / load ############################################# -def save(obj, path=None, name=None, sep=_sep, deep=False, mode='npz', +def save(obj, path=None, name=None, sep=None, deep=False, mode='npz', strip=None, compressed=False, verb=True, return_pfe=False): """ Save the ToFu object @@ -322,9 +330,9 @@ def save(obj, path=None, name=None, sep=_sep, deep=False, mode='npz', pathfileext = os.path.join(path,name+'.'+mode) if mode=='npz': - _save_npz(dd, pathfileext, compressed=compressed) + _save_npz(dd, pathfileext, sep=sep, compressed=compressed) elif mode=='mat': - _save_mat(dd, pathfileext, compressed=compressed) + _save_mat(dd, pathfileext, sep=sep, compressed=compressed) # print if verb: @@ -335,9 +343,10 @@ def save(obj, path=None, name=None, sep=_sep, deep=False, mode='npz', return pathfileext -def _save_npzmat_dict(dd): +def _save_npzmat_dict(dd, sep=None): + key = 'dId%sdall%sSaveName'%(sep, sep) msg = "How to deal with:" - msg += "\n SaveName : {0}".format(dd['dId_dall_SaveName']) + msg += "\n SaveName : {0}".format(dd[key]) msg += "\n Attributes:" err = False dnpzmat, dt = {}, {} @@ -368,14 +377,14 @@ def _save_npzmat_dict(dd): return dnpzmat -def _save_npz(dd, pathfileext, compressed=False): +def _save_npz(dd, pathfileext, sep=None, compressed=False): func = np.savez_compressed if compressed else np.savez - dsave = _save_npzmat_dict(dd) + dsave = _save_npzmat_dict(dd, sep=sep) func(pathfileext, **dsave) -def _save_mat(dd, pathfileext, compressed=False): +def _save_mat(dd, pathfileext, sep=None, compressed=False): # Create intermediate dict to make sure to get rid of None values - dsave = _save_npzmat_dict(dd) + dsave = _save_npzmat_dict(dd, sep=sep) scpio.savemat(pathfileext, dsave, do_compression=compressed, format='5') @@ -474,9 +483,23 @@ def load(name, path=None, strip=None, verb=True): dd = _load_mat(pfe) # Recreate from dict - mod = importlib.import_module( 'tofu.%s'%dd['dId_dall_Mod'] ) - cls = getattr( mod, dd['dId_dall_Cls'] ) - obj = cls(fromdict=dd) + lsep, sep, keyMod = ['_', '.'], None, None + for ss in lsep: + key = 'dId{0}dall{0}Mod'.format(ss) + if key in dd.keys(): + sep = ss + keyMod = key + break + else: + msg = "No known separator in file keys!\n" + msg += " - separators tested: %s\n"%str(lsep) + msg += " - keys:\n" + msg += str(dd.keys()) + raise Exception(msg) + + mod = importlib.import_module( 'tofu.%s'%dd[keyMod] ) + cls = getattr( mod, dd['dId%sdall%sCls'%(sep, sep)] ) + obj = cls(fromdict=dd, sep=sep) if strip is not None: obj.strip(strip=strip) @@ -1274,7 +1297,7 @@ def __init__(self, fromdict=None, self._Done = False self._dstrip = self.__class__._dstrip.copy() if fromdict is not None: - self.from_dict(fromdict) + self.from_dict(fromdict, sep=kwdargs.get('sep', None)) else: self._reset() self._set_Id(**kwdargs) @@ -1315,15 +1338,15 @@ def _extract_kwdargs(din, largs): dout[k] = din[k] return dout - def _set_arrayorder(self, arrayorder='C', verb=True): - d, account = _set_arrayorder(self, arrayorder=arrayorder) + def _set_arrayorder(self, arrayorder='C', sep=None, verb=True): + d, account = _set_arrayorder(self, arrayorder=arrayorder, sep=sep) if len(account['Failed'])>0: msg = "All np.ndarrays were not set to {0} :\n".format(arrayorder) msg += "Success : [{0}]".format(', '.join(account['Success'])) msg += "Failed : [{0}]".format(', '.join(account['Failed'])) raise Exception(msg) else: - self.from_dict(d) + self.from_dict(d, sep=sep) self._dextra['arrayorder'] = arrayorder @staticmethod @@ -1475,7 +1498,7 @@ def strip(self, strip=0, **kwdargs): self._dstrip['strip'] = strip - def to_dict(self, strip=None, sep=_sep, deep='ref'): + def to_dict(self, strip=None, sep=None, deep='ref'): """ Return a flat dict view of the object's attributes Useful for: @@ -1543,7 +1566,7 @@ def _get_dId(self): """ To be overloaded """ return {'dict':{}} - def from_dict(self, fd, sep=_sep, strip=None): + def from_dict(self, fd, sep=None, strip=None): """ Populate the instances attributes using an input dict The input dict must be properly formatted @@ -1561,7 +1584,7 @@ def from_dict(self, fd, sep=_sep, strip=None): """ self._reset() - dd = reshape_dict(fd) + dd = reshape_dict(fd, sep=sep) # --------------------- # Call class-specific @@ -1569,7 +1592,7 @@ def from_dict(self, fd, sep=_sep, strip=None): # --------------------- self._dstrip.update(**dd['dstrip']) if 'dId' in dd.keys(): - self._set_Id(Id=ID(fromdict=dd['dId'])) + self._set_Id(Id=ID(fromdict=dd['dId'], sep=sep)) if strip is None: strip = self._dstrip['strip'] @@ -1840,7 +1863,7 @@ def _get_ind12r_n12(ind1=None, ind2=None, n1=None, n2=None): return ind1, ind2, indr def save(self, path=None, name=None, - strip=None, sep=_sep, deep=True, mode='npz', + strip=None, sep=None, deep=True, mode='npz', compressed=False, verb=True, return_pfe=False): return save(self, path=path, name=name, sep=sep, deep=deep, mode=mode, @@ -1908,7 +1931,7 @@ class ID(ToFuObjectBase): def __init__(self, Cls=None, Name=None, Type=None, Deg=None, Exp=None, Diag=None, shot=None, SaveName=None, SavePath=None, usr=None, dUSR=None, lObj=None, - fromdict=None, include=None): + fromdict=None, include=None, sep=None): # To replace __init_subclass__ for Python 2 if sys.version[0]=='2': From 73c1915d17958e9936465186646c3055b53aabcd Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 4 Nov 2019 14:04:47 +0100 Subject: [PATCH 2/6] [Issue228] Fixed --- tofu/utils.py | 2 ++ tofu/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tofu/utils.py b/tofu/utils.py index ef45b4887..25a873679 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -325,6 +325,8 @@ def save(obj, path=None, name=None, sep=None, deep=False, mode='npz', # Get stripped dictionnary deep = 'dict' if deep else 'ref' + if sep is None: + sep = _SEP dd = obj.to_dict(strip=strip, sep=sep, deep=deep) pathfileext = os.path.join(path,name+'.'+mode) diff --git a/tofu/version.py b/tofu/version.py index cf59f8275..982eb5090 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.1-145-ge5ffef4' +__version__ = '1.4.1-148-gf9a4b88' From 22576b20f6aa0e0c79ac4504bcb90e73a11ff2a2 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 4 Nov 2019 14:08:07 +0100 Subject: [PATCH 3/6] [Issue228] PEP8 compliance 1 --- tofu/utils.py | 4 ---- tofu/version.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tofu/utils.py b/tofu/utils.py index 25a873679..a99b1bc1c 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -126,10 +126,6 @@ def get_figuresize(fs, fsdef=(12,6), return fs - - - - ############################################# # todict formatting ############################################# diff --git a/tofu/version.py b/tofu/version.py index 982eb5090..79ebf9cfe 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.1-148-gf9a4b88' +__version__ = '1.4.1-149-g73c1915' From a72b7f6d62862cd2fd6a7072eb358284a6ea0586 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 4 Nov 2019 14:11:11 +0100 Subject: [PATCH 4/6] [Issue228] PEP8 compliance 2 --- tofu/utils.py | 10 ++++------ tofu/version.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tofu/utils.py b/tofu/utils.py index a99b1bc1c..9eefb00ca 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -154,6 +154,7 @@ def flatten_dict(d, parent_key='', sep=None, deep='ref', items.append((new_key, v)) return dict(items) + def _reshape_dict(ss, vv, dinit={}, sep=None): if sep is None: sep = _SEP @@ -173,6 +174,7 @@ def _reshape_dict(ss, vv, dinit={}, sep=None): assert k not in dinit.keys() dinit[k] = vv + def reshape_dict(d, sep=None, lcls=[]): if sep is None: sep = _SEP @@ -216,8 +218,6 @@ def __dir__(self): return [str(k) for k in self.keys()]+self._extra - - ############################################# # Miscellaneous ############################################# @@ -244,7 +244,6 @@ def _set_arrayorder(obj, arrayorder='C', sep=None): return d, account - ############################################# # save / load ############################################# @@ -495,8 +494,8 @@ def load(name, path=None, strip=None, verb=True): msg += str(dd.keys()) raise Exception(msg) - mod = importlib.import_module( 'tofu.%s'%dd[keyMod] ) - cls = getattr( mod, dd['dId%sdall%sCls'%(sep, sep)] ) + mod = importlib.import_module('tofu.%s'%dd[keyMod]) + cls = getattr(mod, dd['dId%sdall%sCls'%(sep, sep)]) obj = cls(fromdict=dd, sep=sep) if strip is not None: @@ -1457,7 +1456,6 @@ def __repr__(self): return object.__repr__(self) - ############################# # strip and to/from dict ############################# diff --git a/tofu/version.py b/tofu/version.py index 79ebf9cfe..ea29a6a39 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.1-149-g73c1915' +__version__ = '1.4.1-150-g22576b2' From 2211e300b5c00f1bcacd37ba9006c4a7ba47d8ec Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 4 Nov 2019 14:16:31 +0100 Subject: [PATCH 5/6] [Issue228] PEP8 compliance 3 --- tofu/utils.py | 8 ++++---- tofu/version.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tofu/utils.py b/tofu/utils.py index 9eefb00ca..09c518570 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -379,6 +379,7 @@ def _save_npz(dd, pathfileext, sep=None, compressed=False): dsave = _save_npzmat_dict(dd, sep=sep) func(pathfileext, **dsave) + def _save_mat(dd, pathfileext, sep=None, compressed=False): # Create intermediate dict to make sure to get rid of None values dsave = _save_npzmat_dict(dd, sep=sep) @@ -489,13 +490,13 @@ def load(name, path=None, strip=None, verb=True): break else: msg = "No known separator in file keys!\n" - msg += " - separators tested: %s\n"%str(lsep) + msg += " - separators tested: {0}\n".format(lsep) msg += " - keys:\n" msg += str(dd.keys()) raise Exception(msg) - mod = importlib.import_module('tofu.%s'%dd[keyMod]) - cls = getattr(mod, dd['dId%sdall%sCls'%(sep, sep)]) + mod = importlib.import_module('tofu.{0}'.format(dd[keyMod])) + cls = getattr(mod, dd['dId{0}dall{0}Cls'.format(sep)]) obj = cls(fromdict=dd, sep=sep) if strip is not None: @@ -1493,7 +1494,6 @@ def strip(self, strip=0, **kwdargs): self._dstrip['strip'] = strip - def to_dict(self, strip=None, sep=None, deep='ref'): """ Return a flat dict view of the object's attributes diff --git a/tofu/version.py b/tofu/version.py index ea29a6a39..0b13c0b4d 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.1-150-g22576b2' +__version__ = '1.4.1-151-ga72b7f6' From 9f15f9ce1c74d1c4a5b2b68997880d914e582865 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 4 Nov 2019 14:18:43 +0100 Subject: [PATCH 6/6] [Issue228] PEP8 compliance 4 --- tofu/utils.py | 2 +- tofu/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tofu/utils.py b/tofu/utils.py index 09c518570..9b78bde8b 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -341,7 +341,7 @@ def save(obj, path=None, name=None, sep=None, deep=False, mode='npz', def _save_npzmat_dict(dd, sep=None): - key = 'dId%sdall%sSaveName'%(sep, sep) + key = 'dId{0}dall{0}SaveName'.format(sep) msg = "How to deal with:" msg += "\n SaveName : {0}".format(dd[key]) msg += "\n Attributes:" diff --git a/tofu/version.py b/tofu/version.py index 0b13c0b4d..1c8be5ab1 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.1-151-ga72b7f6' +__version__ = '1.4.1-152-g2211e30'