diff --git a/tofu/data/_core.py b/tofu/data/_core.py index e7290bbdb..e5bdae09e 100644 --- a/tofu/data/_core.py +++ b/tofu/data/_core.py @@ -170,7 +170,7 @@ def __init__(self, data=None, t=None, X=None, lamb=None, dchans=None, dlabels=None, dX12='geom', Id=None, Name=None, Exp=None, shot=None, Diag=None, dextra=None, lCam=None, config=None, - fromdict=None, SavePath=os.path.abspath('./'), + fromdict=None, sep=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude): # To replace __init_subclass__ for Python 2 @@ -2278,7 +2278,7 @@ def __init_subclass__(cls, **kwdargs): def __init__(self, dtime=None, dradius=None, d0d=None, d1d=None, d2d=None, dmesh=None, config=None, Id=None, Name=None, Exp=None, shot=None, - fromdict=None, SavePath=os.path.abspath('./'), + fromdict=None, sep=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude): # To replace __init_subclass__ for Python 2 diff --git a/tofu/geom/_core.py b/tofu/geom/_core.py index d29b4f61d..7614c7756 100644 --- a/tofu/geom/_core.py +++ b/tofu/geom/_core.py @@ -163,7 +163,7 @@ def __init__(self, Poly=None, Type=None, Id=None, Name=None, Exp=None, shot=None, sino_RefPt=None, sino_nP=_def.TorNP, Clock=False, arrayorder='C', fromdict=None, - SavePath=os.path.abspath('./'), + sep=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude, color=None): # To replace __init_subclass__ for Python 2 @@ -1686,7 +1686,7 @@ def __init__(self, lStruct=None, Lim=None, dextraprop=None, Id=None, Name=None, Exp=None, shot=None, Type=None, SavePath=os.path.abspath('./'), SavePath_Include=tfpf.defInclude, - fromdict=None): + fromdict=None, sep=None): # To replace __init_subclass__ for Python 2 if sys.version[0]=='2': @@ -2958,7 +2958,7 @@ def _set_color_ddef(cls, color): def __init__(self, dgeom=None, lOptics=None, Etendues=None, Surfaces=None, config=None, dchans=None, dX12='geom', Id=None, Name=None, Exp=None, shot=None, Diag=None, - sino_RefPt=None, fromdict=None, method='optimized', + sino_RefPt=None, fromdict=None, sep=None, method='optimized', SavePath=os.path.abspath('./'), color=None, plotdebug=True): # To replace __init_subclass__ for Python 2 diff --git a/tofu/utils.py b/tofu/utils.py index 7679c6864..9b78bde8b 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -22,7 +22,7 @@ from tofu import __version__ import tofu.pathfile as tfpf -_sep = '_' +_SEP = '.' _dict_lexcept_key = [] _pyv = int(sys.version[0]) @@ -126,18 +126,17 @@ def get_figuresize(fs, fsdef=(12,6), return fs - - - - ############################################# # todict formatting ############################################# -def flatten_dict(d, parent_key='', sep=_sep, deep='ref', +def flatten_dict(d, parent_key='', sep=None, deep='ref', lexcept_key=_dict_lexcept_key): + if sep is None: + sep = _SEP + items = [] lexcept_key = [] if lexcept_key is None else lexcept_key for k, v in d.items(): @@ -155,7 +154,10 @@ def flatten_dict(d, parent_key='', sep=_sep, deep='ref', items.append((new_key, v)) return dict(items) -def _reshape_dict(ss, vv, dinit={}, sep=_sep): + +def _reshape_dict(ss, vv, dinit={}, sep=None): + if sep is None: + sep = _SEP ls = ss.split(sep) k = ss if len(ls)==1 else ls[0] if len(ls) == 2: @@ -172,7 +174,11 @@ def _reshape_dict(ss, vv, dinit={}, sep=_sep): assert k not in dinit.keys() dinit[k] = vv -def reshape_dict(d, sep=_sep, lcls=[]): + +def reshape_dict(d, sep=None, lcls=[]): + if sep is None: + sep = _SEP + # Get all individual keys out = {} for ss, vv in d.items(): @@ -212,18 +218,16 @@ def __dir__(self): return [str(k) for k in self.keys()]+self._extra - - ############################################# # Miscellaneous ############################################# -def _set_arrayorder(obj, arrayorder='C'): +def _set_arrayorder(obj, arrayorder='C', sep=None): """ Set the memory order of all np.ndarrays in a tofu object """ msg = "Arg arrayorder must be in ['C','F']" assert arrayorder in ['C','F'], msg - d = obj.to_dict(strip=-1) + d = obj.to_dict(strip=-1, sep=sep) account = {'Success':[], 'Failed':[]} for k, v in d.items(): if type(v) is np.array and v.ndim>1: @@ -240,12 +244,11 @@ def _set_arrayorder(obj, arrayorder='C'): return d, account - ############################################# # save / load ############################################# -def save(obj, path=None, name=None, sep=_sep, deep=False, mode='npz', +def save(obj, path=None, name=None, sep=None, deep=False, mode='npz', strip=None, compressed=False, verb=True, return_pfe=False): """ Save the ToFu object @@ -317,14 +320,16 @@ def save(obj, path=None, name=None, sep=_sep, deep=False, mode='npz', # Get stripped dictionnary deep = 'dict' if deep else 'ref' + if sep is None: + sep = _SEP dd = obj.to_dict(strip=strip, sep=sep, deep=deep) pathfileext = os.path.join(path,name+'.'+mode) if mode=='npz': - _save_npz(dd, pathfileext, compressed=compressed) + _save_npz(dd, pathfileext, sep=sep, compressed=compressed) elif mode=='mat': - _save_mat(dd, pathfileext, compressed=compressed) + _save_mat(dd, pathfileext, sep=sep, compressed=compressed) # print if verb: @@ -335,9 +340,10 @@ def save(obj, path=None, name=None, sep=_sep, deep=False, mode='npz', return pathfileext -def _save_npzmat_dict(dd): +def _save_npzmat_dict(dd, sep=None): + key = 'dId{0}dall{0}SaveName'.format(sep) msg = "How to deal with:" - msg += "\n SaveName : {0}".format(dd['dId_dall_SaveName']) + msg += "\n SaveName : {0}".format(dd[key]) msg += "\n Attributes:" err = False dnpzmat, dt = {}, {} @@ -368,14 +374,15 @@ def _save_npzmat_dict(dd): return dnpzmat -def _save_npz(dd, pathfileext, compressed=False): +def _save_npz(dd, pathfileext, sep=None, compressed=False): func = np.savez_compressed if compressed else np.savez - dsave = _save_npzmat_dict(dd) + dsave = _save_npzmat_dict(dd, sep=sep) func(pathfileext, **dsave) -def _save_mat(dd, pathfileext, compressed=False): + +def _save_mat(dd, pathfileext, sep=None, compressed=False): # Create intermediate dict to make sure to get rid of None values - dsave = _save_npzmat_dict(dd) + dsave = _save_npzmat_dict(dd, sep=sep) scpio.savemat(pathfileext, dsave, do_compression=compressed, format='5') @@ -474,9 +481,23 @@ def load(name, path=None, strip=None, verb=True): dd = _load_mat(pfe) # Recreate from dict - mod = importlib.import_module( 'tofu.%s'%dd['dId_dall_Mod'] ) - cls = getattr( mod, dd['dId_dall_Cls'] ) - obj = cls(fromdict=dd) + lsep, sep, keyMod = ['_', '.'], None, None + for ss in lsep: + key = 'dId{0}dall{0}Mod'.format(ss) + if key in dd.keys(): + sep = ss + keyMod = key + break + else: + msg = "No known separator in file keys!\n" + msg += " - separators tested: {0}\n".format(lsep) + msg += " - keys:\n" + msg += str(dd.keys()) + raise Exception(msg) + + mod = importlib.import_module('tofu.{0}'.format(dd[keyMod])) + cls = getattr(mod, dd['dId{0}dall{0}Cls'.format(sep)]) + obj = cls(fromdict=dd, sep=sep) if strip is not None: obj.strip(strip=strip) @@ -1274,7 +1295,7 @@ def __init__(self, fromdict=None, self._Done = False self._dstrip = self.__class__._dstrip.copy() if fromdict is not None: - self.from_dict(fromdict) + self.from_dict(fromdict, sep=kwdargs.get('sep', None)) else: self._reset() self._set_Id(**kwdargs) @@ -1315,15 +1336,15 @@ def _extract_kwdargs(din, largs): dout[k] = din[k] return dout - def _set_arrayorder(self, arrayorder='C', verb=True): - d, account = _set_arrayorder(self, arrayorder=arrayorder) + def _set_arrayorder(self, arrayorder='C', sep=None, verb=True): + d, account = _set_arrayorder(self, arrayorder=arrayorder, sep=sep) if len(account['Failed'])>0: msg = "All np.ndarrays were not set to {0} :\n".format(arrayorder) msg += "Success : [{0}]".format(', '.join(account['Success'])) msg += "Failed : [{0}]".format(', '.join(account['Failed'])) raise Exception(msg) else: - self.from_dict(d) + self.from_dict(d, sep=sep) self._dextra['arrayorder'] = arrayorder @staticmethod @@ -1436,7 +1457,6 @@ def __repr__(self): return object.__repr__(self) - ############################# # strip and to/from dict ############################# @@ -1474,8 +1494,7 @@ def strip(self, strip=0, **kwdargs): self._dstrip['strip'] = strip - - def to_dict(self, strip=None, sep=_sep, deep='ref'): + def to_dict(self, strip=None, sep=None, deep='ref'): """ Return a flat dict view of the object's attributes Useful for: @@ -1543,7 +1562,7 @@ def _get_dId(self): """ To be overloaded """ return {'dict':{}} - def from_dict(self, fd, sep=_sep, strip=None): + def from_dict(self, fd, sep=None, strip=None): """ Populate the instances attributes using an input dict The input dict must be properly formatted @@ -1561,7 +1580,7 @@ def from_dict(self, fd, sep=_sep, strip=None): """ self._reset() - dd = reshape_dict(fd) + dd = reshape_dict(fd, sep=sep) # --------------------- # Call class-specific @@ -1569,7 +1588,7 @@ def from_dict(self, fd, sep=_sep, strip=None): # --------------------- self._dstrip.update(**dd['dstrip']) if 'dId' in dd.keys(): - self._set_Id(Id=ID(fromdict=dd['dId'])) + self._set_Id(Id=ID(fromdict=dd['dId'], sep=sep)) if strip is None: strip = self._dstrip['strip'] @@ -1840,7 +1859,7 @@ def _get_ind12r_n12(ind1=None, ind2=None, n1=None, n2=None): return ind1, ind2, indr def save(self, path=None, name=None, - strip=None, sep=_sep, deep=True, mode='npz', + strip=None, sep=None, deep=True, mode='npz', compressed=False, verb=True, return_pfe=False): return save(self, path=path, name=name, sep=sep, deep=deep, mode=mode, @@ -1908,7 +1927,7 @@ class ID(ToFuObjectBase): def __init__(self, Cls=None, Name=None, Type=None, Deg=None, Exp=None, Diag=None, shot=None, SaveName=None, SavePath=None, usr=None, dUSR=None, lObj=None, - fromdict=None, include=None): + fromdict=None, include=None, sep=None): # To replace __init_subclass__ for Python 2 if sys.version[0]=='2': diff --git a/tofu/version.py b/tofu/version.py index cf59f8275..1c8be5ab1 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.1-145-ge5ffef4' +__version__ = '1.4.1-152-g2211e30'