From 03b6311af9908ef772e7ede89adc70fdb6a427f5 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 27 Feb 2020 15:23:38 +0100 Subject: [PATCH 01/22] [Issue356] Created openadas2tofu sub-package with __init__.py --- tofu/openadas2tofu/__init__.py | 99 ++++++++++++++++++++++++++++++++++ tofu/version.py | 2 +- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tofu/openadas2tofu/__init__.py diff --git a/tofu/openadas2tofu/__init__.py b/tofu/openadas2tofu/__init__.py new file mode 100644 index 000000000..c5bf060e5 --- /dev/null +++ b/tofu/openadas2tofu/__init__.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +#! /usr/bin/python +""" +The openadas-compatibility module of tofu + +""" +# import warnings +# import traceback +# import itertools as itt + +# try: + # try: + # from tofu.imas2tofu._core import * + # from tofu.imas2tofu._mat2ids2calc import * + # except Exception: + # from ._core import * + # from ._mat2ids2calc import * +# except Exception as err: + # if str(err) == 'imas not available': + # msg = "" + # msg += "\n\nIMAS python API issue\n" + # msg += "imas could not be imported into tofu ('import imas' failed):\n" + # msg += " - it may not be installed (optional dependency)\n" + # msg += " - or you have loaded the wrong working environment\n\n" + # msg += " => the optional sub-package tofu.imas2tofu is not usable\n" + # else: + # msg = str(traceback.format_exc()) + # msg += "\n\n => the optional sub-package tofu.imas2tofu is not usable\n" + # raise Exception(msg) + + +# # ----------------------------------------------- +# # Check IMAS version vs latest available in linux modules +# # ----------------------------------------------- + +# _KEYSTR = 'IMAS/' + + +# # extract all IMAS versions from a str returned by modules +# def extractIMAS(ss, keystr=_KEYSTR): + # if keystr not in ss: + # raise Exception + # ls = ss[ss.index(keystr):].split('\n') + # ls = itt.chain.from_iterable([s.split(' ') for s in ls]) + # ls = [s for s in ls if keystr in s] + # ls = [s[len(keystr):s.index('(')] if '(' in s else s[len(keystr):] + # for s in ls] + # return sorted(ls) + + +# # Compare current and latest available IMAS versions +# def check_IMAS_version(verb=True, keystr=_KEYSTR): + # import subprocess + + # # Get currently loaded IMAS + # cmd = "module list" + # proc = subprocess.run(cmd, check=True, shell=True, + # stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + # lcur = extractIMAS(proc.stdout.decode(), keystr=keystr) + # if len(lcur) != 1: + # msg = ("You seem to have no / several IMAS version loaded:\n" + # + "\t- module list: {}".format(lcur)) + # raise Exception(msg) + + # # Get all available IMAS + # cmd = "module av IMAS" + # proc = subprocess.run(cmd, check=True, shell=True, + # stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + # lav = extractIMAS(proc.stdout.decode(), keystr=keystr) + # if len(lav) == 0: + # msg = "There is not available IMAS version" + # raise Exception(msg) + + # # Compare and warn + # if lcur[0] not in lav: + # msg = "The current IMAS version is not available!" + # raise Exception(msg) + + # msg = None + # if lav.index(lcur[0]) != len(lav)-1: + # msg = ("\nYou do not seem to be using the latest IMAS version:\n" + # + "'module list' vs 'module av IMAS' suggests:\n" + # + "\t- Current version: {}\n".format(lcur[0]) + # + "\t- Latest version : {}".format(lav[-1])) + # warnings.warn(msg) + # return lcur[0], lav + + +# # Try comparing and warning +# try: + # _, _ = check_IMAS_version(verb=True) +# except Exception as err: + # # This warning is an optional luxury, should not block anything + # pass + + +# __all__ = ['MultiIDSLoader', 'load_Config', 'load_Plasma2D', + # 'load_Cam', 'load_Data'] +# del warnings, traceback, itt, _KEYSTR diff --git a/tofu/version.py b/tofu/version.py index 2ee9c73d6..6fd52b9fe 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-56-g4622b310' +__version__ = '1.4.2b4-61-g1a5ca76a' From cb0c53f2bbe4daf9e09fcb4dfcca2eda23782ff9 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 27 Feb 2020 16:24:36 +0100 Subject: [PATCH 02/22] [Issue356] updated version --- tofu/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tofu/version.py b/tofu/version.py index 6fd52b9fe..591dcbb72 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-61-g1a5ca76a' +__version__ = '1.4.2b4-62-g03b6311a' From 7f7a339c881beb138b3ec419caec994bc9fda1f2 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 27 Feb 2020 18:11:04 +0100 Subject: [PATCH 03/22] [Issue356] Advanced search() --- tofu/openadas2tofu/_requests.py | 72 +++++++++++++++++++++++++++++++++ tofu/version.py | 2 +- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tofu/openadas2tofu/_requests.py diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py new file mode 100644 index 000000000..e9f085676 --- /dev/null +++ b/tofu/openadas2tofu/_requests.py @@ -0,0 +1,72 @@ + +# Built-in +import os +import requests + + + +_URL = 'https://open.adas.ac.uk' +_URL_SEARCH = _URL + '/freeform?searchstring=' +_URL_ADF15 = _URL + '/adf15' + + + +def _checkformat_target(target): + return target + + +def search(searchstr=None, + element=None, charge=None, type=None): + + # Check input + if searchstr is None: + pass + searchurl = '+'.join([requests.utils.quote(kk) + for kk in searchstr.split(' ')]) + + resp = requests.get('{}{}{}'.format(_URL_SEARCH, + searchurl, + '&searching=1')) + + # Extract response from html + out = resp.text.split('\n') + ind0 = out.index('') + out = out[ind0+1:] + ind1 = out.index('
') + out = out[:ind1-1] + nresults = len(out) -1 + + import pdb; pdb.set_trace() # DB + heads = [str.replace(kk.replace('', '').replace('', ''), + '', '').replace('', '') + for kk in out[0].split('')] + nhead = len(heads) + import pdb; pdb.set_trace() # DB + + + # TBF + char = np.chararray((nresults, nhead)) + + for ii in range(0, nresults): + char[ii, :] = out[ii+1] + + + + +def list_PEC(ion=None, charge=None, lamb=None): + + lfn = [] + resp = requests.get(_URL) + + + +def download_PEC(ion=None, charge=None, lamb=None, + target=None): + + # --------------------------- + # Check + target = _checkformat_target(target) + + # --------------------------- + # Download + resp = requests.get() diff --git a/tofu/version.py b/tofu/version.py index 591dcbb72..3d0e10fb1 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-62-g03b6311a' +__version__ = '1.4.2b4-63-gcb0c53f2' From 687249c27e8cde5b4ba1e8fe30be53e7429e5a42 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 28 Feb 2020 13:25:55 +0100 Subject: [PATCH 04/22] [Issue356] Web interface operational for search, download, download_all and clean_downloads, TODO: add search_by_wavelength() and start routines for exploring local database --- tofu/openadas2tofu/_requests.py | 240 +++++++++++++++++++++++++++++--- tofu/version.py | 2 +- 2 files changed, 221 insertions(+), 21 deletions(-) diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index e9f085676..55e7a08fe 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -1,26 +1,93 @@ # Built-in import os +import shutil import requests +import warnings +# Common +import numpy as np - +# Check whether a local .tofu/ repo exists +pfe = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') +if os.path.isdir(pfe): + _PATH_LOCAL = pfe +else: + _PATH_LOCAL = None _URL = 'https://open.adas.ac.uk' _URL_SEARCH = _URL + '/freeform?searchstring=' _URL_ADF15 = _URL + '/adf15' +_URL_DOWNLOAD = _URL + '/download' + +_INCLUDE_PARTIAL = True + +# ############################################################################# +# Utility functions +# ############################################################################# + + +def _getcharray(ar, col=None, sep=' ', line='-', just='l', + verb=True, returnas=str): + """ Format and return char array (for pretty printing) """ + c0 = ar is None or len(ar) == 0 + if c0: + return '' + ar = np.array(ar, dtype='U') + + if ar.ndim == 1: + ar = ar.reshape((1,ar.size)) + # Get just len + nn = np.char.str_len(ar).max(axis=0) + if col is not None: + if len(col) not in ar.shape: + msg = ("len(col) should be in np.array(ar, dtype='U').shape:\n" + + "\t- len(col) = {}\n".format(len(col)) + + "\t- ar.shape = {}".format(ar.shape)) + raise Exception(msg) + if len(col) != ar.shape[1]: + ar = ar.T + nn = np.char.str_len(ar).max(axis=0) + nn = np.fmax(nn, [len(cc) for cc in col]) + # Apply to array + fjust = np.char.ljust if just == 'l' else np.char.rjust + out = np.array([sep.join(v) for v in fjust(ar,nn)]) -def _checkformat_target(target): - return target + # Apply to col + if col is not None: + arcol = np.array([col, [line*n for n in nn]], dtype='U') + arcol = np.array([sep.join(v) for v in fjust(arcol,nn)]) + out = np.append(arcol,out) + if verb is True: + print('\n'.join(out)) + if returnas is str: + return '\n'.join(out) + elif returnas is np.ndarray: + return ar -def search(searchstr=None, - element=None, charge=None, type=None): + +# ############################################################################# +# Web-oriented functions +# ############################################################################# + +class FileAlreayExistsException(Exception): + pass + + +def search(searchstr=None, returnas=None, + include_partial=None, verb=None): # Check input + if returnas is None: + returnas = False + if verb is None: + verb = True + if include_partial is None: + include_partial = _INCLUDE_PARTIAL if searchstr is None: - pass + searchstr = '' searchurl = '+'.join([requests.utils.quote(kk) for kk in searchstr.split(' ')]) @@ -36,37 +103,170 @@ def search(searchstr=None, out = out[:ind1-1] nresults = len(out) -1 - import pdb; pdb.set_trace() # DB heads = [str.replace(kk.replace('', '').replace('', ''), '', '').replace('', '') for kk in out[0].split('')] nhead = len(heads) - import pdb; pdb.set_trace() # DB - - - # TBF - char = np.chararray((nresults, nhead)) + lout = [] for ii in range(0, nresults): - char[ii, :] = out[ii+1] + if 'Partial results are listed below' in out[ii+1]: + if include_partial is False: + break + lout.append(['-', '-', '----- (partial results) -----', '-']) + ind = out[ii+1].index('below') + len('below') + out[ii+1] = out[ii+1][ind:] + lstri = out[ii+1].split('', '') + elm = elmq[:elmq.index('')] + charge = elmq[elmq.index('')+len(''):] + charge = charge.replace('', '') + typ = lstri[2][1:] + fil = lstri[3][lstri[3].index('detail')+len('detail'):] + fil = fil[:fil.index('.dat')+len('.dat')] + lout.append([elm, charge, typ, fil]) + # Format output + char = np.array(lout) + col = ['Element', 'charge', 'type of data', 'full file name'] + arr = _getcharray(char, col=col, + sep=' ', line='-', just='l', + returnas=returnas, verb=verb) + return arr +def _check_exists(filename, update=None): + # if target is None: + # target = filename -def list_PEC(ion=None, charge=None, lamb=None): + # In case a small modification becomes necessary later + target = filename - lfn = [] - resp = requests.get(_URL) + # Check whether the local .tofu repo exists, if not recommend tofu-custom + if _PATH_LOCAL is None: + path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + msg = ("You do not seem to have a local ./tofu repository\n" + + "tofu uses that local repository to store all user-specific " + + "data and downloads\n" + + "In particular, openadas files are downloaded and saved in:\n" + + "\t{}\n".format(path) + + " => to set-up your local .tofu repo, run in a terminal:\n" + + "\ttofu-custom") + raise Exception(msg) + # Parse intermediate repos and create if necessary + lrep = target.split('/')[1:-1] + for ii in range(len(lrep)): + repo = os.path.join(_PATH_LOCAL, *lrep[:ii+1]) + if not os.path.isdir(repo): + os.mkdir(repo) + # Check if file already exists + path = os.path.join(_PATH_LOCAL, *lrep) + pfe = os.path.join(path, target.split('/')[-1]) + if os.path.isfile(pfe): + if update is False: + msg = ("File already exists in your local repo:\n" + + "\t{}\n".format(pfe) + + " => if you want to force download, use update=True" + + " (local file will be overwritten)") + raise FileAlreayExistsException(msg) + else: + return True, pfe + else: + return False, pfe -def download_PEC(ion=None, charge=None, lamb=None, - target=None): + +def download(filename=None, + update=None, verb=None, returnas=None): # --------------------------- # Check - target = _checkformat_target(target) + if verb is None: + verb = True + if update is None: + update = False + if returnas is None: + returnas = False + + if (not isinstance(filename, str) + or filename[:4] != '/adf' or filename[-4:] != '.dat'): + msg = ("filename must be a str (full file name) of the form:\n" + + "\t/adf.../.../....dat") + raise Exception(msg) + url = _URL_DOWNLOAD + filename + + exists, pfe = _check_exists(filename, update=update) + if exists is True and verb is True: + msg = ("File already exists, will be downloaded and overwritten:\n" + + "\t{}".format(pfe)) + warnings.warn(msg) # --------------------------- # Download - resp = requests.get() + # Note the stream=True parameter below + with requests.get(url, stream=True) as rr: + rr.raise_for_status() + with open(pfe, 'wb') as ff: + for chunk in rr.iter_content(chunk_size=8192): + # filter-out keep-alive new chunks + if chunk: + ff.write(chunk) + # ff.flush() + + if verb is True: + msg = ("file {} was copied to:\n".format(filename) + + "\t{}".format(pfe)) + print(msg) + if returnas is str: + return pfe + + +def download_all(searchstr=None, + include_partial=None, update=None, verb=None): + # Check + if include_partial is None: + include_partial = False + if update is None: + update = False + if verb is None: + verb = True + + # Download + arr = search(searchstr=searchstr, include_partial=include_partial, + verb=False, returnas=np.ndarray) + if verb is True: + msg = "Downloading from {} into {}:".format(_URL, _PATH_LOCAL) + print(msg) + for ii in range(arr.shape[0]): + try: + exists, pfe = _check_exists(arr[ii, 3], update=update) + pfe = download(filename=arr[ii, 3], update=update, + verb=False, returnas=str) + if exists is True: + msg = "\toverwritten: \t{}".format(arr[ii, 3]) + else: + msg = "\tdownloaded: \t{}".format(arr[ii, 3]) + except FileAlreayExistsException: + msg = "\talready exists: {}".format(arr[ii, 3]) + except Exception as err: + msg = (str(err) + + "\n\nCould not download file {}".format(arr[ii, 3])) + raise err + + if verb is True: + print(msg) + + +def clean_downloads(): + if _PATH_LOCAL is None: + return + lf = [ff for ff in os.listdir(_PATH_LOCAL) + if os.path.isfile(os.path.join(_PATH_LOCAL, ff))] + ld = [ff for ff in os.listdir(_PATH_LOCAL) + if os.path.isdir(os.path.join(_PATH_LOCAL, ff))] + for ff in lf: + os.remove(ff) + for dd in ld: + shutil.rmtree(os.path.join(_PATH_LOCAL, dd)) diff --git a/tofu/version.py b/tofu/version.py index 3d0e10fb1..c6b1a377f 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-63-gcb0c53f2' +__version__ = '1.4.2b4-64-g7f7a339c' From d28fd3daf63e653dc1450ea14f4ce9f613b43210 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 28 Feb 2020 18:17:13 +0100 Subject: [PATCH 05/22] [Issue356] Started reading routines, ADF15 to be finished and tested --- tofu/openadas2tofu/_read_files.py | 140 ++++++++++++++++++++++++++++++ tofu/version.py | 2 +- 2 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 tofu/openadas2tofu/_read_files.py diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py new file mode 100644 index 000000000..106ea3017 --- /dev/null +++ b/tofu/openadas2tofu/_read_files.py @@ -0,0 +1,140 @@ + +# Built-in +import os + +# Common +import numpy as np +from scipy.interpolate import RectBivariateSpline as scpRectSpline + + +_LTYPES = ['adf11', 'adf15'] +_DEG = 1 + + +def read(pfe): + """ Read openadas-formatted files and return a dict of interpolators """ + + if not os.path.isfile(pfe): + msg = ("Provided file does not seem to exist:\n" + + "\t{}\n".format(pfe) + + " => Search it online with tofu.openadas2tofu.search()\n" + + " => Download it locally with ofu.openadas2tofu.download()") + raise FileNotFoundError(msg) + + lc = [ss for ss in _LTYPES if ss in pfe] + if not len(lc) == 1: + msg = ("File type could not be derived from absolute path:\n" + + "\t- provided: {}\n".format(pfe) + + "\t- supported: {}".format(sorted(_LTYPES))) + raise Exception(msg) + + func = eval('_read_{}'.format(lc[0])) + return func(pfe) + + + +# ############################################################################# +# ADF 15 +# ############################################################################# + +def _read_adf11(): + pass + + +# ############################################################################# +# ADF 15 +# ############################################################################# + + +def _read_adf15(pfe, deg=None): + + if deg is None: + deg = _DEG + + # Get summary of transitions + flagblock = 'isel =' + flag0 = 'superstage partition information' + dout = {} + + + nlines, nblock = None, 0 + in_ne, in_te, in_pec, in_tab, itab = False, False, False, False, np.inf + with open(pfe) as search: + for ii, line in enumerate(search): + + # Get number of lines (transitions) stored in this file + if ii == 0: + nlines = int(line[:line.index('/AR')].replace(' ','')) + continue + + # Get info about the transition being scanned (block) + if flagblock in line and nblock < nlines: + lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] + lamb = float(lstr[0])*1.e-10 + nne, nte = int(lstr[1]), int(lstr[2]) + typ = [ss[ss.index('type=')+len('type='):ss.index('/ispb')] + for ss in lstr[3:] if 'type=' in ss] + assert len(typ) == 1 + # To be updated : proper rezading from line + isoel = nblock+1 + in_ne = True + ne = np.array([]) + te = np.array([]) + pec = np.full((nne*nte,), np.nan) + ind = 0 + continue + + # Get ne for the transition being scanned (block) + if in_ne is True: + ne = np.append(ne, + np.array(line.rstrip().strip().split(' '), + dtype=float)) + if ne.size == nne: + in_ne = False + in_te = True + + # Get te for the transition being scanned (block) + elif in_te is True: + te = np.append(te, + np.array(line.rstrip().strip().split(' '), + dtype=float)) + if te.size == nte: + in_te = False + in_pec = True + + # Get pec for the transition being scanned (block) + elif in_pec is True: + data = np.array(line.rstrip().strip().split(' '), + dtype=float) + pec[ind:ind+data.size] = data + ind += data.size + if ind == pec.size: + in_pec = False + key = 'Ar{}_{}_openadas_adf15ic'.format(16, isoel) + dout[key] = { + 'lamb': lamb, + 'origin': 'openadas_adf15_{}_ic', + 'type': typ[0], + 'ne': ne, 'te': te, + 'interp_log': scpRectSpline(np.log(ne), + np.log(te), + np.log(pec).reshape((nne, + nte)), + kx=deg, ky=deg)} + nblock += 1 + + # Get transitions from table at the end + if 'photon emissivity atomic transitions' in line: + itab = ii + 6 + if ii == itab: + in_tab = True + if in_tab is True: + lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] + isoel = int(lstr[1]) + key = 'Ar{}_{}_openadas_adf15ic'.format(16, isoel) + assert dout[key]['lamb'] == float(lstr[2])*1.e-10 + dout[key]['transition'] = None + if isoel == nlines: + in_tab = False + import pdb; pdb.set_trace() # DB + return dout diff --git a/tofu/version.py b/tofu/version.py index c6b1a377f..bb6e075ad 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-64-g7f7a339c' +__version__ = '1.4.2b4-65-g687249c2' From ee6f48b74912edf8c19afa8ea09ff90bb73edd38 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Wed, 4 Mar 2020 18:12:49 +0100 Subject: [PATCH 06/22] [Issue356] Debugged tofucustom for openadas/ creation and debugged imports --- tofu/__init__.py | 2 +- tofu/openadas2tofu/__init__.py | 108 +++++---------------------------- tofu/scripts/tofucustom.py | 12 +++- tofu/version.py | 2 +- 4 files changed, 27 insertions(+), 97 deletions(-) diff --git a/tofu/__init__.py b/tofu/__init__.py index 4ef97c747..7bf166d8c 100644 --- a/tofu/__init__.py +++ b/tofu/__init__.py @@ -74,7 +74,7 @@ # ------------------------------------- msg = None -dsub = dict.fromkeys(['imas2tofu', 'mag']) +dsub = dict.fromkeys(['imas2tofu', 'openadas2tofu', 'mag']) for sub in dsub.keys(): try: exec('import tofu.{0} as {0}'.format(sub)) diff --git a/tofu/openadas2tofu/__init__.py b/tofu/openadas2tofu/__init__.py index c5bf060e5..8daa3885c 100644 --- a/tofu/openadas2tofu/__init__.py +++ b/tofu/openadas2tofu/__init__.py @@ -4,96 +4,18 @@ The openadas-compatibility module of tofu """ -# import warnings -# import traceback -# import itertools as itt - -# try: - # try: - # from tofu.imas2tofu._core import * - # from tofu.imas2tofu._mat2ids2calc import * - # except Exception: - # from ._core import * - # from ._mat2ids2calc import * -# except Exception as err: - # if str(err) == 'imas not available': - # msg = "" - # msg += "\n\nIMAS python API issue\n" - # msg += "imas could not be imported into tofu ('import imas' failed):\n" - # msg += " - it may not be installed (optional dependency)\n" - # msg += " - or you have loaded the wrong working environment\n\n" - # msg += " => the optional sub-package tofu.imas2tofu is not usable\n" - # else: - # msg = str(traceback.format_exc()) - # msg += "\n\n => the optional sub-package tofu.imas2tofu is not usable\n" - # raise Exception(msg) - - -# # ----------------------------------------------- -# # Check IMAS version vs latest available in linux modules -# # ----------------------------------------------- - -# _KEYSTR = 'IMAS/' - - -# # extract all IMAS versions from a str returned by modules -# def extractIMAS(ss, keystr=_KEYSTR): - # if keystr not in ss: - # raise Exception - # ls = ss[ss.index(keystr):].split('\n') - # ls = itt.chain.from_iterable([s.split(' ') for s in ls]) - # ls = [s for s in ls if keystr in s] - # ls = [s[len(keystr):s.index('(')] if '(' in s else s[len(keystr):] - # for s in ls] - # return sorted(ls) - - -# # Compare current and latest available IMAS versions -# def check_IMAS_version(verb=True, keystr=_KEYSTR): - # import subprocess - - # # Get currently loaded IMAS - # cmd = "module list" - # proc = subprocess.run(cmd, check=True, shell=True, - # stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - # lcur = extractIMAS(proc.stdout.decode(), keystr=keystr) - # if len(lcur) != 1: - # msg = ("You seem to have no / several IMAS version loaded:\n" - # + "\t- module list: {}".format(lcur)) - # raise Exception(msg) - - # # Get all available IMAS - # cmd = "module av IMAS" - # proc = subprocess.run(cmd, check=True, shell=True, - # stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - # lav = extractIMAS(proc.stdout.decode(), keystr=keystr) - # if len(lav) == 0: - # msg = "There is not available IMAS version" - # raise Exception(msg) - - # # Compare and warn - # if lcur[0] not in lav: - # msg = "The current IMAS version is not available!" - # raise Exception(msg) - - # msg = None - # if lav.index(lcur[0]) != len(lav)-1: - # msg = ("\nYou do not seem to be using the latest IMAS version:\n" - # + "'module list' vs 'module av IMAS' suggests:\n" - # + "\t- Current version: {}\n".format(lcur[0]) - # + "\t- Latest version : {}".format(lav[-1])) - # warnings.warn(msg) - # return lcur[0], lav - - -# # Try comparing and warning -# try: - # _, _ = check_IMAS_version(verb=True) -# except Exception as err: - # # This warning is an optional luxury, should not block anything - # pass - - -# __all__ = ['MultiIDSLoader', 'load_Config', 'load_Plasma2D', - # 'load_Cam', 'load_Data'] -# del warnings, traceback, itt, _KEYSTR +import traceback + +try: + try: + from tofu.openadas2tofu._requests import * + from tofu.openadas2tofu._read_files import * + except Exception: + from ._requests import * + from ._read_files import * +except Exception as err: + msg = str(traceback.format_exc()) + msg += "\n\n => the optional sub-package tofu.imas2tofu is not usable\n" + raise Exception(msg) + +del traceback diff --git a/tofu/scripts/tofucustom.py b/tofu/scripts/tofucustom.py index c0281f9cd..b1449b004 100755 --- a/tofu/scripts/tofucustom.py +++ b/tofu/scripts/tofucustom.py @@ -17,14 +17,15 @@ _USER_HOME = os.path.expanduser('~') _TARGET = os.path.join(_USER_HOME, '.tofu') _LF = ['_imas2tofu_def.py'] - +_LD = ['openadas2tofu'] ################################################### ################################################### # function ################################################### -def custom(target=_TARGET, source=_SOURCE, files=_LF): +def custom(target=_TARGET, source=_SOURCE, + files=_LF, directories=_LD): # Caveat (up to now only relevant for _TARGET) if target != _TARGET: @@ -40,9 +41,16 @@ def custom(target=_TARGET, source=_SOURCE, files=_LF): # Try creating directory and copying modules try: + # Create .tofu/ if non-existent if not os.path.isdir(target): os.mkdir(target) + # Create directories + for dd in directories: + if not os.path.isdir(os.path.join(target, dd)): + os.mkdir(os.path.join(target, dd)) + + # Copy files for ff in files: mod, f0 = ff.split('_')[1:] copyfile(os.path.join(source, mod, '_'+f0), diff --git a/tofu/version.py b/tofu/version.py index e04f68cf0..a1431e340 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-66-gd28fd3da' +__version__ = '1.4.2b4-73-gecbc0a2d' From a4c55fe3ad2ae9a1a8ec9fce5c0e8e7926fd7668 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Wed, 4 Mar 2020 18:13:50 +0100 Subject: [PATCH 07/22] [Issue356] ADF15 file reading ok with proper keys and checks, now implementing search_from_wavelength(), TBF --- tofu/openadas2tofu/_read_files.py | 74 ++++++++++++++++++----- tofu/openadas2tofu/_requests.py | 99 +++++++++++++++++++++++++------ tofu/version.py | 2 +- 3 files changed, 142 insertions(+), 33 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 106ea3017..bccc2b849 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -1,24 +1,56 @@ # Built-in import os +import re # Common import numpy as np -from scipy.interpolate import RectBivariateSpline as scpRectSpline +from scipy.interpolate import RectBivariateSpline as scpRectSpl + + +__all__ = ['read'] _LTYPES = ['adf11', 'adf15'] _DEG = 1 -def read(pfe): +def _get_PATH_LOCAL(): + pfe = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + if os.path.isdir(pfe): + return pfe + else: + return None + + +def read(adas_path): """ Read openadas-formatted files and return a dict of interpolators """ + path_local = _get_PATH_LOCAL() + + # Check whether the local .tofu repo exists, if not recommend tofu-custom + if path_local is None: + path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + msg = ("You do not seem to have a local ./tofu repository\n" + + "tofu uses that local repository to store all user-specific " + + "data and downloads\n" + + "In particular, openadas files are downloaded and saved in:\n" + + "\t{}\n".format(path) + + " => to set-up your local .tofu repo, run in a terminal:\n" + + "\ttofu-custom") + raise Exception(msg) + + # make sure adas_path is not understood as absolute local path + if adas_path[0] == '/': + adas_path = adas_path[1:] + + # Check file was downloaded locally + pfe = os.path.join(path_local, adas_path) if not os.path.isfile(pfe): msg = ("Provided file does not seem to exist:\n" + "\t{}\n".format(pfe) + " => Search it online with tofu.openadas2tofu.search()\n" - + " => Download it locally with ofu.openadas2tofu.download()") + + " => Download it locally with tofu.openadas2tofu.download()") raise FileNotFoundError(msg) lc = [ss for ss in _LTYPES if ss in pfe] @@ -46,6 +78,10 @@ def _read_adf11(): # ############################################################################# +def _get_adf15_key(elem, charge, isoel, typ0, typ1): + return '{}{}_{}_openadas_{}_{}'.format(elem, charge, isoel, + typ0, typ1) + def _read_adf15(pfe, deg=None): if deg is None: @@ -56,7 +92,17 @@ def _read_adf15(pfe, deg=None): flag0 = 'superstage partition information' dout = {} - + # Get file markers from name (elem, charge, typ0, typ1) + typ0, typ1, elemq = pfe.split('][')[1:] + ind = re.search(r'\d', elemq).start() + elem = elemq[:ind].title() + charge = int(elemq[ind:-4]) + assert elem.lower() in typ0[:2] + assert elem.lower() == typ1.split('_')[0] + typ0 = typ0[len(elem)+1:] + typ1 = typ1.split('_')[1] + + # Extract data from file nlines, nblock = None, 0 in_ne, in_te, in_pec, in_tab, itab = False, False, False, False, np.inf with open(pfe) as search: @@ -64,7 +110,8 @@ def _read_adf15(pfe, deg=None): # Get number of lines (transitions) stored in this file if ii == 0: - nlines = int(line[:line.index('/AR')].replace(' ','')) + lstr = line.split('/') + nlines = int(lstr[0].replace(' ','')) continue # Get info about the transition being scanned (block) @@ -110,17 +157,17 @@ def _read_adf15(pfe, deg=None): ind += data.size if ind == pec.size: in_pec = False - key = 'Ar{}_{}_openadas_adf15ic'.format(16, isoel) + key = _get_adf15_key(elem, charge, isoel, typ0, typ1) dout[key] = { 'lamb': lamb, - 'origin': 'openadas_adf15_{}_ic', + 'origin': pfe, 'type': typ[0], 'ne': ne, 'te': te, - 'interp_log': scpRectSpline(np.log(ne), - np.log(te), - np.log(pec).reshape((nne, - nte)), - kx=deg, ky=deg)} + 'interp2d_log_nete': + scpRectSpl(np.log(ne), np.log(te), + np.log(pec).reshape((nne, nte)), + kx=deg, ky=deg) + } nblock += 1 # Get transitions from table at the end @@ -131,10 +178,9 @@ def _read_adf15(pfe, deg=None): if in_tab is True: lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] isoel = int(lstr[1]) - key = 'Ar{}_{}_openadas_adf15ic'.format(16, isoel) + key = _get_adf15_key(elem, charge, isoel, typ0, typ1) assert dout[key]['lamb'] == float(lstr[2])*1.e-10 dout[key]['transition'] = None if isoel == nlines: in_tab = False - import pdb; pdb.set_trace() # DB return dout diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index 55e7a08fe..df3ce1932 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -8,14 +8,15 @@ # Common import numpy as np + + +__all__ = ['search', 'download', 'download_all', 'clean_downloads'] + + # Check whether a local .tofu/ repo exists -pfe = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') -if os.path.isdir(pfe): - _PATH_LOCAL = pfe -else: - _PATH_LOCAL = None _URL = 'https://open.adas.ac.uk' _URL_SEARCH = _URL + '/freeform?searchstring=' +_URL_SEARCH_WAVL = _URL + '/wavelength?' _URL_ADF15 = _URL + '/adf15' _URL_DOWNLOAD = _URL + '/download' @@ -26,6 +27,14 @@ # ############################################################################# +def _get_PATH_LOCAL(): + pfe = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + if os.path.isdir(pfe): + return pfe + else: + return None + + def _getcharray(ar, col=None, sep=' ', line='-', just='l', verb=True, returnas=str): """ Format and return char array (for pretty printing) """ @@ -69,9 +78,10 @@ def _getcharray(ar, col=None, sep=' ', line='-', just='l', # ############################################################################# -# Web-oriented functions +# online search # ############################################################################# + class FileAlreayExistsException(Exception): pass @@ -91,9 +101,9 @@ def search(searchstr=None, returnas=None, searchurl = '+'.join([requests.utils.quote(kk) for kk in searchstr.split(' ')]) - resp = requests.get('{}{}{}'.format(_URL_SEARCH, + resp = requests.get('{}{}&{}'.format(_URL_SEARCH, searchurl, - '&searching=1')) + 'searching=1')) # Extract response from html out = resp.text.split('\n') @@ -136,15 +146,67 @@ def search(searchstr=None, returnas=None, return arr +def search_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, + returnas=None, verb=None): + + # Check input + if returnas is None: + returnas = False + if verb is None: + verb = True + if lambmin is None: + lambmin = '' + if lambmax is None: + lambmax = '' + if resolveby is None: + resolveby = 'transition' + if resolveby not in ['transition', 'file']: + msg = ("Arg resolveeby must be:\n" + + "\t- 'transition': list all available transitions\n" + + "\t- 'file': list all files containing relevant transitions") + raise Exception(msg) + + searchurl = '&'.join(['wavemin={}'.format(lambmin), + 'wavemax={}'.format(lambmax), + 'resolveby={}'.format(resolveby)]) + + resp = requests.get('{}{}&{}'.format(_URL_SEARCH_WAVL, + searchurl, + 'searching=1')) + + # Extract response from html + out = resp.text.split('\n') + import pdb; pdb.set_trace() # DB + ind0 = out.index('') + out = out[ind0+1:] + ind1 = out.index('
') + out = out[:ind1-1] + nresults = len(out) -1 + + # Format output + char = np.array(lout) + col = ['Ion', 'charge', 'type of data', 'full file name'] + arr = _getcharray(char, col=col, + sep=' ', line='-', just='l', + returnas=returnas, verb=verb) + return arr + + +# ############################################################################# +# Download +# ############################################################################# + + def _check_exists(filename, update=None): # if target is None: # target = filename # In case a small modification becomes necessary later target = filename + path_local = _get_PATH_LOCAL() # Check whether the local .tofu repo exists, if not recommend tofu-custom - if _PATH_LOCAL is None: + if path_local is None: path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') msg = ("You do not seem to have a local ./tofu repository\n" + "tofu uses that local repository to store all user-specific " @@ -158,12 +220,12 @@ def _check_exists(filename, update=None): # Parse intermediate repos and create if necessary lrep = target.split('/')[1:-1] for ii in range(len(lrep)): - repo = os.path.join(_PATH_LOCAL, *lrep[:ii+1]) + repo = os.path.join(path_local, *lrep[:ii+1]) if not os.path.isdir(repo): os.mkdir(repo) # Check if file already exists - path = os.path.join(_PATH_LOCAL, *lrep) + path = os.path.join(path_local, *lrep) pfe = os.path.join(path, target.split('/')[-1]) if os.path.isfile(pfe): if update is False: @@ -237,7 +299,7 @@ def download_all(searchstr=None, arr = search(searchstr=searchstr, include_partial=include_partial, verb=False, returnas=np.ndarray) if verb is True: - msg = "Downloading from {} into {}:".format(_URL, _PATH_LOCAL) + msg = "Downloading from {} into {}:".format(_URL, _get_PATH_LOCAL()) print(msg) for ii in range(arr.shape[0]): try: @@ -260,13 +322,14 @@ def download_all(searchstr=None, def clean_downloads(): - if _PATH_LOCAL is None: + path_local = _get_PATH_LOCAL() + if path_local is None: return - lf = [ff for ff in os.listdir(_PATH_LOCAL) - if os.path.isfile(os.path.join(_PATH_LOCAL, ff))] - ld = [ff for ff in os.listdir(_PATH_LOCAL) - if os.path.isdir(os.path.join(_PATH_LOCAL, ff))] + lf = [ff for ff in os.listdir(path_local) + if os.path.isfile(os.path.join(path_local, ff))] + ld = [ff for ff in os.listdir(path_local) + if os.path.isdir(os.path.join(path_local, ff))] for ff in lf: os.remove(ff) for dd in ld: - shutil.rmtree(os.path.join(_PATH_LOCAL, dd)) + shutil.rmtree(os.path.join(path_local, dd)) diff --git a/tofu/version.py b/tofu/version.py index a1431e340..da8d03812 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-73-gecbc0a2d' +__version__ = '1.4.2b4-74-gee6f48b7' From b7e90073356472cfefadc8c82cb5a769aa094df6 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 5 Mar 2020 10:47:16 +0100 Subject: [PATCH 08/22] [Issue356] search_online(), search_online_by_wavelength() and download_all() all operational and consistent --- tofu/openadas2tofu/_read_files.py | 11 +- tofu/openadas2tofu/_requests.py | 195 +++++++++++++++++++++++++----- tofu/version.py | 2 +- 3 files changed, 176 insertions(+), 32 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index bccc2b849..598071f40 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -82,7 +82,10 @@ def _get_adf15_key(elem, charge, isoel, typ0, typ1): return '{}{}_{}_openadas_{}_{}'.format(elem, charge, isoel, typ0, typ1) -def _read_adf15(pfe, deg=None): +def _read_adf15(pfe, + lambmin=None, + lambmax=None, + deg=None): if deg is None: deg = _DEG @@ -131,6 +134,12 @@ def _read_adf15(pfe, deg=None): ind = 0 continue + # Check lamb is ok + if lambmin is not None and lamb < lambmin: + continue + if lambmax is not None and lamb > lambmax: + continue + # Get ne for the transition being scanned (block) if in_ne is True: ne = np.append(ne, diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index df3ce1932..a83f41fd9 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -10,7 +10,8 @@ -__all__ = ['search', 'download', 'download_all', 'clean_downloads'] +__all__ = ['search_online', 'search_online_by_wavelengthA', + 'download', 'download_all', 'clean_downloads'] # Check whether a local .tofu/ repo exists @@ -86,8 +87,21 @@ class FileAlreayExistsException(Exception): pass -def search(searchstr=None, returnas=None, - include_partial=None, verb=None): +def search_online(searchstr=None, returnas=None, + include_partial=None, verb=None): + """ Perform an online freeform search on https://open.adas.ac.uk + + Pass searchstr to the online freeform search + Prints the results (if verb=True) + Optionally return the result as: + - a char array (returnas = np.ndarray) + - a formatted str (returnas = str) + + example + ------- + >>> import tofu as tf + >>> tf.openadas2tofu.search_online('ar+16 ADF15') + """ # Check input if returnas is None: @@ -107,17 +121,28 @@ def search(searchstr=None, returnas=None, # Extract response from html out = resp.text.split('\n') - ind0 = out.index('') - out = out[ind0+1:] - ind1 = out.index('
') - out = out[:ind1-1] + flag0 = '' + flag1 = '
' + ind0 = [ii for ii, vv in enumerate(out) if flag0 in vv] + ind1 = [ii for ii, vv in enumerate(out) if flag1 in vv] + import pdb; pdb.set_trace() # DB + if len(ind0) != 1 or len(ind1) != 1: + msg = ("Format of html response seems to have changed!\n" + + "Cannot find flags:\n" + + "\t- {}\n".format(flag0) + + "\t- {}\n".format(flag1) + + "in requests.get({}).text".format(total_url)) + raise Exception(msg) + out = out[ind0+1:ind1-1] nresults = len(out) -1 + # Get columns heads = [str.replace(kk.replace('', '').replace('', ''), '', '').replace('', '') for kk in out[0].split('')] nhead = len(heads) + # Get results lout = [] for ii in range(0, nresults): if 'Partial results are listed below' in out[ii+1]: @@ -146,8 +171,24 @@ def search(searchstr=None, returnas=None, return arr -def search_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, - returnas=None, verb=None): +def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, + element=None, returnas=None, verb=None): + """ Perform an online search by wavelength on https://open.adas.ac.uk + + Pass the min / max wavelength (in Angstrom) to the online wavelength search + Prints the results (if verb=True) + Optionally return the result as: + - a char array (returnas = np.ndarray) + - a formatted str (returnas = str) + + The result can be resolve by transition or by adas file + Optionally filter by element to return only the results of one element + + example + ------- + >>> import tofu as tf + >>> tf.openadas2tofu.search_online_by_wavelengthA(3., 4., element='ar') + """ # Check input if returnas is None: @@ -165,27 +206,66 @@ def search_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, + "\t- 'transition': list all available transitions\n" + "\t- 'file': list all files containing relevant transitions") raise Exception(msg) + if element is not None and not isinstance(element, str): + msg = ("Arg element must be a str!\n" + + "\te.g.: element='ar'") + raise Exception(msg) - searchurl = '&'.join(['wavemin={}'.format(lambmin), - 'wavemax={}'.format(lambmax), + searchurl = '&'.join(['wave_min={}'.format(lambmin), + 'wave_max={}'.format(lambmax), 'resolveby={}'.format(resolveby)]) - resp = requests.get('{}{}&{}'.format(_URL_SEARCH_WAVL, - searchurl, - 'searching=1')) + total_url = '{}{}&{}'.format(_URL_SEARCH_WAVL, searchurl, 'searching=1') + resp = requests.get(total_url) # Extract response from html out = resp.text.split('\n') - import pdb; pdb.set_trace() # DB - ind0 = out.index('') - out = out[ind0+1:] - ind1 = out.index('
') - out = out[:ind1-1] + flag0 = '' + flag1 = '
' + ind0 = [ii for ii, vv in enumerate(out) if flag0 in vv] + ind1 = [ii for ii, vv in enumerate(out) if flag1 in vv] + if len(ind0) != 1 or len(ind1) != 1: + msg = ("Format of html response seems to have changed!\n" + + "Cannot find flags:\n" + + "\t- {}\n".format(flag0) + + "\t- {}\n".format(flag1) + + "in requests.get({}).text".format(total_url)) + raise Exception(msg) + out = out[ind0[0]+1].split('') nresults = len(out) -1 + # Get columns + col = [kk.replace('', '').replace('', '').strip() + for kk in out[0].split('')] + ncol = len(col) + colex = ['Wavelength', 'Ion', 'Data Type', 'Transition', 'File Details'] + if col != colex: + msg = ("Format of table columns in html seems to have changed!\n" + + "\t- expected: {}\n".format(colex) + + "\t- observed: {}".format(col)) + raise Exception(msg) + + lout = [] + for ii in range(0, nresults): + lstri = out[ii+1].split('') + assert len(lstri) == ncol + lamb = lstri[0].replace('Å', '') + elm, charge = lstri[1].replace('', '').split('') + if element is not None and elm.lower() != element.lower(): + continue + typ = lstri[2].replace('', '').split('>')[1] + trans = lstri[3].replace(' ', ' ') + trans = trans.replace('', '^{').replace('', '}') + trans = trans.replace('', '_{').replace('', '}') + trans = trans.replace('→', '->') + fil = lstri[4][lstri[4].index('detail')+len('detail'):] + fil = fil[:fil.index('.dat')+len('.dat')] + lout.append([lamb, elm, charge, typ, trans, fil]) + # Format output char = np.array(lout) - col = ['Ion', 'charge', 'type of data', 'full file name'] + col = ['Wavelength', 'Element', 'Charge', 'Data Type', + 'Transition', 'Full file name'] arr = _getcharray(char, col=col, sep=' ', line='-', just='l', returnas=returnas, verb=verb) @@ -242,6 +322,16 @@ def _check_exists(filename, update=None): def download(filename=None, update=None, verb=None, returnas=None): + """ Download desired file from https://open.adas.ac.uk + + All downloaded files are stored in your local tofu directory (~/.tofu/) + + example + ------- + >>> import tofu as tf + >>> filename = '/adf15/pec40][ar/pec40][ar_ls][ar16.dat' + >>> tf.openadas2tofu.download(filename) + """ # --------------------------- # Check @@ -285,8 +375,25 @@ def download(filename=None, return pfe -def download_all(searchstr=None, +def download_all(files=None, searchstr=None, + lambmin=None, lambmax=None, element=None, include_partial=None, update=None, verb=None): + """ Download all desired files from https://open.adas.ac.uk + + The files to download can be provided either as: + - a list of full openadas file names (files) + - the result of an online freeform search + (searchstr fed to search_online) + - the result of an online search by wavelength + (lambmin, lambmax and element fed to search_online_by_wavelengthA) + + All downloaded files are stored in your local tofu directory (~/.tofu/) + + example + ------- + >>> import tofu as tf + >>> tf.openadas2tofu.download_all(lambmin=3., lambmax=4., element='ar') + """ # Check if include_partial is None: include_partial = False @@ -294,27 +401,54 @@ def download_all(searchstr=None, update = False if verb is None: verb = True + lc = [files is not None, + searchstr is not None, + any([ss is not None for ss in [lambmin, lambmax, element]])] + if np.sum(lc) != 1: + msg = ( + "Please either searchstr xor (lambmin, lambmax, element)\n" + + "\t- files: list of full file names to be downloaded\n" + + "\t- searchstr: uses search_online()\n" + + "\t- lambmin, lambmax, element: search_online_by_wavelengthA") + raise Exception(msg) + + # Get list of files + if lc[0]: + if isinstance(files, str): + files = [files] + if not (isinstance(files, list) + and all([isinstance(ss, str) for ss in files])): + msg = "files must be a list of full openadas file names!" + raise Exception(msg) + elif lc[1]: + arr = search(searchstr=searchstr, include_partial=include_partial, + verb=False, returnas=np.ndarray) + files = arr[:, -1] + elif lc[2]: + arr = search_online_by_wavelengthA(lambmin=lambmin, + lambmax=lambmax, + element=element, verb=False, + returnas=np.ndarray) + files = np.unique(arr[:, -1]) # Download - arr = search(searchstr=searchstr, include_partial=include_partial, - verb=False, returnas=np.ndarray) if verb is True: msg = "Downloading from {} into {}:".format(_URL, _get_PATH_LOCAL()) print(msg) - for ii in range(arr.shape[0]): + for ii in range(len(files)): try: - exists, pfe = _check_exists(arr[ii, 3], update=update) - pfe = download(filename=arr[ii, 3], update=update, + exists, pfe = _check_exists(files[ii], update=update) + pfe = download(filename=files[ii], update=update, verb=False, returnas=str) if exists is True: - msg = "\toverwritten: \t{}".format(arr[ii, 3]) + msg = "\toverwritten: \t{}".format(files[ii]) else: - msg = "\tdownloaded: \t{}".format(arr[ii, 3]) + msg = "\tdownloaded: \t{}".format(files[ii]) except FileAlreayExistsException: - msg = "\talready exists: {}".format(arr[ii, 3]) + msg = "\talready exists: {}".format(files[ii]) except Exception as err: msg = (str(err) - + "\n\nCould not download file {}".format(arr[ii, 3])) + + "\n\nCould not download file {}".format(files[ii])) raise err if verb is True: @@ -322,6 +456,7 @@ def download_all(searchstr=None, def clean_downloads(): + """ Delete all openadas files downloaded in your ~/.tofu/ directory """ path_local = _get_PATH_LOCAL() if path_local is None: return diff --git a/tofu/version.py b/tofu/version.py index da8d03812..4bd58db22 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-74-gee6f48b7' +__version__ = '1.4.2b4-75-ga4c55fe3' From 480d04d9d199e64bdd391cac853eaae134e0175a Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 5 Mar 2020 11:14:21 +0100 Subject: [PATCH 09/22] [Issue356] read() operational for both absolute path and full adas names for ADF15, and lambmin, lambmax implemented for ADF15 --- tofu/openadas2tofu/_read_files.py | 53 +++++++++++++++++++------------ tofu/version.py | 2 +- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 598071f40..e48da3564 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -23,8 +23,8 @@ def _get_PATH_LOCAL(): return None -def read(adas_path): - """ Read openadas-formatted files and return a dict of interpolators """ +def read(adas_path, **kwdargs): + """ Read openadas-formatted files and return a dict with the data """ path_local = _get_PATH_LOCAL() @@ -40,18 +40,22 @@ def read(adas_path): + "\ttofu-custom") raise Exception(msg) - # make sure adas_path is not understood as absolute local path - if adas_path[0] == '/': - adas_path = adas_path[1:] - - # Check file was downloaded locally - pfe = os.path.join(path_local, adas_path) - if not os.path.isfile(pfe): - msg = ("Provided file does not seem to exist:\n" - + "\t{}\n".format(pfe) - + " => Search it online with tofu.openadas2tofu.search()\n" - + " => Download it locally with tofu.openadas2tofu.download()") - raise FileNotFoundError(msg) + # Determine whether adas_path is an absolute path or an adas full name + if os.path.isfile(adas_path): + pfe = adas_path + else: + # make sure adas_path is not understood as absolute local path + if adas_path[0] == '/': + adas_path = adas_path[1:] + + # Check file was downloaded locally + pfe = os.path.join(path_local, adas_path) + if not os.path.isfile(pfe): + msg = ("Provided file does not seem to exist:\n" + + "\t{}\n".format(pfe) + + " => Search it online with tofu.openadas2tofu.search()\n" + + " => Download it with tofu.openadas2tofu.download()") + raise FileNotFoundError(msg) lc = [ss for ss in _LTYPES if ss in pfe] if not len(lc) == 1: @@ -61,7 +65,7 @@ def read(adas_path): raise Exception(msg) func = eval('_read_{}'.format(lc[0])) - return func(pfe) + return func(pfe, **kwdargs) @@ -91,7 +95,7 @@ def _read_adf15(pfe, deg = _DEG # Get summary of transitions - flagblock = 'isel =' + flagblock = '/isel =' flag0 = 'superstage partition information' dout = {} @@ -118,7 +122,7 @@ def _read_adf15(pfe, continue # Get info about the transition being scanned (block) - if flagblock in line and nblock < nlines: + if flagblock in line and 'C' not in line and nblock < nlines: lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] lamb = float(lstr[0])*1.e-10 nne, nte = int(lstr[1]), int(lstr[2]) @@ -135,9 +139,9 @@ def _read_adf15(pfe, continue # Check lamb is ok - if lambmin is not None and lamb < lambmin: + if in_ne is True and lambmin is not None and lamb < lambmin: continue - if lambmax is not None and lamb > lambmax: + if in_ne is True and lambmax is not None and lamb > lambmax: continue # Get ne for the transition being scanned (block) @@ -172,7 +176,7 @@ def _read_adf15(pfe, 'origin': pfe, 'type': typ[0], 'ne': ne, 'te': te, - 'interp2d_log_nete': + 'pec_interp2d_log_nete': scpRectSpl(np.log(ne), np.log(te), np.log(pec).reshape((nne, nte)), kx=deg, ky=deg) @@ -189,7 +193,14 @@ def _read_adf15(pfe, isoel = int(lstr[1]) key = _get_adf15_key(elem, charge, isoel, typ0, typ1) assert dout[key]['lamb'] == float(lstr[2])*1.e-10 - dout[key]['transition'] = None + if (dout[key]['type'] not in lstr + or lstr.index(dout[key]['type']) < 4): + msg = ("Inconsistency in table, type not found:\n" + + "\t- expected: {}\n".format(dout[key]['type']) + + "\t- line: {}".format(line)) + raise Exception(msg) + trans = lstr[3:lstr.index(dout[key]['type'])] + dout[key]['transition'] = ''.join(trans) if isoel == nlines: in_tab = False return dout diff --git a/tofu/version.py b/tofu/version.py index 4bd58db22..aeef7182a 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-75-ga4c55fe3' +__version__ = '1.4.2b4-76-gb7e90073' From c42e00c09396298af5b8eab39dce1052a32e389f Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 5 Mar 2020 14:01:41 +0100 Subject: [PATCH 10/22] [Issue356] read_all() operational, with element, Type and optional charge --- tofu/openadas2tofu/_read_files.py | 130 ++++++++++++++++++++++++++---- tofu/version.py | 2 +- 2 files changed, 116 insertions(+), 16 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index e48da3564..18fc3957b 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -8,7 +8,7 @@ from scipy.interpolate import RectBivariateSpline as scpRectSpl -__all__ = ['read'] +__all__ = ['read', 'read_all'] _LTYPES = ['adf11', 'adf15'] @@ -67,6 +67,89 @@ def read(adas_path, **kwdargs): func = eval('_read_{}'.format(lc[0])) return func(pfe, **kwdargs) +def read_all(element=None, charge=None, Type=None, verb=None, **kwdargs): + + # Check + if Type is None: + Type = 'adf15' + if not isinstance(Type, str) or Type.lower() not in _LTYPES: + msg = ("Please choose a valid adas file type:\n" + + "\t- allowed: {}\n".format(_LTYPES) + + "\t- provided: {}".format(Type)) + raise Exception(msg) + Type = Type.lower() + if not isinstance(element, str): + msg = "Please choose an element!" + raise Exception(msg) + element = element.lower() + if verb is None: + verb = True + if charge is not None and not isinstance(charge, int): + msg = "charge must be a int!" + raise Exception(msg) + + path_local = _get_PATH_LOCAL() + + # Check whether the local .tofu repo exists, if not recommend tofu-custom + if path_local is None: + path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + msg = ("You do not seem to have a local ./tofu repository\n" + + "tofu uses that local repository to store all user-specific " + + "data and downloads\n" + + "In particular, openadas files are downloaded and saved in:\n" + + "\t{}\n".format(path) + + " => to set-up your local .tofu repo, run in a terminal:\n" + + "\ttofu-custom") + raise Exception(msg) + + # Get list of relevant directories + ld = [dd for dd in os.listdir(path_local) + if (os.path.isdir(os.path.join(path_local, dd)) + and Type in dd)] + if len(ld) != 1: + av = [dd for dd in os.listdir(path_local) + if os.path.isdir(os.path.join(path_local, dd))] + msg = ("You have no / many directories in your local " + + "~/.tofu/openadas2tofu/ matching the desired file type:\n" + + "\t- provided Type: {}\n".format(Type) + + "\t- available: {}\n".format(av) + + " => download the data with tf.openadas2tofu.download()") + raise Exception(msg) + path = os.path.join(path_local, ld[0]) + ld = [dd for dd in os.listdir(path) + if (os.path.isdir(os.path.join(path, dd)) + and element in dd)] + if len(ld) != 1: + av = [dd for dd in os.listdir(path) + if os.path.isdir(os.path.join(path, dd))] + msg = ("You have no / many directories in your local " + + "~/.tofu/openadas2tofu/ matching the desired element:\n" + + "\t- provided element: {}\n".format(element) + + "\t- available: {}\n".format(av) + + " => download the data with tf.openadas2tofu.download()") + raise Exception(msg) + path = os.path.join(path, ld[0]) + + # Get list of relevant files pfe + lpfe = [os.path.join(path, ff) for ff in os.listdir(path) + if (os.path.isfile(os.path.join(path, ff)) + and ff[-4:] == '.dat' + and element in ff)] + if charge is not None: + lpfe = [ff for ff in lpfe if str(charge) in ff] + + # Extract data from each file + func = eval('_read_{}'.format(Type)) + if Type == 'adf15': + out = {} + for pfe in lpfe: + if verb is True: + msg = "\tLoading data from {}".format(pfe) + print(msg) + out.update(func(pfe, **kwdargs)) + return out + + # ############################################################################# @@ -112,6 +195,7 @@ def _read_adf15(pfe, # Extract data from file nlines, nblock = None, 0 in_ne, in_te, in_pec, in_tab, itab = False, False, False, False, np.inf + skip = False with open(pfe) as search: for ii, line in enumerate(search): @@ -125,12 +209,18 @@ def _read_adf15(pfe, if flagblock in line and 'C' not in line and nblock < nlines: lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] lamb = float(lstr[0])*1.e-10 + isoel = nblock + 1 + nblock += 1 + if ((lambmin is not None and lamb < lambmin) + or (lambmax is not None and lamb > lambmax)): + skip = True + continue + skip = False nne, nte = int(lstr[1]), int(lstr[2]) typ = [ss[ss.index('type=')+len('type='):ss.index('/ispb')] for ss in lstr[3:] if 'type=' in ss] assert len(typ) == 1 # To be updated : proper rezading from line - isoel = nblock+1 in_ne = True ne = np.array([]) te = np.array([]) @@ -138,10 +228,11 @@ def _read_adf15(pfe, ind = 0 continue + if 'root partition information' in line and skip is True: + skip = False + # Check lamb is ok - if in_ne is True and lambmin is not None and lamb < lambmin: - continue - if in_ne is True and lambmax is not None and lamb > lambmax: + if skip is True: continue # Get ne for the transition being scanned (block) @@ -181,7 +272,6 @@ def _read_adf15(pfe, np.log(pec).reshape((nne, nte)), kx=deg, ky=deg) } - nblock += 1 # Get transitions from table at the end if 'photon emissivity atomic transitions' in line: @@ -191,16 +281,26 @@ def _read_adf15(pfe, if in_tab is True: lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] isoel = int(lstr[1]) + lamb = float(lstr[2])*1.e-10 key = _get_adf15_key(elem, charge, isoel, typ0, typ1) - assert dout[key]['lamb'] == float(lstr[2])*1.e-10 - if (dout[key]['type'] not in lstr - or lstr.index(dout[key]['type']) < 4): - msg = ("Inconsistency in table, type not found:\n" - + "\t- expected: {}\n".format(dout[key]['type']) - + "\t- line: {}".format(line)) - raise Exception(msg) - trans = lstr[3:lstr.index(dout[key]['type'])] - dout[key]['transition'] = ''.join(trans) + if ((lambmin is None or lambmin < lamb) + and (lambmax is None or lambmax > lamb)): + if key not in dout.keys(): + msg = "Inconsistency in file {}".format(pfe) + raise Exception(msg) + if key in dout.keys(): + if dout[key]['lamb'] != lamb: + msg = "Inconsistency in file {}".format(pfe) + raise Exception(msg) + if (dout[key]['type'] not in lstr + or lstr.index(dout[key]['type']) < 4): + msg = ("Inconsistency in table, type not found:\n" + + "\t- expected: {}\n".format(dout[key]['type']) + + "\t- line: {}".format(line)) + raise Exception(msg) + trans = lstr[3:lstr.index(dout[key]['type'])] + dout[key]['transition'] = ''.join(trans) if isoel == nlines: in_tab = False + assert all(['transition' in vv.keys() for vv in dout.values()]) return dout diff --git a/tofu/version.py b/tofu/version.py index aeef7182a..cda5907fd 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-76-gb7e90073' +__version__ = '1.4.2b4-77-g480d04d9' From 39b1824efeffeb73e4d2143f0f5fa94dc2bb3693 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 5 Mar 2020 14:34:30 +0100 Subject: [PATCH 11/22] [Issue356] Debugged search_online() and now operational for adf11 --- tofu/openadas2tofu/_requests.py | 28 ++++++++++++++++------------ tofu/version.py | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index a83f41fd9..e607429f2 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -115,25 +115,24 @@ def search_online(searchstr=None, returnas=None, searchurl = '+'.join([requests.utils.quote(kk) for kk in searchstr.split(' ')]) - resp = requests.get('{}{}&{}'.format(_URL_SEARCH, - searchurl, - 'searching=1')) + total_url = '{}{}&{}'.format(_URL_SEARCH, searchurl, 'searching=1') + resp = requests.get(total_url) # Extract response from html out = resp.text.split('\n') flag0 = '' - flag1 = '
' + flag1 = '' ind0 = [ii for ii, vv in enumerate(out) if flag0 in vv] ind1 = [ii for ii, vv in enumerate(out) if flag1 in vv] - import pdb; pdb.set_trace() # DB - if len(ind0) != 1 or len(ind1) != 1: + if len(ind0) != 1 or len(ind1) == 0: msg = ("Format of html response seems to have changed!\n" + "Cannot find flags:\n" + "\t- {}\n".format(flag0) + "\t- {}\n".format(flag1) + "in requests.get({}).text".format(total_url)) raise Exception(msg) - out = out[ind0+1:ind1-1] + ind1 = np.min([ii for ii in ind1 if ii > ind0[0]]) + out = out[ind0[0]+1:ind1-1] nresults = len(out) -1 # Get columns @@ -154,9 +153,13 @@ def search_online(searchstr=None, returnas=None, lstri = out[ii+1].split('', '') - elm = elmq[:elmq.index('')] - charge = elmq[elmq.index('')+len(''):] - charge = charge.replace('', '') + if '' in elmq: + elm = elmq[:elmq.index('')] + charge = elmq[elmq.index('')+len(''):] + charge = charge.replace('', '') + else: + charge = '' + elm = elmq typ = lstri[2][1:] fil = lstri[3][lstri[3].index('detail')+len('detail'):] fil = fil[:fil.index('.dat')+len('.dat')] @@ -421,8 +424,9 @@ def download_all(files=None, searchstr=None, msg = "files must be a list of full openadas file names!" raise Exception(msg) elif lc[1]: - arr = search(searchstr=searchstr, include_partial=include_partial, - verb=False, returnas=np.ndarray) + arr = search_online(searchstr=searchstr, + include_partial=include_partial, + verb=False, returnas=np.ndarray) files = arr[:, -1] elif lc[2]: arr = search_online_by_wavelengthA(lambmin=lambmin, diff --git a/tofu/version.py b/tofu/version.py index cda5907fd..7505d545c 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-77-g480d04d9' +__version__ = '1.4.2b4-78-gc42e00c0' From 11e72e08eb92d5169960332c382a29a77ca049a9 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 5 Mar 2020 14:38:38 +0100 Subject: [PATCH 12/22] [Issue356] Replace '+' by '1+' in search_online() --- tofu/openadas2tofu/_requests.py | 2 ++ tofu/version.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index e607429f2..6ac667a0e 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -157,6 +157,8 @@ def search_online(searchstr=None, returnas=None, elm = elmq[:elmq.index('')] charge = elmq[elmq.index('')+len(''):] charge = charge.replace('', '') + if charge == '+': + charge = '1+' else: charge = '' elm = elmq diff --git a/tofu/version.py b/tofu/version.py index 7505d545c..0fdcc51f9 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-78-gc42e00c0' +__version__ = '1.4.2b4-79-g39b1824e' From 029850ea0c6395b7ca9b6c589096893e10eca632 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Thu, 5 Mar 2020 16:15:24 +0100 Subject: [PATCH 13/22] [Issue356] Finishing ADF11/SCD reading routine --- tofu/openadas2tofu/_read_files.py | 113 +++++++++++++++++++++++++----- tofu/openadas2tofu/_requests.py | 2 + tofu/version.py | 2 +- 3 files changed, 99 insertions(+), 18 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 18fc3957b..5401335d9 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -11,7 +11,8 @@ __all__ = ['read', 'read_all'] -_LTYPES = ['adf11', 'adf15'] +_DTYPES = {'adf11': ['scd'], + 'adf15': None} _DEG = 1 @@ -57,11 +58,11 @@ def read(adas_path, **kwdargs): + " => Download it with tofu.openadas2tofu.download()") raise FileNotFoundError(msg) - lc = [ss for ss in _LTYPES if ss in pfe] + lc = [ss for ss in _DTYPES.keys() if ss in pfe] if not len(lc) == 1: msg = ("File type could not be derived from absolute path:\n" + "\t- provided: {}\n".format(pfe) - + "\t- supported: {}".format(sorted(_LTYPES))) + + "\t- supported: {}".format(sorted(_DTYPES.keys()))) raise Exception(msg) func = eval('_read_{}'.format(lc[0])) @@ -72,9 +73,9 @@ def read_all(element=None, charge=None, Type=None, verb=None, **kwdargs): # Check if Type is None: Type = 'adf15' - if not isinstance(Type, str) or Type.lower() not in _LTYPES: + if not isinstance(Type, str) or Type.lower() not in _DTYPES.keys(): msg = ("Please choose a valid adas file type:\n" - + "\t- allowed: {}\n".format(_LTYPES) + + "\t- allowed: {}\n".format(_DTYPES.keys()) + "\t- provided: {}".format(Type)) raise Exception(msg) Type = Type.lower() @@ -156,8 +157,88 @@ def read_all(element=None, charge=None, Type=None, verb=None, **kwdargs): # ADF 15 # ############################################################################# -def _read_adf11(): - pass +def _read_adf11(pfe, deg=None): + if deg is None: + deg = _DEG + + # Get second order file type + typ1 = [vv for vv in _DTYPES['adf11'] if vv in pfe] + if len(typ1) != 1: + msg = ("Second order file type culd not be inferred from file name!\n" + + "\t- available: {}\n".format(_DTYPES['adf11']) + + "\t- provided: {}".format(pfe)) + raise Exception(msg) + + # Get element + elem = pfe[:-4].split('_')[1] + + if typ1[0] == 'scd': + # read blocks + nelog10 = np.array([]) + telog10 = np.array([]) + with open(pfe) as search: + for ii, line in enumerate(search): + + # Get atomic number (transitions) stored in this file + if ii == 0: + lstr = line.split('/') + lin = [ss for ss in lstr[0].strip().split(' ') + if ss.strip() != ''] + lc = [len(lin) == 5 and all([ss.isdigit() for ss in lin]), + elem.upper() in lstr[1], + 'ADF11' in lstr[2]] + if not all(lc): + msg = ("File header format seems to have changed!\n" + + "\t- lc = {}".format(lc)) + raise Exception(msg) + Z, nne, nte, q0, qend = map(int, lin) + coefs = np.full((nne, nte), np.nan) + # Not TRue !! one extra line in-between !!! + in_ne = True + continue + + # Get nelog10 + if in_ne: + nelog10 = np.append( + nelog10, + np.array(line.rstrip().strip().split(' '), + dtype=float)) + if nelog10.size == nne: + in_ne = False + in_te = True + + # Get telog10 + elif in_te is True: + telog10 = np.append( + telog10, + np.array(line.rstrip().strip().split(' '), + dtype=float)) + if telog10.size == nte: + in_te = False + in_ion = True + + # Get ion block + elif in_ion is True and '/IONIS' in line and 'Z1=' in line: + charge = line[line.index('Z1='+len('Z1=')):].split('/')[0] + charge = int(charge.strip())-1 + elif in_ion is True and charge is not None: + coefs = np.append( + coefs, + np.array(line.rstrip().strip().split(' '), + dtype=float)) + if coefs.size == nne*nte: + key = '{}{}_openadas_{}'.format(elem, charge, typ1) + func = scpRectSpl(nelog10, telog10, + coefs.reshape((nne, nte)), + kx=deg, ky=deg) + dout[key] = {'element': elem, + 'Z': Z, + 'charge': charge, + 'ionisation_interp2d_log10_nete': func} + if charge == Z-1: + break + + return dout # ############################################################################# @@ -262,16 +343,14 @@ def _read_adf15(pfe, if ind == pec.size: in_pec = False key = _get_adf15_key(elem, charge, isoel, typ0, typ1) - dout[key] = { - 'lamb': lamb, - 'origin': pfe, - 'type': typ[0], - 'ne': ne, 'te': te, - 'pec_interp2d_log_nete': - scpRectSpl(np.log(ne), np.log(te), - np.log(pec).reshape((nne, nte)), - kx=deg, ky=deg) - } + func = scpRectSpl(np.log(ne), np.log(te), + np.log(pec).reshape((nne, nte)), + kx=deg, ky=deg) + dout[key] = {'lamb': lamb, + 'origin': pfe, + 'type': typ[0], + 'ne': ne, 'te': te, + 'pec_interp2d_log_nete': func} # Get transitions from table at the end if 'photon emissivity atomic transitions' in line: diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index 6ac667a0e..27c96f5cc 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -256,6 +256,8 @@ def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, assert len(lstri) == ncol lamb = lstri[0].replace('Å', '') elm, charge = lstri[1].replace('', '').split('') + if charge == '+': + charge = '1+' if element is not None and elm.lower() != element.lower(): continue typ = lstri[2].replace('', '').split('>')[1] diff --git a/tofu/version.py b/tofu/version.py index 0fdcc51f9..83bbd1602 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-79-g39b1824e' +__version__ = '1.4.2b4-80-g11e72e08' From f657de57a3792ed848914064dbf6aa1dcebc0cd8 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 11:37:00 +0100 Subject: [PATCH 14/22] [Issue356] _read_adf11 operational and read_all() operational for adf15 and adf11 --- tofu/openadas2tofu/_read_files.py | 278 ++++++++++++++++++++---------- tofu/version.py | 2 +- 2 files changed, 191 insertions(+), 89 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 5401335d9..e065822e6 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -2,6 +2,7 @@ # Built-in import os import re +import itertools as itt # Common import numpy as np @@ -11,11 +12,16 @@ __all__ = ['read', 'read_all'] -_DTYPES = {'adf11': ['scd'], +_DTYPES = {'adf11': ['acd', 'ccd', 'scd', 'plt', 'prb'], 'adf15': None} _DEG = 1 +# ############################################################################# +# Utility functions +# ############################################################################# + + def _get_PATH_LOCAL(): pfe = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') if os.path.isdir(pfe): @@ -23,6 +29,26 @@ def _get_PATH_LOCAL(): else: return None +def _get_subdir_from_pattern(path, pattern): + ld = [dd for dd in os.listdir(path) + if (os.path.isdir(os.path.join(path, dd)) + and pattern in dd)] + if len(ld) != 1: + av = [dd for dd in os.listdir(path) + if os.path.isdir(os.path.join(path, dd))] + msg = ("You have no / many directories in your local " + + "~/.tofu/openadas2tofu/ matching the desired file type:\n" + + "\t- provided : {}\n".format(pattern) + + "\t- available: {}\n".format(av) + + " => download the data with tf.openadas2tofu.download()") + raise Exception(msg) + return os.path.join(path, ld[0]) + + +# ############################################################################# +# Main functions +# ############################################################################# + def read(adas_path, **kwdargs): """ Read openadas-formatted files and return a dict with the data """ @@ -68,30 +94,13 @@ def read(adas_path, **kwdargs): func = eval('_read_{}'.format(lc[0])) return func(pfe, **kwdargs) -def read_all(element=None, charge=None, Type=None, verb=None, **kwdargs): - # Check - if Type is None: - Type = 'adf15' - if not isinstance(Type, str) or Type.lower() not in _DTYPES.keys(): - msg = ("Please choose a valid adas file type:\n" - + "\t- allowed: {}\n".format(_DTYPES.keys()) - + "\t- provided: {}".format(Type)) - raise Exception(msg) - Type = Type.lower() - if not isinstance(element, str): - msg = "Please choose an element!" - raise Exception(msg) - element = element.lower() - if verb is None: - verb = True - if charge is not None and not isinstance(charge, int): - msg = "charge must be a int!" - raise Exception(msg) - - path_local = _get_PATH_LOCAL() +def read_all(element=None, charge=None, typ1=None, typ2=None, + verb=None, **kwdargs): + # -------------------- # Check whether the local .tofu repo exists, if not recommend tofu-custom + path_local = _get_PATH_LOCAL() if path_local is None: path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') msg = ("You do not seem to have a local ./tofu repository\n" @@ -103,63 +112,96 @@ def read_all(element=None, charge=None, Type=None, verb=None, **kwdargs): + "\ttofu-custom") raise Exception(msg) - # Get list of relevant directories - ld = [dd for dd in os.listdir(path_local) - if (os.path.isdir(os.path.join(path_local, dd)) - and Type in dd)] - if len(ld) != 1: - av = [dd for dd in os.listdir(path_local) - if os.path.isdir(os.path.join(path_local, dd))] - msg = ("You have no / many directories in your local " - + "~/.tofu/openadas2tofu/ matching the desired file type:\n" - + "\t- provided Type: {}\n".format(Type) - + "\t- available: {}\n".format(av) - + " => download the data with tf.openadas2tofu.download()") + # -------------------- + # Check / format input + if typ1 is None: + typ1 = 'adf15' + if not isinstance(typ1, str) or typ1.lower() not in _DTYPES.keys(): + msg = ("Please choose a valid adas file type:\n" + + "\t- allowed: {}\n".format(_DTYPES.keys()) + + "\t- provided: {}".format(typ1)) raise Exception(msg) - path = os.path.join(path_local, ld[0]) - ld = [dd for dd in os.listdir(path) - if (os.path.isdir(os.path.join(path, dd)) - and element in dd)] - if len(ld) != 1: - av = [dd for dd in os.listdir(path) - if os.path.isdir(os.path.join(path, dd))] - msg = ("You have no / many directories in your local " - + "~/.tofu/openadas2tofu/ matching the desired element:\n" - + "\t- provided element: {}\n".format(element) - + "\t- available: {}\n".format(av) - + " => download the data with tf.openadas2tofu.download()") + typ1 = typ1.lower() + if typ1 == 'adf11' and typ2 is None: + typ2 = _DTYPES[typ1] + fd = os.listdir(os.path.join(path_local, typ1)) + typ2 = [sorted([ss for ss in fd if tt in ss])[-1] for tt in typ2] + if isinstance(typ2, str): + typ2 = [typ2] + if (_DTYPES[typ1] is not None + and (not isinstance(typ2, list) + or not all([any([s1 in ss for s1 in _DTYPES[typ1]]) + for ss in typ2]))): + msg = ("typ2 must be a list of valid openadas file types for typ1:\n" + + "\t- provided: {}\n".format(typ2) + + "\t- available for {}: {}".format(typ1, _DTYPES[typ1])) + raise Exception(msg) + + if not isinstance(element, str): + msg = "Please choose an element!" + raise Exception(msg) + element = element.lower() + if charge is not None and not isinstance(charge, int): + msg = "charge must be a int!" raise Exception(msg) - path = os.path.join(path, ld[0]) + if verb is None: + verb = True + + # -------------------- + # Get list of relevant directories + + # Level 1: Type + path = _get_subdir_from_pattern(path_local, typ1) + # Level 2: element or typ2 + if typ1 == 'adf11': + lpath = [_get_subdir_from_pattern(path, tt) for tt in typ2] + elif typ1 == 'adf15': + lpath = [_get_subdir_from_pattern(path, element)] + + # -------------------- # Get list of relevant files pfe - lpfe = [os.path.join(path, ff) for ff in os.listdir(path) - if (os.path.isfile(os.path.join(path, ff)) - and ff[-4:] == '.dat' - and element in ff)] - if charge is not None: + lpfe = list(itt.chain.from_iterable( + [[os.path.join(path, ff) for ff in os.listdir(path) + if (os.path.isfile(os.path.join(path, ff)) + and ff[-4:] == '.dat' + and element in ff)] + for path in lpath])) + + if charge is not None and typ1 == 'adf15': lpfe = [ff for ff in lpfe if str(charge) in ff] + # -------------------- # Extract data from each file - func = eval('_read_{}'.format(Type)) - if Type == 'adf15': + func = eval('_read_{}'.format(typ1)) + if typ1 == 'adf15': out = {} for pfe in lpfe: if verb is True: msg = "\tLoading data from {}".format(pfe) print(msg) out.update(func(pfe, **kwdargs)) + elif typ1 == 'adf11': + dout = {} + for pfe in lpfe: + if verb is True: + msg = "\tLoading data from {}".format(pfe) + print(msg) + out = func(pfe, dout=dout, **kwdargs) return out # ############################################################################# -# ADF 15 +# Specialized functions for ADF 11 # ############################################################################# -def _read_adf11(pfe, deg=None): +def _read_adf11(pfe, deg=None, dout=None): if deg is None: deg = _DEG + if dout is None: + dout = {} # Get second order file type typ1 = [vv for vv in _DTYPES['adf11'] if vv in pfe] @@ -168,17 +210,22 @@ def _read_adf11(pfe, deg=None): + "\t- available: {}\n".format(_DTYPES['adf11']) + "\t- provided: {}".format(pfe)) raise Exception(msg) + typ1 = typ1[0] # Get element elem = pfe[:-4].split('_')[1] + comline = '-'*60 + comline2 = 'C'+comline + + if typ1 in ['acd', 'ccd', 'scd', 'plt', 'prb']: - if typ1[0] == 'scd': # read blocks - nelog10 = np.array([]) - telog10 = np.array([]) with open(pfe) as search: for ii, line in enumerate(search): + if comline2 in line: + break + # Get atomic number (transitions) stored in this file if ii == 0: lstr = line.split('/') @@ -192,57 +239,112 @@ def _read_adf11(pfe, deg=None): + "\t- lc = {}".format(lc)) raise Exception(msg) Z, nne, nte, q0, qend = map(int, lin) - coefs = np.full((nne, nte), np.nan) - # Not TRue !! one extra line in-between !!! + nelog10 = np.array([]) + telog10 = np.array([]) in_ne = True continue + if comline in line: + continue + # Get nelog10 if in_ne: - nelog10 = np.append( - nelog10, - np.array(line.rstrip().strip().split(' '), - dtype=float)) + li = [ss for ss in line.strip().split(' ') + if ss.strip() != ''] + nelog10 = np.append(nelog10, np.array(li, dtype=float)) if nelog10.size == nne: in_ne = False in_te = True # Get telog10 elif in_te is True: - telog10 = np.append( - telog10, - np.array(line.rstrip().strip().split(' '), - dtype=float)) + li = [ss for ss in line.strip().split(' ') + if ss.strip() != ''] + telog10 = np.append(telog10, np.array(li, dtype=float)) if telog10.size == nte: in_te = False in_ion = True # Get ion block - elif in_ion is True and '/IONIS' in line and 'Z1=' in line: - charge = line[line.index('Z1='+len('Z1=')):].split('/')[0] - charge = int(charge.strip())-1 + elif (in_ion is True and 'Z1=' in line + and ('------/' in line and 'DATE=' in line)): + nion = int( + line[line.index('Z1=')+len('Z1='):].split('/')[0]) + if typ1 in ['scd', 'plt']: + charge = nion - 1 + else: + charge = nion + coefslog10 = np.array([]) elif in_ion is True and charge is not None: - coefs = np.append( - coefs, - np.array(line.rstrip().strip().split(' '), - dtype=float)) - if coefs.size == nne*nte: - key = '{}{}_openadas_{}'.format(elem, charge, typ1) - func = scpRectSpl(nelog10, telog10, - coefs.reshape((nne, nte)), - kx=deg, ky=deg) - dout[key] = {'element': elem, - 'Z': Z, - 'charge': charge, - 'ionisation_interp2d_log10_nete': func} - if charge == Z-1: - break + li = [ss for ss in line.strip().split(' ') + if ss.strip() != ''] + coefslog10 = np.append(coefslog10, + np.array(li, dtype=float)) + if coefslog10.size == nne*nte: + key = '{}{}'.format(elem, charge) + tkv = [('element', elem), ('Z', Z), ('charge', charge)] + if key in dout.keys(): + assert all([dout[key][ss] == vv for ss, vv in tkv]) + else: + dout[key] = {ss: vv for ss, vv in tkv} + if typ1 == 'scd': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10-6 to convert cm3/s -> m3/s + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))-6, + kx=deg, ky=deg) + dout[key]['ionis'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(m3.s)', + 'source': pfe} + elif typ1 == 'acd': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10-6 to convert cm3/s -> m3/s + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))-6, + kx=deg, ky=deg) + dout[key]['recomb'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(m3.s)', + 'source': pfe} + elif typ1 == 'ccd': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10-6 to convert cm3/s -> m3/s + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))-6, + kx=deg, ky=deg) + dout[key]['recomb_ce'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(m3.s)', + 'source': pfe} + elif typ1 == 'plt': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10+6 to convert W.cm3 -> W.m3 + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))+6, + kx=deg, ky=deg) + dout[key]['rad_bb'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(W.m3)', + 'source': pfe} + elif typ1 == 'prb': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10+6 to convert W.cm3 -> W.m3 + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))+6, + kx=deg, ky=deg) + dout[key]['rad_fffb'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(W.m3)', + 'source': pfe} + if nion == Z: + break return dout # ############################################################################# -# ADF 15 +# Specialized functions for ADF 15 # ############################################################################# diff --git a/tofu/version.py b/tofu/version.py index 83bbd1602..480206e21 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-80-g11e72e08' +__version__ = '1.4.2b4-81-g029850ea' From 76d020d871ab064b4f987336ead44b7e97439faf Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 11:47:08 +0100 Subject: [PATCH 15/22] [Issue356] Harmonized output for adf11 and adf15 in read_all() --- tofu/openadas2tofu/_read_files.py | 25 +++++++++---------------- tofu/version.py | 2 +- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index e065822e6..3b3cbfa4c 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -174,20 +174,12 @@ def read_all(element=None, charge=None, typ1=None, typ2=None, # -------------------- # Extract data from each file func = eval('_read_{}'.format(typ1)) - if typ1 == 'adf15': - out = {} - for pfe in lpfe: - if verb is True: - msg = "\tLoading data from {}".format(pfe) - print(msg) - out.update(func(pfe, **kwdargs)) - elif typ1 == 'adf11': - dout = {} - for pfe in lpfe: - if verb is True: - msg = "\tLoading data from {}".format(pfe) - print(msg) - out = func(pfe, dout=dout, **kwdargs) + dout = {} + for pfe in lpfe: + if verb is True: + msg = "\tLoading data from {}".format(pfe) + print(msg) + out = func(pfe, dout=dout, **kwdargs) return out @@ -352,18 +344,19 @@ def _get_adf15_key(elem, charge, isoel, typ0, typ1): return '{}{}_{}_openadas_{}_{}'.format(elem, charge, isoel, typ0, typ1) -def _read_adf15(pfe, +def _read_adf15(pfe, dout=None, lambmin=None, lambmax=None, deg=None): if deg is None: deg = _DEG + if dout is None: + dout = {} # Get summary of transitions flagblock = '/isel =' flag0 = 'superstage partition information' - dout = {} # Get file markers from name (elem, charge, typ0, typ1) typ0, typ1, elemq = pfe.split('][')[1:] diff --git a/tofu/version.py b/tofu/version.py index 480206e21..ec6a5547d 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-81-g029850ea' +__version__ = '1.4.2b4-82-gf657de57' From f4208ff77274e1e223c6edea17e6f75440ec5377 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 13:04:20 +0100 Subject: [PATCH 16/22] [Issue356] Added docstr to read() and read_all() --- tofu/openadas2tofu/_read_files.py | 57 +++++++++++++++++++++++++++---- tofu/version.py | 2 +- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 3b3cbfa4c..191aac47e 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -51,7 +51,19 @@ def _get_subdir_from_pattern(path, pattern): def read(adas_path, **kwdargs): - """ Read openadas-formatted files and return a dict with the data """ + """ Read openadas-formatted files and return a dict with the data + + Povide the full adas file name + The result is returned as a dict + + example + ------- + >>> import tofu as tf + >>> fn = '/adf11/scd74/scd74_ar.dat' + >>> out = tf.openadas2tofu.read(fn) + >>> fn = '/adf15/pec40][ar/pec40][ar_ca][ar16.dat' + >>> out = tf.openadas2tofu.read(fn) + """ path_local = _get_PATH_LOCAL() @@ -97,6 +109,32 @@ def read(adas_path, **kwdargs): def read_all(element=None, charge=None, typ1=None, typ2=None, verb=None, **kwdargs): + """ Read all relevant openadas files for chosen typ1 + + Please specify: + - typ1: 'adf11' or 'adf15' + - element: the symbol of the element + + If typ1 = 'adf11', you can also provide typ2 to specify the coefficients: + - 'scd': effective ionisation coefficients + - 'acd': effective electron-impact recombination coefficients + - 'ccd': effective hydrogen-impact recombination coefficients + - 'plt': line power due to electron-impact excitation + - 'prc': line power due to hydrogen-impact excitation + - 'prb': rad. recombination and bremmstrahlung due to electron-impact + + If typ1 = 'adf15', you can optioanlly provide a min/max wavelength + + The result is returned as a dict + + examples + -------- + >>> import tofu as tf + >>> dout = tf.openadas2tofu.read_all(element='ar', typ1='adf11') + >>> dout = tf.openadas2tofu.read_all(element='ar', typ1='adf15', + charge=16, + lambmin=3.94e-10, lambmax=4.e-10) + """ # -------------------- # Check whether the local .tofu repo exists, if not recommend tofu-custom @@ -287,7 +325,7 @@ def _read_adf11(pfe, deg=None, dout=None): kx=deg, ky=deg) dout[key]['ionis'] = {'func': func, 'type': 'log10_nete', - 'units': 'log10(m3.s)', + 'units': 'log10(m3/s)', 'source': pfe} elif typ1 == 'acd': # nelog10+6 to convert /cm3 -> /m3 @@ -297,7 +335,7 @@ def _read_adf11(pfe, deg=None, dout=None): kx=deg, ky=deg) dout[key]['recomb'] = {'func': func, 'type': 'log10_nete', - 'units': 'log10(m3.s)', + 'units': 'log10(m3/s)', 'source': pfe} elif typ1 == 'ccd': # nelog10+6 to convert /cm3 -> /m3 @@ -307,7 +345,7 @@ def _read_adf11(pfe, deg=None, dout=None): kx=deg, ky=deg) dout[key]['recomb_ce'] = {'func': func, 'type': 'log10_nete', - 'units': 'log10(m3.s)', + 'units': 'log10(m3/s)', 'source': pfe} elif typ1 == 'plt': # nelog10+6 to convert /cm3 -> /m3 @@ -438,14 +476,19 @@ def _read_adf15(pfe, dout=None, if ind == pec.size: in_pec = False key = _get_adf15_key(elem, charge, isoel, typ0, typ1) - func = scpRectSpl(np.log(ne), np.log(te), - np.log(pec).reshape((nne, nte)), + # log(ne)+6 to convert /cm3 -> /m3 + # log(pec)+6 to convert cm3/s -> m3/s + func = scpRectSpl(np.log(ne)+6, np.log(te), + np.log(pec).reshape((nne, nte))+6, kx=deg, ky=deg) dout[key] = {'lamb': lamb, 'origin': pfe, 'type': typ[0], 'ne': ne, 'te': te, - 'pec_interp2d_log_nete': func} + 'pec': {'func': func + 'type': 'log_nete', + 'units': 'log(m3/s)', + 'source': 'pfe'}} # Get transitions from table at the end if 'photon emissivity atomic transitions' in line: diff --git a/tofu/version.py b/tofu/version.py index ec6a5547d..671b1f4e5 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-82-gf657de57' +__version__ = '1.4.2b4-83-g76d020d8' From 26371bf7eaefe489706534af13034aab56de6c33 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 13:28:48 +0100 Subject: [PATCH 17/22] [Issue356] Minor typo --- tofu/openadas2tofu/_read_files.py | 2 +- tofu/openadas2tofu/_requests.py | 23 ++++++++++++++++------- tofu/version.py | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 191aac47e..439e77eb9 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -485,7 +485,7 @@ def _read_adf15(pfe, dout=None, 'origin': pfe, 'type': typ[0], 'ne': ne, 'te': te, - 'pec': {'func': func + 'pec': {'func': func, 'type': 'log_nete', 'units': 'log(m3/s)', 'source': 'pfe'}} diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index 27c96f5cc..52b37d954 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -177,7 +177,8 @@ def search_online(searchstr=None, returnas=None, def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, - element=None, returnas=None, verb=None): + element=None, charge=None, + returnas=None, verb=None): """ Perform an online search by wavelength on https://open.adas.ac.uk Pass the min / max wavelength (in Angstrom) to the online wavelength search @@ -212,9 +213,15 @@ def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, + "\t- 'file': list all files containing relevant transitions") raise Exception(msg) if element is not None and not isinstance(element, str): - msg = ("Arg element must be a str!\n" - + "\te.g.: element='ar'") + msg = ("Arg element must be a str (e.g.: element='ar')\n" + + "\t- provided: {}".format(element)) raise Exception(msg) + if charge is not None: + if not isinstance(charge, int): + msg = ("Arg charge must be a int!\n" + + "\t- provided: {}".format(charge)) + raise Exception(msg) + charge = '0' if charge == 0 else '{}+'.format(charge) searchurl = '&'.join(['wave_min={}'.format(lambmin), 'wave_max={}'.format(lambmax), @@ -255,11 +262,13 @@ def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, lstri = out[ii+1].split('') assert len(lstri) == ncol lamb = lstri[0].replace('Å', '') - elm, charge = lstri[1].replace('', '').split('') - if charge == '+': - charge = '1+' + elm, charg = lstri[1].replace('', '').split('') + if charg == '+': + charg = '1+' if element is not None and elm.lower() != element.lower(): continue + if charge is not None and charg != charge: + continue typ = lstri[2].replace('', '').split('>')[1] trans = lstri[3].replace(' ', ' ') trans = trans.replace('', '^{').replace('', '}') @@ -267,7 +276,7 @@ def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, trans = trans.replace('→', '->') fil = lstri[4][lstri[4].index('detail')+len('detail'):] fil = fil[:fil.index('.dat')+len('.dat')] - lout.append([lamb, elm, charge, typ, trans, fil]) + lout.append([lamb, elm, charg, typ, trans, fil]) # Format output char = np.array(lout) diff --git a/tofu/version.py b/tofu/version.py index 671b1f4e5..2ab67bd31 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-83-g76d020d8' +__version__ = '1.4.2b4-84-gf4208ff7' From 2d4ab610c4c4a27cf00d5190679b4bdfef69ee13 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 13:38:41 +0100 Subject: [PATCH 18/22] [Issue356] PEPA compliance --- tofu/openadas2tofu/_read_files.py | 19 +++++++++++-------- tofu/openadas2tofu/_requests.py | 23 +++++++++++------------ tofu/scripts/tofucustom.py | 3 +++ tofu/version.py | 2 +- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 439e77eb9..62b971e2f 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -29,6 +29,7 @@ def _get_PATH_LOCAL(): else: return None + def _get_subdir_from_pattern(path, pattern): ld = [dd for dd in os.listdir(path) if (os.path.isdir(os.path.join(path, dd)) @@ -221,12 +222,11 @@ def read_all(element=None, charge=None, typ1=None, typ2=None, return out - - # ############################################################################# # Specialized functions for ADF 11 # ############################################################################# + def _read_adf11(pfe, deg=None, dout=None): if deg is None: deg = _DEG @@ -260,7 +260,7 @@ def _read_adf11(pfe, deg=None, dout=None): if ii == 0: lstr = line.split('/') lin = [ss for ss in lstr[0].strip().split(' ') - if ss.strip() != ''] + if ss.strip() != ''] lc = [len(lin) == 5 and all([ss.isdigit() for ss in lin]), elem.upper() in lstr[1], 'ADF11' in lstr[2]] @@ -382,6 +382,7 @@ def _get_adf15_key(elem, charge, isoel, typ0, typ1): return '{}{}_{}_openadas_{}_{}'.format(elem, charge, isoel, typ0, typ1) + def _read_adf15(pfe, dout=None, lambmin=None, lambmax=None, @@ -416,7 +417,7 @@ def _read_adf15(pfe, dout=None, # Get number of lines (transitions) stored in this file if ii == 0: lstr = line.split('/') - nlines = int(lstr[0].replace(' ','')) + nlines = int(lstr[0].replace(' ', '')) continue # Get info about the transition being scanned (block) @@ -425,8 +426,9 @@ def _read_adf15(pfe, dout=None, lamb = float(lstr[0])*1.e-10 isoel = nblock + 1 nblock += 1 - if ((lambmin is not None and lamb < lambmin) - or (lambmax is not None and lamb > lambmax)): + c0 = ((lambmin is not None and lamb < lambmin) + and (lambmax is not None and lamb > lambmax)) + if c0: skip = True continue skip = False @@ -500,8 +502,9 @@ def _read_adf15(pfe, dout=None, isoel = int(lstr[1]) lamb = float(lstr[2])*1.e-10 key = _get_adf15_key(elem, charge, isoel, typ0, typ1) - if ((lambmin is None or lambmin < lamb) - and (lambmax is None or lambmax > lamb)): + c0 = ((lambmin is None or lambmin < lamb) + and (lambmax is None or lambmax > lamb)) + if c0: if key not in dout.keys(): msg = "Inconsistency in file {}".format(pfe) raise Exception(msg) diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index 52b37d954..3e9e3e920 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -9,7 +9,6 @@ import numpy as np - __all__ = ['search_online', 'search_online_by_wavelengthA', 'download', 'download_all', 'clean_downloads'] @@ -23,6 +22,7 @@ _INCLUDE_PARTIAL = True + # ############################################################################# # Utility functions # ############################################################################# @@ -45,7 +45,7 @@ def _getcharray(ar, col=None, sep=' ', line='-', just='l', ar = np.array(ar, dtype='U') if ar.ndim == 1: - ar = ar.reshape((1,ar.size)) + ar = ar.reshape((1, ar.size)) # Get just len nn = np.char.str_len(ar).max(axis=0) @@ -62,13 +62,13 @@ def _getcharray(ar, col=None, sep=' ', line='-', just='l', # Apply to array fjust = np.char.ljust if just == 'l' else np.char.rjust - out = np.array([sep.join(v) for v in fjust(ar,nn)]) + out = np.array([sep.join(v) for v in fjust(ar, nn)]) # Apply to col if col is not None: arcol = np.array([col, [line*n for n in nn]], dtype='U') - arcol = np.array([sep.join(v) for v in fjust(arcol,nn)]) - out = np.append(arcol,out) + arcol = np.array([sep.join(v) for v in fjust(arcol, nn)]) + out = np.append(arcol, out) if verb is True: print('\n'.join(out)) @@ -132,7 +132,7 @@ def search_online(searchstr=None, returnas=None, + "in requests.get({}).text".format(total_url)) raise Exception(msg) ind1 = np.min([ii for ii in ind1 if ii > ind0[0]]) - out = out[ind0[0]+1:ind1-1] + out = out[ind0[0] + 1:ind1-1] nresults = len(out) -1 # Get columns @@ -243,7 +243,7 @@ def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, + "\t- {}\n".format(flag1) + "in requests.get({}).text".format(total_url)) raise Exception(msg) - out = out[ind0[0]+1].split('') + out = out[ind0[0] + 1].split('') nresults = len(out) -1 # Get columns @@ -294,8 +294,6 @@ def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, def _check_exists(filename, update=None): - # if target is None: - # target = filename # In case a small modification becomes necessary later target = filename @@ -358,8 +356,9 @@ def download(filename=None, if returnas is None: returnas = False - if (not isinstance(filename, str) - or filename[:4] != '/adf' or filename[-4:] != '.dat'): + c0 = (not isinstance(filename, str) + or filename[:4] != '/adf' or filename[-4:] != '.dat') + if c0: msg = ("filename must be a str (full file name) of the form:\n" + "\t/adf.../.../....dat") raise Exception(msg) @@ -462,7 +461,7 @@ def download_all(files=None, searchstr=None, else: msg = "\tdownloaded: \t{}".format(files[ii]) except FileAlreayExistsException: - msg = "\talready exists: {}".format(files[ii]) + msg = "\talready exists: {}".format(files[ii]) except Exception as err: msg = (str(err) + "\n\nCould not download file {}".format(files[ii])) diff --git a/tofu/scripts/tofucustom.py b/tofu/scripts/tofucustom.py index b1449b004..397d3fb4f 100755 --- a/tofu/scripts/tofucustom.py +++ b/tofu/scripts/tofucustom.py @@ -12,6 +12,7 @@ # default values ################################################### + _SOURCE = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) _USER = getpass.getuser() _USER_HOME = os.path.expanduser('~') @@ -19,11 +20,13 @@ _LF = ['_imas2tofu_def.py'] _LD = ['openadas2tofu'] + ################################################### ################################################### # function ################################################### + def custom(target=_TARGET, source=_SOURCE, files=_LF, directories=_LD): diff --git a/tofu/version.py b/tofu/version.py index 2ab67bd31..8b390266c 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-84-gf4208ff7' +__version__ = '1.4.2b4-85-g26371bf7' From 328d320bacb7fb0787d6ae4dc659789c2cc4dd8b Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 13:40:41 +0100 Subject: [PATCH 19/22] [Issue356] PEP8 compliance 2 --- tofu/openadas2tofu/_read_files.py | 5 +++-- tofu/openadas2tofu/_requests.py | 4 ++-- tofu/version.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 62b971e2f..734cfb8b4 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -512,8 +512,9 @@ def _read_adf15(pfe, dout=None, if dout[key]['lamb'] != lamb: msg = "Inconsistency in file {}".format(pfe) raise Exception(msg) - if (dout[key]['type'] not in lstr - or lstr.index(dout[key]['type']) < 4): + c0 = (dout[key]['type'] not in lstr + or lstr.index(dout[key]['type']) < 4) + if c0: msg = ("Inconsistency in table, type not found:\n" + "\t- expected: {}\n".format(dout[key]['type']) + "\t- line: {}".format(line)) diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py index 3e9e3e920..ab1f24ebb 100644 --- a/tofu/openadas2tofu/_requests.py +++ b/tofu/openadas2tofu/_requests.py @@ -133,7 +133,7 @@ def search_online(searchstr=None, returnas=None, raise Exception(msg) ind1 = np.min([ii for ii in ind1 if ii > ind0[0]]) out = out[ind0[0] + 1:ind1-1] - nresults = len(out) -1 + nresults = len(out) - 1 # Get columns heads = [str.replace(kk.replace('', '').replace('', ''), @@ -244,7 +244,7 @@ def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, + "in requests.get({}).text".format(total_url)) raise Exception(msg) out = out[ind0[0] + 1].split('') - nresults = len(out) -1 + nresults = len(out) - 1 # Get columns col = [kk.replace('', '').replace('', '').strip() diff --git a/tofu/version.py b/tofu/version.py index 8b390266c..51812ea0c 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-85-g26371bf7' +__version__ = '1.4.2b4-86-g2d4ab610' From b9a55dc7efe8438eee63c1217fe74eb7506c33b2 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 14:39:36 +0100 Subject: [PATCH 20/22] [Issue356] Typo in lambmin / lambmax selection in _read_adf15() --- tofu/openadas2tofu/_read_files.py | 10 +++++----- tofu/version.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 734cfb8b4..7feb8abf5 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -427,7 +427,7 @@ def _read_adf15(pfe, dout=None, isoel = nblock + 1 nblock += 1 c0 = ((lambmin is not None and lamb < lambmin) - and (lambmax is not None and lamb > lambmax)) + or (lambmax is not None and lamb > lambmax)) if c0: skip = True continue @@ -504,10 +504,10 @@ def _read_adf15(pfe, dout=None, key = _get_adf15_key(elem, charge, isoel, typ0, typ1) c0 = ((lambmin is None or lambmin < lamb) and (lambmax is None or lambmax > lamb)) - if c0: - if key not in dout.keys(): - msg = "Inconsistency in file {}".format(pfe) - raise Exception(msg) + if c0 and key not in dout.keys(): + msg = ("Inconsistency in file {}:\n".format(pfe) + + "\t- line should be present".format(key)) + raise Exception(msg) if key in dout.keys(): if dout[key]['lamb'] != lamb: msg = "Inconsistency in file {}".format(pfe) diff --git a/tofu/version.py b/tofu/version.py index 0cc03c599..a488d22d5 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-86-g2d4ab610' \ No newline at end of file +__version__ = '1.4.2b4-90-g0d3af17c' From f832ab7e754cd32bf743a7f9713bbc42337fc3af Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 14:50:37 +0100 Subject: [PATCH 21/22] [Issue356] Improve consistency with Issue202 for dlines dict out of _read_adf15() --- tofu/openadas2tofu/_read_files.py | 4 +++- tofu/version.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 7feb8abf5..8a7bf9fc4 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -483,7 +483,9 @@ def _read_adf15(pfe, dout=None, func = scpRectSpl(np.log(ne)+6, np.log(te), np.log(pec).reshape((nne, nte))+6, kx=deg, ky=deg) - dout[key] = {'lamb': lamb, + dout[key] = {'lambda': lamb, + 'ION': '{}{}+'.format(elem, charge), + 'symbol': '{}{}-{}'.format(typ0, typ1, isoel), 'origin': pfe, 'type': typ[0], 'ne': ne, 'te': te, diff --git a/tofu/version.py b/tofu/version.py index a488d22d5..c1f164581 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-90-g0d3af17c' +__version__ = '1.4.2b4-91-gb9a55dc7' From 3ee8536ea38c1c1b70c8654210d39567f54b3224 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 6 Mar 2020 14:53:18 +0100 Subject: [PATCH 22/22] [Issue356] Typo --- tofu/openadas2tofu/_read_files.py | 2 +- tofu/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tofu/openadas2tofu/_read_files.py b/tofu/openadas2tofu/_read_files.py index 8a7bf9fc4..cebe3a623 100644 --- a/tofu/openadas2tofu/_read_files.py +++ b/tofu/openadas2tofu/_read_files.py @@ -511,7 +511,7 @@ def _read_adf15(pfe, dout=None, + "\t- line should be present".format(key)) raise Exception(msg) if key in dout.keys(): - if dout[key]['lamb'] != lamb: + if dout[key]['lambda'] != lamb: msg = "Inconsistency in file {}".format(pfe) raise Exception(msg) c0 = (dout[key]['type'] not in lstr diff --git a/tofu/version.py b/tofu/version.py index c1f164581..443c713ec 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b4-91-gb9a55dc7' +__version__ = '1.4.2b4-92-gf832ab7e'