From f577c2873b17d9f305426f00dd0c8e82b7a79f8c Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 31 Jan 2020 10:43:39 +0100 Subject: [PATCH 1/6] [1.4.2] Added docstr to MultiIDSLoader.open_get_close(), and try to force tagged commit on master --- tofu/imas2tofu/_core.py | 12 ++++++++++++ tofu/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tofu/imas2tofu/_core.py b/tofu/imas2tofu/_core.py index 65ee4ef48..604cf5128 100644 --- a/tofu/imas2tofu/_core.py +++ b/tofu/imas2tofu/_core.py @@ -1262,6 +1262,18 @@ def get_list_notget_ids(self): return lids def open_get_close(self, ids=None, occ=None, verb=True): + """ Force data loading + + If at instanciation or when using method add_ids() you specified option + get = False, then the latest added ids may not have been loaded. + + This method forces a refresh and loads all ids contained in the instance + + The name comes from: + - open (all the idd) + - get (all the ids) + - close (all the idd) + """ llids = self._checkformat_get_ids(ids) lidd = [lids[0] for lids in llids] self._open(idd=lidd) diff --git a/tofu/version.py b/tofu/version.py index 5db546bba..72a6b4746 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b13-86-g0840bdc4' +__version__ = '1.4.2b13-89-g45919172' From b5104f988c9715bd6306b675b38aaa111e6c64f2 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Fri, 31 Jan 2020 11:18:07 +0100 Subject: [PATCH 2/6] [1.4.2] Forcing version number by typing in tofu/version.py and using git commit --no-verify --- tofu/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tofu/version.py b/tofu/version.py index 72a6b4746..fbadd4a44 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2b13-89-g45919172' +__version__ = '1.4.2' From d37e4a41d4caca5313a74b76e4c26421f56062f1 Mon Sep 17 00:00:00 2001 From: Laura Mendoza Date: Fri, 31 Jan 2020 14:50:45 +0100 Subject: [PATCH 3/6] [bf #340] trying to solve deployment issues --- .travis.yml | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index f512b11ba..5cc833b73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,23 +68,29 @@ after_success: - codecov - chmod +x $START/anaconda_upload.sh - echo $TRAVIS_TAG + before_deploy: -- echo "BEFORE DEPLOY START........" -- ls $START -- cd $START -- conda config --set anaconda_upload no -- conda install anaconda-client conda-build -- conda build conda_recipe -- export PKG_REAL=$(conda build . --output | tail -1) -- echo "BEFORE DEPLOY END.........." + - > + if ! [ "$BEFORE_DEPLOY_RUN" ]; then + export BEFORE_DEPLOY_RUN=1; + echo "BEFORE DEPLOY START........" + ls $START + cd $START + conda config --set anaconda_upload no + conda install anaconda-client conda-build + conda build conda_recipe + export PKG_REAL=$(conda build . --output | tail -1) + echo "BEFORE DEPLOY END.........." + fi deploy: - provider: pypi distributions: sdist user: __token__ skip_existing: true + skip_cleanup: true on: - tags: true branch: deploy-test + tags: true server: https://test.pypi.org/legacy/ password: secure: xfVFuoz9YYNChzmT8DC9y+8eH6zdFkfoy3B51uqy8b+vhJNzCzLay4F0uSHvhHy6iYorM6UQKr6soC4D7n3PhmnFOTX/cgLtd/p4gBWGYZF6yXacvw+UHKMshgbAhn2sEynxdSAqdAlNttMI8jsUu9RhbzGiv1l5zSNnFWF4Zsly02G68UnztxIGoz8AYTRW2N2oQhGrl/ryj/YG4mSRKjled6BzK7kNoJUqLGl12DqdMMTEmdJ9NHBXgK3Dv0ya17ReFz3TcxE/4+Yc38NwSR4Ia2EvVSMtyIaccQ1uSrXwW8JQOMn+9CmDWZVUMDD2bzKYbm2WGGM9Fh8WrHnwlWRujoLDofhYEK0Cus11gULFF+J88XucOJlyJNrHP6TWxdSVVoQfwWr2ABqZIvilsvHpF+sjDLqomTNHdi+BbzP2koRv0nJb9K1W24bjPLtSK8+plX7suv7gdBNwlsJ+dPLDM87v4+jGHGthQ6P4X2guTMHZm1PU0PSPB9LCbENCN1uktLLhkgx7gZ42Ag+Jwiu02ENkChLaEB4WpPb9mjLnomu5LDYXFGtPJ/uLMOi3VCXyda0LrzqDhXYT3Cg4hvXySwJcgMYSXalfTxnTm9oouePiEXDbK+XwjMP9mjC5CeMg3SaFFTywqaTH0WUqiOBUJ6H3Gsm0sB15Tj4lNKQ= @@ -92,10 +98,11 @@ deploy: distributions: bdist_wheel user: __token__ skip_existing: true + skip_cleanup: true on: + branch: deploy-test condition: $OS = osx-64 tags: true - branch: deploy-test server: https://test.pypi.org/legacy/ password: secure: xfVFuoz9YYNChzmT8DC9y+8eH6zdFkfoy3B51uqy8b+vhJNzCzLay4F0uSHvhHy6iYorM6UQKr6soC4D7n3PhmnFOTX/cgLtd/p4gBWGYZF6yXacvw+UHKMshgbAhn2sEynxdSAqdAlNttMI8jsUu9RhbzGiv1l5zSNnFWF4Zsly02G68UnztxIGoz8AYTRW2N2oQhGrl/ryj/YG4mSRKjled6BzK7kNoJUqLGl12DqdMMTEmdJ9NHBXgK3Dv0ya17ReFz3TcxE/4+Yc38NwSR4Ia2EvVSMtyIaccQ1uSrXwW8JQOMn+9CmDWZVUMDD2bzKYbm2WGGM9Fh8WrHnwlWRujoLDofhYEK0Cus11gULFF+J88XucOJlyJNrHP6TWxdSVVoQfwWr2ABqZIvilsvHpF+sjDLqomTNHdi+BbzP2koRv0nJb9K1W24bjPLtSK8+plX7suv7gdBNwlsJ+dPLDM87v4+jGHGthQ6P4X2guTMHZm1PU0PSPB9LCbENCN1uktLLhkgx7gZ42Ag+Jwiu02ENkChLaEB4WpPb9mjLnomu5LDYXFGtPJ/uLMOi3VCXyda0LrzqDhXYT3Cg4hvXySwJcgMYSXalfTxnTm9oouePiEXDbK+XwjMP9mjC5CeMg3SaFFTywqaTH0WUqiOBUJ6H3Gsm0sB15Tj4lNKQ= @@ -103,6 +110,7 @@ deploy: distributions: sdist user: "Didou09" skip_existing: true + skip_cleanup: true on: tags: true branch: master @@ -112,6 +120,7 @@ deploy: distributions: bdist_wheel user: "Didou09" skip_existing: true + skip_cleanup: true on: condition: $OS = osx-64 tags: true @@ -119,8 +128,8 @@ deploy: password: secure: JNEDTDJVx/2fXNfHntNQ99iDRNuQ4uB3y+DBWVIBycCT95+UCb36YPtKzmruEk/UUS29Xgq4IYCGdfCSWE9smKqG8tV1PcHiw705m+AzcpKy77YtzbVECFBxqY4W36O2pHrkwEUzP/7acjFwNsnUFzArqEzsBJ+KdLaa4OPHJXCh30GA0GyqlrXYbBKG+DA9hX5vtsGo4C6w9noALYF3fS7pKPiI6ipKFnAlzGgHQ7Ke0uQME8N3IAFhmh+Z5xMtIIDWxlnqv+KszdG4DIaGV/W6NIJNAbRhzkqUd+Chu6LoPAd/XkHDTeirR/MBkNUc5UcRJxRnP9rUTRo1gCO/buTYuNRgFkMvqhV5a033+x9edWgtUiKNJIMPLXOxe0RJvc5GWji+Co77HtHxRmGRM2rnYqWMtZeYZlFbUdvHu/8jf0d6I8jyUgAoJYdlMA2u/ipENP3S6by4epE9qycUPXiIVh6r3DZbf3vPTMFvTZYAjBrA0NOzihv1xgcXwemmNUFOQSpe0io4UcFxtS9lLMo+30UMQjCHSnbEVM3zSlZmbMOKpkVOlKlt8Lz5NxwVgWtu9FuW2pGukLtE8AWbqvY9urXAPZCQqZlOIklIjJQIqOITnuw9LEV09cgvPHXfdvNni3ldbMlIQ89zryM6dYvhYryTiEZGK4JDR3wAKJA= - provider: script - user: "ToFuProject" script: $START/anaconda_upload.sh + skip_cleanup: true on: tags: true branch: master From 05ba14fe6ecca803dcdc14d60c88dc3143763fa3 Mon Sep 17 00:00:00 2001 From: Laura Mendoza Date: Fri, 31 Jan 2020 15:05:55 +0100 Subject: [PATCH 4/6] [bf #340] added back package_dir --- anaconda_upload.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/anaconda_upload.sh b/anaconda_upload.sh index 8d4ee2f68..e0f5a9a66 100644 --- a/anaconda_upload.sh +++ b/anaconda_upload.sh @@ -6,6 +6,7 @@ set -e echo "Deploying to anaconda.org..." export USER=ToFuProject +export PKG_DIR=$HOME/miniconda/conda-bld/$OS/ anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l main $PKG_DIR/tofu-*.tar.bz2 echo "Successfully uploaded !" exit 0 From 8869ea3b726afa03b66715d7d4b5a09e63d52870 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 3 Feb 2020 16:30:22 +0100 Subject: [PATCH 5/6] [1.4.2 Hotfix] Added extra keyword to tofuplot to plot data without equilibrium / heating --- tofu/imas2tofu/_core.py | 12 +++++++++--- tofu/utils.py | 4 ++-- tofuplot.py | 10 +++++++--- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tofu/imas2tofu/_core.py b/tofu/imas2tofu/_core.py index db1bc1f74..2b47dfb5b 100644 --- a/tofu/imas2tofu/_core.py +++ b/tofu/imas2tofu/_core.py @@ -3882,7 +3882,11 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, ipdb.set_trace() raise Exception(msg) - assert out[dsig[kk]].ndim in [1, 2, 3] + if out[dsig[kk]].size == 0 or out[dsig[kk]].ndim not in [1, 2, 3]: + msg = ("\nSome data seem to have inconsistent shape:\n" + + "\t- out[{}].shape = {}".format(dsig[kk], + out[dsig[kk]].shape)) + raise Exception(msg) if out[dsig[kk]].ndim == 1: out[dsig[kk]] = np.atleast_2d(out[dsig[kk]]) @@ -3890,6 +3894,7 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, if out[dsig[kk]].ndim == 2: if dsig[kk] in ['X','lamb']: if np.allclose(out[dsig[kk]], out[dsig[kk]][:,0:1]): + import pdb; pdb.set_trace() # DB dins[kk] = out[dsig[kk]][:,0] else: dins[kk] = out[dsig[kk]] @@ -3945,8 +3950,9 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, if t0 != False: if 't' in dins.keys(): dins['t'] = dins['t'] - t0 - for tt in dextra.keys(): - dextra[tt]['t'] = dextra[tt]['t'] - t0 + if dextra is not None: + for tt in dextra.keys(): + dextra[tt]['t'] = dextra[tt]['t'] - t0 # -------------- # Create objects diff --git a/tofu/utils.py b/tofu/utils.py index 7868d9b39..b2343fb02 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -651,7 +651,7 @@ def load_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, occ=None, indch=None, description_2d=None, equilibrium=None, dsig=None, data=None, X=None, t0=None, dextra=None, plot=True, plot_sig=None, plot_X=None, - sharex=False, invertx=None, + sharex=False, invertx=None, extra=True, bck=True, indch_auto=True, t=None, init=None, dR_sep=None): # ------------------- # import imas2tofu @@ -922,7 +922,7 @@ def load_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, if nDat > 0 or nCam > 0 or nPla > 0: if 'wall' not in lids: lids.append('wall') - if nDat > 0 or nPla > 0 and dextra is None: + if (nDat > 0 or nPla > 0) and extra is True: if 'equilibrium' not in lids: lids.append('equilibrium') if 'lh_antennas' not in lids: diff --git a/tofuplot.py b/tofuplot.py index 7b8ed9906..5259d3854 100755 --- a/tofuplot.py +++ b/tofuplot.py @@ -53,6 +53,7 @@ _T0 = 'IGNITRON' _SHAREX = False _BCK = True +_EXTRA = True ################################################### ################################################### @@ -72,7 +73,7 @@ def _get_exception(q, ids, qtype='quantity'): def call_tfloadimas(shot=None, run=_RUN, user=_USER, - tokamak=_TOKAMAK, version=_VERSION, + tokamak=_TOKAMAK, version=_VERSION, extra=_EXTRA, ids=None, quantity=None, X=None, t0=_T0, sharex=_SHAREX, indch=None, indch_auto=None, background=_BCK, t=None, dR_sep=None, init=None): @@ -84,7 +85,7 @@ def call_tfloadimas(shot=None, run=_RUN, user=_USER, tf.load_from_imas(shot=shot, run=run, user=user, tokamak=tokamak, version=version, ids=ids, indch=indch, indch_auto=indch_auto, - plot_sig=quantity, plot_X=X, + plot_sig=quantity, plot_X=X, extra=extra, t0=t0, plot=True, sharex=sharex, bck=background, t=t, dR_sep=dR_sep, init=init) @@ -156,9 +157,12 @@ def _str2bool(v): parser.add_argument('-ich', '--indch', type=int, required=False, help='indices of channels to be loaded', nargs='+', default=None) - parser.add_argument('-ichauto', '--indch_auto', type=bool, required=False, + parser.add_argument('-ichauto', '--indch_auto', type=_str2bool, required=False, help='automatically determine indices of' + ' channels to be loaded', default=True) + parser.add_argument('-e', '--extra', type=_str2bool, required=False, + help='If True loads separatrix and heating power', + default=_EXTRA) parser.add_argument('-sx', '--sharex', type=_str2bool, required=False, help='Should X axis be shared between diagnostics ids ?', default=_SHAREX, const=True, nargs='?') From 45b15957d596addf4db02f670ba8599adab5a0e7 Mon Sep 17 00:00:00 2001 From: VEZINET Didier Date: Mon, 3 Feb 2020 16:44:42 +0100 Subject: [PATCH 6/6] [1.4.2 Hotfix] Added --extra to tofucalc and synthdiag=True to simplify code in tofu/utils.calc_from_imas() --- tofu/utils.py | 26 +++++++++++++------------- tofucalc.py | 8 ++++++-- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tofu/utils.py b/tofu/utils.py index b2343fb02..82e053a2c 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -1039,7 +1039,7 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, ids=None, Name=None, out=None, tlim=None, config=None, occ=None, indch=None, description_2d=None, equilibrium=None, dsig=None, data=None, X=None, t0=None, dextra=None, - Brightness=None, res=None, interp_t=None, + Brightness=None, res=None, interp_t=None, extra=True, plot=True, plot_compare=True, sharex=False, bck=True, indch_auto=True, t=None, init=None): # ------------------- @@ -1161,7 +1161,7 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, if nDat > 0 or nCam > 0 or nPla > 0: if 'wall' not in lids: lids.append('wall') - if nDat > 0 or nPla > 0 and dextra is None: + if (nDat > 0 or nPla > 0) and extra is True: if 'equilibrium' not in lids: lids.append('equilibrium') if 'lh_antennas' not in lids: @@ -1173,16 +1173,16 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, lids.append('pulse_schedule') # Complement ids in diag-specific way - for ids in lids: - if ids in imas2tofu.MultiIDSLoader._didsdiag.keys(): - dd = imas2tofu.MultiIDSLoader._didsdiag[ids] - if dd.get('synth') is not None: - for v0 in dd['synth']['dsynth'].values(): - for v1 in v0: - if '.' in v1: - v20, v21 = v1.split('.') - if v20 not in lids: - lids.append(v20) + # for ids in lids: + # if ids in imas2tofu.MultiIDSLoader._didsdiag.keys(): + # dd = imas2tofu.MultiIDSLoader._didsdiag[ids] + # if dd.get('synth') is not None: + # for v0 in dd['synth']['dsynth'].values(): + # for v1 in v0: + # if '.' in v1: + # v20, v21 = v1.split('.') + # if v20 not in lids: + # lids.append(v20) # ------------------- # If plot and plasma, default dsig, plot_sig, plot_X @@ -1218,7 +1218,7 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, for ss in shot: multi = imas2tofu.MultiIDSLoader(shot=ss, run=run, user=user, tokamak=tokamak, version=version, - ids=lids) + ids=lids, synthdiag=True) # export to instances for ii in range(0,nids): diff --git a/tofucalc.py b/tofucalc.py index 94ec39381..79dc306a9 100755 --- a/tofucalc.py +++ b/tofucalc.py @@ -55,6 +55,7 @@ _T0 = 'IGNITRON' _SHAREX = False _BCK = True +_EXTRA = True ################################################### ################################################### @@ -75,7 +76,7 @@ def _get_exception(q, ids, qtype='quantity'): def call_tfcalcimas(shot=None, run=_RUN, user=_USER, tokamak=_TOKAMAK, version=_VERSION, - ids=None, t0=_T0, + ids=None, t0=_T0, extra=_EXTRA, plot_compare=True, Brightness=None, res=None, interp_t=None, sharex=_SHAREX, indch=None, indch_auto=None, @@ -87,7 +88,7 @@ def call_tfcalcimas(shot=None, run=_RUN, user=_USER, tf.calc_from_imas(shot=shot, run=run, user=user, tokamak=tokamak, version=version, ids=ids, indch=indch, indch_auto=indch_auto, - plot_compare=plot_compare, + plot_compare=plot_compare, extra=extra, Brightness=Brightness, res=res, interp_t=interp_t, t0=t0, plot=True, sharex=sharex, bck=background) @@ -153,6 +154,9 @@ def _str2bool(v): parser.add_argument('-ichauto', '--indch_auto', type=bool, required=False, help='automatically determine indices of channels to be loaded', default=True) + parser.add_argument('-e', '--extra', type=_str2bool, required=False, + help='If True loads separatrix and heating power', + default=_EXTRA) parser.add_argument('-sx', '--sharex', type=_str2bool, required=False, help='Should X axis be shared between diagnostics ids ?', default=_SHAREX, const=True, nargs='?')