Updating devel with master changes

.travis.yml

@@ -68,41 +68,49 @@ after_success:
 - codecov
 - chmod +x $START/anaconda_upload.sh
 - echo $TRAVIS_TAG
+
 before_deploy:
-- echo "BEFORE DEPLOY START........"
-- ls $START
-- cd $START
-- conda config --set anaconda_upload no
-- conda install anaconda-client conda-build
-- conda build conda_recipe
-- export PKG_REAL=$(conda build . --output | tail -1)
-- echo "BEFORE DEPLOY END.........."
+  - >
+    if ! [ "$BEFORE_DEPLOY_RUN" ]; then
+      export BEFORE_DEPLOY_RUN=1;
+      echo "BEFORE DEPLOY START........"
+      ls $START
+      cd $START
+      conda config --set anaconda_upload no
+      conda install anaconda-client conda-build
+      conda build conda_recipe
+      export PKG_REAL=$(conda build . --output | tail -1)
+      echo "BEFORE DEPLOY END.........."
+    fi
 deploy:
   - provider: pypi
     distributions: sdist
     user: __token__
     skip_existing: true
+    skip_cleanup: true
     on:
-      tags: true
       branch: deploy-test
+      tags: true
     server: https://test.pypi.org/legacy/
     password:
       secure: xfVFuoz9YYNChzmT8DC9y+8eH6zdFkfoy3B51uqy8b+vhJNzCzLay4F0uSHvhHy6iYorM6UQKr6soC4D7n3PhmnFOTX/cgLtd/p4gBWGYZF6yXacvw+UHKMshgbAhn2sEynxdSAqdAlNttMI8jsUu9RhbzGiv1l5zSNnFWF4Zsly02G68UnztxIGoz8AYTRW2N2oQhGrl/ryj/YG4mSRKjled6BzK7kNoJUqLGl12DqdMMTEmdJ9NHBXgK3Dv0ya17ReFz3TcxE/4+Yc38NwSR4Ia2EvVSMtyIaccQ1uSrXwW8JQOMn+9CmDWZVUMDD2bzKYbm2WGGM9Fh8WrHnwlWRujoLDofhYEK0Cus11gULFF+J88XucOJlyJNrHP6TWxdSVVoQfwWr2ABqZIvilsvHpF+sjDLqomTNHdi+BbzP2koRv0nJb9K1W24bjPLtSK8+plX7suv7gdBNwlsJ+dPLDM87v4+jGHGthQ6P4X2guTMHZm1PU0PSPB9LCbENCN1uktLLhkgx7gZ42Ag+Jwiu02ENkChLaEB4WpPb9mjLnomu5LDYXFGtPJ/uLMOi3VCXyda0LrzqDhXYT3Cg4hvXySwJcgMYSXalfTxnTm9oouePiEXDbK+XwjMP9mjC5CeMg3SaFFTywqaTH0WUqiOBUJ6H3Gsm0sB15Tj4lNKQ=
   - provider: pypi
     distributions: bdist_wheel
     user: __token__
     skip_existing: true
+    skip_cleanup: true
     on:
+      branch: deploy-test
       condition: $OS = osx-64
       tags: true
-      branch: deploy-test
     server: https://test.pypi.org/legacy/
     password:
       secure: xfVFuoz9YYNChzmT8DC9y+8eH6zdFkfoy3B51uqy8b+vhJNzCzLay4F0uSHvhHy6iYorM6UQKr6soC4D7n3PhmnFOTX/cgLtd/p4gBWGYZF6yXacvw+UHKMshgbAhn2sEynxdSAqdAlNttMI8jsUu9RhbzGiv1l5zSNnFWF4Zsly02G68UnztxIGoz8AYTRW2N2oQhGrl/ryj/YG4mSRKjled6BzK7kNoJUqLGl12DqdMMTEmdJ9NHBXgK3Dv0ya17ReFz3TcxE/4+Yc38NwSR4Ia2EvVSMtyIaccQ1uSrXwW8JQOMn+9CmDWZVUMDD2bzKYbm2WGGM9Fh8WrHnwlWRujoLDofhYEK0Cus11gULFF+J88XucOJlyJNrHP6TWxdSVVoQfwWr2ABqZIvilsvHpF+sjDLqomTNHdi+BbzP2koRv0nJb9K1W24bjPLtSK8+plX7suv7gdBNwlsJ+dPLDM87v4+jGHGthQ6P4X2guTMHZm1PU0PSPB9LCbENCN1uktLLhkgx7gZ42Ag+Jwiu02ENkChLaEB4WpPb9mjLnomu5LDYXFGtPJ/uLMOi3VCXyda0LrzqDhXYT3Cg4hvXySwJcgMYSXalfTxnTm9oouePiEXDbK+XwjMP9mjC5CeMg3SaFFTywqaTH0WUqiOBUJ6H3Gsm0sB15Tj4lNKQ=
   - provider: pypi
     distributions: sdist
     user: "Didou09"
     skip_existing: true
+    skip_cleanup: true
     on:
       tags: true
       branch: master
@@ -112,15 +120,16 @@ deploy:
     distributions: bdist_wheel
     user: "Didou09"
     skip_existing: true
+    skip_cleanup: true
     on:
       condition: $OS = osx-64
       tags: true
       branch: master
     password:
       secure: JNEDTDJVx/2fXNfHntNQ99iDRNuQ4uB3y+DBWVIBycCT95+UCb36YPtKzmruEk/UUS29Xgq4IYCGdfCSWE9smKqG8tV1PcHiw705m+AzcpKy77YtzbVECFBxqY4W36O2pHrkwEUzP/7acjFwNsnUFzArqEzsBJ+KdLaa4OPHJXCh30GA0GyqlrXYbBKG+DA9hX5vtsGo4C6w9noALYF3fS7pKPiI6ipKFnAlzGgHQ7Ke0uQME8N3IAFhmh+Z5xMtIIDWxlnqv+KszdG4DIaGV/W6NIJNAbRhzkqUd+Chu6LoPAd/XkHDTeirR/MBkNUc5UcRJxRnP9rUTRo1gCO/buTYuNRgFkMvqhV5a033+x9edWgtUiKNJIMPLXOxe0RJvc5GWji+Co77HtHxRmGRM2rnYqWMtZeYZlFbUdvHu/8jf0d6I8jyUgAoJYdlMA2u/ipENP3S6by4epE9qycUPXiIVh6r3DZbf3vPTMFvTZYAjBrA0NOzihv1xgcXwemmNUFOQSpe0io4UcFxtS9lLMo+30UMQjCHSnbEVM3zSlZmbMOKpkVOlKlt8Lz5NxwVgWtu9FuW2pGukLtE8AWbqvY9urXAPZCQqZlOIklIjJQIqOITnuw9LEV09cgvPHXfdvNni3ldbMlIQ89zryM6dYvhYryTiEZGK4JDR3wAKJA=
   - provider: script
-    user: "ToFuProject"
     script: $START/anaconda_upload.sh
+    skip_cleanup: true
     on:
       tags: true
       branch: master

anaconda_upload.sh

@@ -6,6 +6,7 @@ set -e
 
 echo "Deploying to anaconda.org..."
 export USER=ToFuProject
+export PKG_DIR=$HOME/miniconda/conda-bld/$OS/
 anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l main $PKG_DIR/tofu-*.tar.bz2
 echo "Successfully uploaded !"
 exit 0

tofu/imas2tofu/_core.py

@@ -1262,6 +1262,18 @@ def get_list_notget_ids(self):
         return lids
 
     def open_get_close(self, ids=None, occ=None, verb=True):
+        """ Force data loading
+
+        If at instanciation or when using method add_ids() you specified option
+        get = False, then the latest added ids may not have been loaded.
+
+        This method forces a refresh and loads all ids contained in the instance
+
+        The name comes from:
+            - open (all the idd)
+            - get (all the ids)
+            - close (all the idd)
+        """
         llids = self._checkformat_get_ids(ids)
         lidd = [lids[0] for lids in llids]
         self._open(idd=lidd)
@@ -3870,14 +3882,19 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None,
                 ipdb.set_trace()
                 raise Exception(msg)
 
-            assert out[dsig[kk]].ndim in [1, 2, 3]
+            if out[dsig[kk]].size == 0 or out[dsig[kk]].ndim not in [1, 2, 3]:
+                msg = ("\nSome data seem to have inconsistent shape:\n"
+                       + "\t- out[{}].shape = {}".format(dsig[kk],
+                                                         out[dsig[kk]].shape))
+                raise Exception(msg)
 
             if out[dsig[kk]].ndim == 1:
                 out[dsig[kk]] = np.atleast_2d(out[dsig[kk]])
 
             if out[dsig[kk]].ndim == 2:
                 if dsig[kk] in ['X','lamb']:
                     if np.allclose(out[dsig[kk]], out[dsig[kk]][:,0:1]):
+                        import pdb; pdb.set_trace() # DB
                         dins[kk] = out[dsig[kk]][:,0]
                     else:
                         dins[kk] = out[dsig[kk]]
@@ -3933,8 +3950,9 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None,
         if t0 != False:
             if 't' in dins.keys():
                 dins['t'] = dins['t'] - t0
-            for tt in dextra.keys():
-                dextra[tt]['t'] = dextra[tt]['t'] - t0
+            if dextra is not None:
+                for tt in dextra.keys():
+                    dextra[tt]['t'] = dextra[tt]['t'] - t0
 
         # --------------
         # Create objects

tofu/utils.py

@@ -651,7 +651,7 @@ def load_from_imas(shot=None, run=None, user=None, tokamak=None, version=None,
                    occ=None, indch=None, description_2d=None, equilibrium=None,
                    dsig=None, data=None, X=None, t0=None, dextra=None,
                    plot=True, plot_sig=None, plot_X=None,
-                   sharex=False, invertx=None,
+                   sharex=False, invertx=None, extra=True,
                    bck=True, indch_auto=True, t=None, init=None, dR_sep=None):
     # -------------------
     # import imas2tofu
@@ -922,7 +922,7 @@ def load_from_imas(shot=None, run=None, user=None, tokamak=None, version=None,
     if nDat > 0 or nCam > 0 or nPla > 0:
         if 'wall' not in lids:
             lids.append('wall')
-        if nDat > 0 or nPla > 0 and dextra is None:
+        if (nDat > 0 or nPla > 0) and extra is True:
             if 'equilibrium' not in lids:
                 lids.append('equilibrium')
             if 'lh_antennas' not in lids:
@@ -1039,7 +1039,7 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None,
                    ids=None, Name=None, out=None, tlim=None, config=None,
                    occ=None, indch=None, description_2d=None, equilibrium=None,
                    dsig=None, data=None, X=None, t0=None, dextra=None,
-                   Brightness=None, res=None, interp_t=None,
+                   Brightness=None, res=None, interp_t=None, extra=True,
                    plot=True, plot_compare=True, sharex=False,
                    bck=True, indch_auto=True, t=None, init=None):
     # -------------------
@@ -1161,7 +1161,7 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None,
     if nDat > 0 or nCam > 0 or nPla > 0:
         if 'wall' not in lids:
             lids.append('wall')
-        if nDat > 0 or nPla > 0 and dextra is None:
+        if (nDat > 0 or nPla > 0) and extra is True:
             if 'equilibrium' not in lids:
                 lids.append('equilibrium')
             if 'lh_antennas' not in lids:
@@ -1173,16 +1173,16 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None,
                 lids.append('pulse_schedule')
 
     # Complement ids in diag-specific way
-    for ids in lids:
-        if ids in imas2tofu.MultiIDSLoader._didsdiag.keys():
-            dd = imas2tofu.MultiIDSLoader._didsdiag[ids]
-            if dd.get('synth') is not None:
-                for v0 in dd['synth']['dsynth'].values():
-                    for v1 in v0:
-                        if '.' in v1:
-                            v20, v21 = v1.split('.')
-                            if v20 not in lids:
-                                lids.append(v20)
+    # for ids in lids:
+        # if ids in imas2tofu.MultiIDSLoader._didsdiag.keys():
+            # dd = imas2tofu.MultiIDSLoader._didsdiag[ids]
+            # if dd.get('synth') is not None:
+                # for v0 in dd['synth']['dsynth'].values():
+                    # for v1 in v0:
+                        # if '.' in v1:
+                            # v20, v21 = v1.split('.')
+                            # if v20 not in lids:
+                                # lids.append(v20)
 
     # -------------------
     # If plot and plasma, default dsig, plot_sig, plot_X
@@ -1218,7 +1218,7 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None,
     for ss in shot:
         multi = imas2tofu.MultiIDSLoader(shot=ss, run=run, user=user,
                                          tokamak=tokamak, version=version,
-                                         ids=lids)
+                                         ids=lids, synthdiag=True)
 
         # export to instances
         for ii in range(0,nids):

tofu/version.py

@@ -1,2 +1,2 @@
 # Do not edit, pipeline versioning governed by git tags!
-__version__ = '1.4.2b13-86-g0840bdc4'
+__version__ = '1.4.2'

tofucalc.py

@@ -55,6 +55,7 @@
 _T0 = 'IGNITRON'
 _SHAREX = False
 _BCK = True
+_EXTRA = True
 
 ###################################################
 ###################################################
@@ -75,7 +76,7 @@ def _get_exception(q, ids, qtype='quantity'):
 
 def call_tfcalcimas(shot=None, run=_RUN, user=_USER,
                     tokamak=_TOKAMAK, version=_VERSION,
-                    ids=None, t0=_T0,
+                    ids=None, t0=_T0, extra=_EXTRA,
                     plot_compare=True, Brightness=None,
                     res=None, interp_t=None,
                     sharex=_SHAREX, indch=None, indch_auto=None,
@@ -87,7 +88,7 @@ def call_tfcalcimas(shot=None, run=_RUN, user=_USER,
     tf.calc_from_imas(shot=shot, run=run, user=user,
                       tokamak=tokamak, version=version,
                       ids=ids, indch=indch, indch_auto=indch_auto,
-                      plot_compare=plot_compare,
+                      plot_compare=plot_compare, extra=extra,
                       Brightness=Brightness, res=res, interp_t=interp_t,
                       t0=t0, plot=True, sharex=sharex, bck=background)
 
@@ -153,6 +154,9 @@ def _str2bool(v):
     parser.add_argument('-ichauto', '--indch_auto', type=bool, required=False,
                         help='automatically determine indices of channels to be loaded',
                         default=True)
+    parser.add_argument('-e', '--extra', type=_str2bool, required=False,
+                        help='If True loads separatrix and heating power',
+                        default=_EXTRA)
     parser.add_argument('-sx', '--sharex', type=_str2bool, required=False,
                         help='Should X axis be shared between diagnostics ids ?',
                         default=_SHAREX, const=True, nargs='?')

tofuplot.py

@@ -53,6 +53,7 @@
 _T0 = 'IGNITRON'
 _SHAREX = False
 _BCK = True
+_EXTRA = True
 
 ###################################################
 ###################################################
@@ -72,7 +73,7 @@ def _get_exception(q, ids, qtype='quantity'):
 
 
 def call_tfloadimas(shot=None, run=_RUN, user=_USER,
-                    tokamak=_TOKAMAK, version=_VERSION,
+                    tokamak=_TOKAMAK, version=_VERSION, extra=_EXTRA,
                     ids=None, quantity=None, X=None, t0=_T0,
                     sharex=_SHAREX, indch=None, indch_auto=None,
                     background=_BCK, t=None, dR_sep=None, init=None):
@@ -84,7 +85,7 @@ def call_tfloadimas(shot=None, run=_RUN, user=_USER,
     tf.load_from_imas(shot=shot, run=run, user=user,
                       tokamak=tokamak, version=version,
                       ids=ids, indch=indch, indch_auto=indch_auto,
-                      plot_sig=quantity, plot_X=X,
+                      plot_sig=quantity, plot_X=X, extra=extra,
                       t0=t0, plot=True, sharex=sharex, bck=background,
                       t=t, dR_sep=dR_sep, init=init)
 
@@ -156,9 +157,12 @@ def _str2bool(v):
     parser.add_argument('-ich', '--indch', type=int, required=False,
                         help='indices of channels to be loaded',
                         nargs='+', default=None)
-    parser.add_argument('-ichauto', '--indch_auto', type=bool, required=False,
+    parser.add_argument('-ichauto', '--indch_auto', type=_str2bool, required=False,
                         help='automatically determine indices of'
                         + ' channels to be loaded', default=True)
+    parser.add_argument('-e', '--extra', type=_str2bool, required=False,
+                        help='If True loads separatrix and heating power',
+                        default=_EXTRA)
     parser.add_argument('-sx', '--sharex', type=_str2bool, required=False,
                         help='Should X axis be shared between diagnostics ids ?',
                         default=_SHAREX, const=True, nargs='?')