diff --git a/.travis.yml b/.travis.yml index 5cc833b73..83bf997a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ language: python -matrix: +jobs: include: - name: Bionic python 3.7 os: linux @@ -61,9 +61,16 @@ install: - export VERSION=$(echo $REV | tr - .) - echo $REV - pip install -e ".[dev]" +- export IS_MASTER=$(git ls-remote origin | grep "$TRAVIS_COMMIT\s\+refs/heads/master$" | grep -o "master") +- export IS_DEPLOY=$(git ls-remote origin | grep "$TRAVIS_COMMIT\s\+refs/heads/deploy-test$" | grep -o "deploy-test") +- echo $TRAVIS_COMMIT +- echo $IS_MASTER +- echo $IS_DEPLOY + script: - cd $HOME - nosetests tofu.tests --nocapture -v --with-id --with-timer --with-coverage --cover-package=tofu +- tofu-custom after_success: - codecov - chmod +x $START/anaconda_upload.sh @@ -76,55 +83,51 @@ before_deploy: echo "BEFORE DEPLOY START........" ls $START cd $START - conda config --set anaconda_upload no - conda install anaconda-client conda-build - conda build conda_recipe - export PKG_REAL=$(conda build . --output | tail -1) echo "BEFORE DEPLOY END.........." fi deploy: - provider: pypi distributions: sdist - user: __token__ + username: __token__ skip_existing: true skip_cleanup: true on: - branch: deploy-test tags: true + condition: $IS_DEPLOY = "deploy-test" server: https://test.pypi.org/legacy/ password: secure: xfVFuoz9YYNChzmT8DC9y+8eH6zdFkfoy3B51uqy8b+vhJNzCzLay4F0uSHvhHy6iYorM6UQKr6soC4D7n3PhmnFOTX/cgLtd/p4gBWGYZF6yXacvw+UHKMshgbAhn2sEynxdSAqdAlNttMI8jsUu9RhbzGiv1l5zSNnFWF4Zsly02G68UnztxIGoz8AYTRW2N2oQhGrl/ryj/YG4mSRKjled6BzK7kNoJUqLGl12DqdMMTEmdJ9NHBXgK3Dv0ya17ReFz3TcxE/4+Yc38NwSR4Ia2EvVSMtyIaccQ1uSrXwW8JQOMn+9CmDWZVUMDD2bzKYbm2WGGM9Fh8WrHnwlWRujoLDofhYEK0Cus11gULFF+J88XucOJlyJNrHP6TWxdSVVoQfwWr2ABqZIvilsvHpF+sjDLqomTNHdi+BbzP2koRv0nJb9K1W24bjPLtSK8+plX7suv7gdBNwlsJ+dPLDM87v4+jGHGthQ6P4X2guTMHZm1PU0PSPB9LCbENCN1uktLLhkgx7gZ42Ag+Jwiu02ENkChLaEB4WpPb9mjLnomu5LDYXFGtPJ/uLMOi3VCXyda0LrzqDhXYT3Cg4hvXySwJcgMYSXalfTxnTm9oouePiEXDbK+XwjMP9mjC5CeMg3SaFFTywqaTH0WUqiOBUJ6H3Gsm0sB15Tj4lNKQ= - provider: pypi distributions: bdist_wheel - user: __token__ + username: __token__ skip_existing: true skip_cleanup: true on: - branch: deploy-test - condition: $OS = osx-64 tags: true + condition: $IS_DEPLOY = "deploy-test" + condition: $OS = osx-64 server: https://test.pypi.org/legacy/ password: secure: xfVFuoz9YYNChzmT8DC9y+8eH6zdFkfoy3B51uqy8b+vhJNzCzLay4F0uSHvhHy6iYorM6UQKr6soC4D7n3PhmnFOTX/cgLtd/p4gBWGYZF6yXacvw+UHKMshgbAhn2sEynxdSAqdAlNttMI8jsUu9RhbzGiv1l5zSNnFWF4Zsly02G68UnztxIGoz8AYTRW2N2oQhGrl/ryj/YG4mSRKjled6BzK7kNoJUqLGl12DqdMMTEmdJ9NHBXgK3Dv0ya17ReFz3TcxE/4+Yc38NwSR4Ia2EvVSMtyIaccQ1uSrXwW8JQOMn+9CmDWZVUMDD2bzKYbm2WGGM9Fh8WrHnwlWRujoLDofhYEK0Cus11gULFF+J88XucOJlyJNrHP6TWxdSVVoQfwWr2ABqZIvilsvHpF+sjDLqomTNHdi+BbzP2koRv0nJb9K1W24bjPLtSK8+plX7suv7gdBNwlsJ+dPLDM87v4+jGHGthQ6P4X2guTMHZm1PU0PSPB9LCbENCN1uktLLhkgx7gZ42Ag+Jwiu02ENkChLaEB4WpPb9mjLnomu5LDYXFGtPJ/uLMOi3VCXyda0LrzqDhXYT3Cg4hvXySwJcgMYSXalfTxnTm9oouePiEXDbK+XwjMP9mjC5CeMg3SaFFTywqaTH0WUqiOBUJ6H3Gsm0sB15Tj4lNKQ= - provider: pypi distributions: sdist - user: "Didou09" + username: "Didou09" skip_existing: true skip_cleanup: true on: tags: true - branch: master + condition: $IS_MASTER = "master" password: secure: JNEDTDJVx/2fXNfHntNQ99iDRNuQ4uB3y+DBWVIBycCT95+UCb36YPtKzmruEk/UUS29Xgq4IYCGdfCSWE9smKqG8tV1PcHiw705m+AzcpKy77YtzbVECFBxqY4W36O2pHrkwEUzP/7acjFwNsnUFzArqEzsBJ+KdLaa4OPHJXCh30GA0GyqlrXYbBKG+DA9hX5vtsGo4C6w9noALYF3fS7pKPiI6ipKFnAlzGgHQ7Ke0uQME8N3IAFhmh+Z5xMtIIDWxlnqv+KszdG4DIaGV/W6NIJNAbRhzkqUd+Chu6LoPAd/XkHDTeirR/MBkNUc5UcRJxRnP9rUTRo1gCO/buTYuNRgFkMvqhV5a033+x9edWgtUiKNJIMPLXOxe0RJvc5GWji+Co77HtHxRmGRM2rnYqWMtZeYZlFbUdvHu/8jf0d6I8jyUgAoJYdlMA2u/ipENP3S6by4epE9qycUPXiIVh6r3DZbf3vPTMFvTZYAjBrA0NOzihv1xgcXwemmNUFOQSpe0io4UcFxtS9lLMo+30UMQjCHSnbEVM3zSlZmbMOKpkVOlKlt8Lz5NxwVgWtu9FuW2pGukLtE8AWbqvY9urXAPZCQqZlOIklIjJQIqOITnuw9LEV09cgvPHXfdvNni3ldbMlIQ89zryM6dYvhYryTiEZGK4JDR3wAKJA= - provider: pypi distributions: bdist_wheel - user: "Didou09" + username: "Didou09" skip_existing: true skip_cleanup: true on: condition: $OS = osx-64 tags: true - branch: master + condition: $IS_MASTER = "master" password: secure: JNEDTDJVx/2fXNfHntNQ99iDRNuQ4uB3y+DBWVIBycCT95+UCb36YPtKzmruEk/UUS29Xgq4IYCGdfCSWE9smKqG8tV1PcHiw705m+AzcpKy77YtzbVECFBxqY4W36O2pHrkwEUzP/7acjFwNsnUFzArqEzsBJ+KdLaa4OPHJXCh30GA0GyqlrXYbBKG+DA9hX5vtsGo4C6w9noALYF3fS7pKPiI6ipKFnAlzGgHQ7Ke0uQME8N3IAFhmh+Z5xMtIIDWxlnqv+KszdG4DIaGV/W6NIJNAbRhzkqUd+Chu6LoPAd/XkHDTeirR/MBkNUc5UcRJxRnP9rUTRo1gCO/buTYuNRgFkMvqhV5a033+x9edWgtUiKNJIMPLXOxe0RJvc5GWji+Co77HtHxRmGRM2rnYqWMtZeYZlFbUdvHu/8jf0d6I8jyUgAoJYdlMA2u/ipENP3S6by4epE9qycUPXiIVh6r3DZbf3vPTMFvTZYAjBrA0NOzihv1xgcXwemmNUFOQSpe0io4UcFxtS9lLMo+30UMQjCHSnbEVM3zSlZmbMOKpkVOlKlt8Lz5NxwVgWtu9FuW2pGukLtE8AWbqvY9urXAPZCQqZlOIklIjJQIqOITnuw9LEV09cgvPHXfdvNni3ldbMlIQ89zryM6dYvhYryTiEZGK4JDR3wAKJA= - provider: script @@ -132,5 +135,4 @@ deploy: skip_cleanup: true on: tags: true - branch: master - skip_cleanup: true + condition: $IS_MASTER = "master" diff --git a/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.bib b/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.bib new file mode 100644 index 000000000..b98edc6a2 --- /dev/null +++ b/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.bib @@ -0,0 +1,10 @@ +@article{didier2016, + title={Non-monotonic growth rates of sawtooth precursors evidenced with a new method on ASDEX Upgrade}, + author={Vezinet, D and Igochine, V and Weiland, M and Yu, Q and Gude, A and Meshcheriakov, D and Sertoli, M and EUROfusion MST1 Team and others}, + journal={Nuclear Fusion}, + volume={56}, + number={8}, + pages={086001}, + year={2016}, + publisher={IOP Publishing} +} \ No newline at end of file diff --git a/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.pdf b/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.pdf new file mode 100644 index 000000000..3ce376732 Binary files /dev/null and b/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.pdf differ diff --git a/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.tex b/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.tex new file mode 100644 index 000000000..0471be66e --- /dev/null +++ b/Notes_Upgrades/Communications/SeminaireStras20/2019_LM_SeminaireStras.tex @@ -0,0 +1,1182 @@ +\documentclass[10pt]{beamer} + +\usepackage{appendixnumberbeamer} +% +%\usepackage{lmodern} +%\usepackage{wrapfig} +\usepackage{tikz} +%\usetikzlibrary{positioning} +\usetikzlibrary{arrows,shapes} +\usepackage{amsmath,esint,esvect} % arrow for vector, maths and more +%\usepackage{datetime} +\usepackage{gensymb}%for degree symbol +%\usepackage{beamerthemesplit} +%\usepackage{graphicx} %trimed images +%\usepackage{pifont} %for dings +% +%\usepackage{import} +%\usepackage{cases} +\usepackage{booktabs} %beautiful tables +\usepackage{pgfplots} +\usepgfplotslibrary{dateplot} +% +\usepackage{siunitx} % SI notations +\sisetup{table-number-alignment=center, exponent-product=\cdot} + + + + +% Bibliography includes +\usepackage[backend=bibtex, citestyle=verbose-trad2,bibstyle=authortitle-icomp,sortcites=true,block=space,firstinits=true]{biblatex} +\addbibresource{2019_LM_SeminaireStras.bib} +\usepackage{hyperref} + +\usetheme[progressbar=frametitle]{metropolis} + + + + \setbeamertemplate{itemize subitem}{\color{orange}$\blacktriangleright$} + +% --------------Math mode------------------ +\renewcommand\mathfamilydefault{\rmdefault} +% ----------------------------------------- + +% ------ Images path ------------------ +\newcommand{\figurespath}{./figures} + +%% ---------- Editing the footer ------------ +%\makeatother +%\setbeamertemplate{footline} +%{ +% \leavevmode% +% \hbox{% +% \begin{beamercolorbox}[wd=.6\paperwidth,ht=2.25ex,dp=1ex,center]{institute in head/foot}% +% \usebeamerfont{institute in head/foot}\insertshortinstitute +% \end{beamercolorbox}% +% \begin{beamercolorbox}[wd=.4\paperwidth,ht=2.25ex,dp=1ex,center]{date in head/foot}% +% \usebeamerfont{date in head/foot}\insertshortdate\hspace*{2em} +% \insertframenumber{} \hspace*{2ex} +% \end{beamercolorbox}}% +% \vskip0pt% +%} +%\makeatletter +%\setbeamertemplate{navigation symbols}{} +%% ----------------------------------------- + +% ------ Date definition ------------------ +% \newdate{date}{04}{09}{2019} +% ----------------------------------------- + +% ----------New color---------------------- +\definecolor{myblue}{RGB}{51,51,179} +\definecolor{beaubleu2}{cmyk}{1,0,0,0.71} +\definecolor{SchoolColor}{rgb}{0.6471, 0.1098, 0.1882} % Crimson +\definecolor{beaurouge}{HTML}{405684} +\definecolor{beaubleu}{HTML}{CC382C} +\definecolor{myyellow}{rgb}{0.99, 0.82, 0.35} +% ----------------------------------------- + +%% ---------Redifining foot cites----------- +\newrobustcmd*{\footlessfullcite}{\AtNextCite{\renewbibmacro{title}{}\renewbibmacro{in:}{}}\footfullcite} + +%\renewbibmacro{in:}{\hspace{-5pt}} +\AtEveryCitekey{\clearfield{pages}\clearfield{volume}\clearfield{url} +\clearfield{doi}\clearfield{issn}} +%% ----------------------------------------- + +% ---------First page variables------------ +\title{ToFu} +\subtitle{An open-source python/cython library for synthetic tomography diagnostics on tokamaks} +\author[Laura S. Mendoza] % (optional, for multiple authors) +{\textbf{Laura~S.~Mendoza}\inst{1}, \and Didier~Vezinet\inst{2}} +\institute[] % (optional) +{ + \inst{1}% + INRIA Grand-Est, + TONUS Team, Strasbourg, France\\ + + \inst{2}% + CEA, + Cadarache, France +} +\date[\displaydate{date}] % (optional) +{\alert{Seminaire, Strasbourg, France}} +\subject{} + + +% ----------------------------------------- + +%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{document} + +\newcommand{\gradx}{\nablax} +\newcommand{\vpar}{v_\parallel} +\newcommand{\xvec}{\mathbf{x}} +\newcommand{\nablax}{\nabla_{\!\xvec}} + + +% ================================= +%\usebackgroundtemplate{\includegraphics[height=\paperheight,width=\paperwidth]{figures/background.png}}% +\begin{frame} + \titlepage +\end{frame} +% ================================= + +% ================================= +\begin{frame}{Table of contents} + \setbeamertemplate{section in toc}[sections numbered] + \tableofcontents[hideallsubsections] +\end{frame} +% ================================= + +\section{Context} + +% ================================= +\begin{frame} +\frametitle{Context: energy needs vs resources and climate change} + + \begin{center} + \includegraphics[height=1.8cm]{figures/oil-power.jpg}% + \includegraphics[height=1.8cm]{figures/solar-power.jpg}% + \includegraphics[height=1.8cm]{figures/nuclear-power.jpg}% + \includegraphics[height=1.8cm]{figures/wind-power.jpg} + \end{center} + +Worldwide growing energy needs (population, standards of living...)\\ +$\quad \Rightarrow$ Production of $CO_2$, limited resources, radioactive waste, not efficient enough, harmful to surrounding environment\\ + +$\quad \Rightarrow$ need to decrease consumption + alternative production means\\ + +$\quad \Rightarrow$ Need for cleaner, more reliable, more powerful energy source\\ + +%\begin{itemize}%[leftmargin=1em] +%\setbeamertemplate{itemize item}{$\Rightarrow$} +% \item \alert{Fusion}: cleaner, more reliable, more powerful energy source? +%\end{itemize} + +\end{frame} +% ================================= + + + +% ================================= +\begin{frame} +\frametitle{Context: Controlled fusion and magnetic confinement} + +\begin{columns} + \begin{column}{0.65\textwidth} + \begin{center} + \textbf{D-T Fusion reaction\;\;\;}\vspace{0.3cm} + %\includegraphics[width=3.5cm]{figures/DT_reaction3.png} + \resizebox{0.5\textwidth}{!}{\input{figures/reaction.tex}} + \end{center} + \vspace{-0.5cm} + \begin{itemize}%[leftmargin=1em] + \item Gas $>$ 100 Million\degree K composed of positive ions and negative electrons: plasma + \item Confinement using electromagnetic fields + \item break-even not obtained yet + \end{itemize} + + \end{column} + + \begin{column}{0.35\textwidth} + \includegraphics[width=1.\textwidth]{figures/ITER.pdf} + \end{column} + +\end{columns} + +%\begin{itemize}%[leftmargin=1em] +\begin{center} +In a nutshell: toroidal vacuum vessel, filled with H plasma +\end{center}%\end{itemize} +\vspace{-0.2cm} +%%\begin{itemize}%[leftmargin=1em] +%%\setbeamertemplate{itemize item}{$\Rightarrow$} +%% \item \alert{Fusion codes:} complexity due to the number of parameters, geometry, model, etc. +%%\end{itemize} +\end{frame} +% ================================= + + +% ================================= +\begin{frame} +\frametitle{Tokamak diagnostics to measure plasma quantities} + \begin{center} + \includegraphics[width=0.7\textwidth]{figures/ITER.pdf} + \end{center} +\end{frame} +% ================================= + +\section{Tomography diagnostics} + + +% ================================= +\begin{frame} +\frametitle{Tokamak diagnostics to measure plasma quantities} + +\metroset{block=fill} +\begin{alertblock}{Diagnostics} +Set of instruments to measure plasma quantities, for understanding, control, optimization. +\end{alertblock} + +e.g: magnetic field, neutrons, \textbf{emitted light}, temperature, density...\\[1cm] + + +$\quad \Rightarrow$ cameras (1D or 2D) for measuring light in various wavelengths + + +% +%\begin{itemize} +%\item \textbf{Magnetic} diagnostics: currents, plasma stored energy, plasma shape and position; +% +%\item \textbf{Neutron} diagnostics (ie. cameras, spectrometers, etc.): fusion power; +% +%\item Optical systems (\textbf{interferometers}): temperature and density profiles; +% +%\item Bolometric systems (\textbf{tomography}): spatial distribution of radiated power; +% +%\item \textbf{Spectroscopic}: X-ray wavelength range, impurity species and density, input particle flux, ion temperature, helium density, fueling ratio, plasma rotation, and current density. + +%\item Microwave diagnostics probe the main plasma and the plasma in the divertor region in order to measure plasma position. + +%\end{itemize} + +\end{frame} +% ================================= + + +%% ================================= +%\begin{frame} +%\frametitle{A tokamak as a poloidal + horizontal projections} +% +%\begin{columns} +% +% \begin{column}{0.25\textwidth} +% \includegraphics[width=1.\textwidth]{figures/ITER.pdf} +% \end{column} +% +% \begin{column}{0.75\textwidth} +% \begin{center} +% \textbf{tofu.geom.Config class} +% \end{center} +% \includegraphics[width=1.\textwidth]{figures/Config01.pdf} +% \end{column} +% +%\end{columns} +% +% +%\end{frame} +%% ================================= + + +% ================================= +\begin{frame} +\frametitle{Tomography diagnostics - numerical context} + \vspace{-0.75cm} + + $$M_i(t) = \iiint\limits_{V_i} \vv{\varepsilon(x,t)}\cdot\vv{n} \,\Omega_i \;{d}V$$ + \vspace{-0.75cm} +\begin{columns} + \begin{column}{0.325\textwidth} + + %\def\svgwidth{\linewidth} + %\import{figures/}{detectors.pdf_tex} + \includegraphics[width=\linewidth,trim={0 0 1cm 0}]{figures/Tomography.pdf} + + \end{column} + + \begin{column}{0.675\textwidth} + \begin{center} + + %DIRECT PROBLEM + \begin{block}{} + \begin{itemize} + \item \textcolor{myblue}{\textbf{Direct problem} (synthetic diagnostic):\\ + } + \end{itemize} + \end{block} + %\textbf{Direct problem} (synthetic diagnostic):\\ +Simulated emissivity $\longrightarrow$ integrated measurements\\ + + %INVERSE PROBLEM + \begin{block}{} + \begin{itemize} + \item \textcolor{myblue}{\textbf{Inverse problem} (tomography):\\ + } + \end{itemize} + \end{block} + %\textbf{Direct problem} (synthetic diagnostic):\\ +Integrated measurements $\longrightarrow$ Reconstructed emissivity \\ + + \end{center} + + \end{column} +% +\end{columns} + + +\end{frame} +% ================================= + + + + +% ================================= +\begin{frame} +\frametitle{Tomography diagnostics - numerical context} + + $$M_i(t) = \iiint\limits_{V_i} \vv{\varepsilon(x,t)}\cdot\vv{n} \,\Omega_i \;{d}V$$ + \vspace{-1cm} +\begin{columns} + \begin{column}{0.325\textwidth} + + %\def\svgwidth{\linewidth} + %\import{figures/}{detectors.pdf_tex} + \includegraphics[width=\linewidth,trim={0 0 1cm 0}]{figures/Tomography.pdf} + + \end{column} + + \begin{column}{0.675\textwidth} + \begin{center} + + %DIRECT PROBLEM + \begin{block}{} + \begin{itemize} + \item \textcolor{myblue}{\textbf{Direct problem} (synthetic diagnostic):\\ + } + \end{itemize} + \end{block} + %\textbf{Direct problem} (synthetic diagnostic):\\ +Simulated emissivity $\longrightarrow$ measurements\\ + \alert{Spatial integration} + \vspace{-0.5cm} + + %INVERSE PROBLEM + \begin{block}{} + \begin{itemize} + \item \textcolor{myblue}{\textbf{Inverse problem} (tomography):\\ + } + \end{itemize} + \end{block} + %\textbf{Direct problem} (synthetic diagnostic):\\ +Integrated measurements $\longrightarrow$ Reconstructed emissivity \\ + \alert{Mesh and basis functions construction, spatial integration, data filtering, inversion routines, etc.} + + \end{center} + + \end{column} +% +\end{columns} + +\pause +\vspace{-0.5cm} +\begin{block}{} +\begin{center} +Tomography is \textbf{ill-posed}, very sensitive to errors, noise and bias\\ +$\longrightarrow$ \alert{Reputation for low reproducibility / reliability } +\end{center} + \end{block} +\end{frame} +% ================================= + + + +\section{The ToFu package} + + +% ================================= +\begin{frame} +\frametitle{Motivation: ``current" state} + + In the fusion community, codes for tomography diagnostic are often: + \begin{itemize} + \item developed by physicists (with little programming experience) + \item in Matlab (or IDL) + \item written from scratch, re-done by new students + \item not distributed (few users), rarely documented + \end{itemize} + +... which means +\begin{itemize} + \item waste of resources: time, man-power + \item low traceability, reproducibility + \item low standardization, unclear assumptions / methods + \end{itemize} + +\end{frame} +% ================================= + + +% ================================= +\begin{frame} +\frametitle{A code for Tomography for Fusion} +\textbf{Develop a common tool:} +\begin{columns} +\begin{column}{0.65\textwidth} + \begin{itemize} + \item Generic (geometry independent) + \item Portable (Python) + \item Optimized / parallelized + \item Documented online + \item Continuous integration + \end{itemize} +\end{column} +\begin{column}{0.35\textwidth} +\vspace{-1cm} +\begin{center} + \includegraphics[width=0.8\linewidth]{figures/ci.pdf} +\end{center} +\end{column} +\end{columns} +\vspace{-0.9cm} + +\begin{block}{} + \begin{center} + \alert{\textbf{ToFu}\footnote{repository: \url{https://github.com/ToFuProject/tofu}}\footnote{documentation: \url{https://tofuproject.github.io/tofu/index.html} }\footcite{didier2016}} = \alert{\textbf{To}mography for \textbf{Fu}sion} + \end{center} +\vspace{-5mm} +\end{block} + +\end{frame} +% ================================= + +% ================================= +\begin{frame} +\frametitle{More about Tofu} + +\begin{columns} +\begin{column}{0.65\textwidth} +\begin{itemize} + \item Created in 2014 + \item Open Source:\textbf{ MIT license} + \item \textbf{Python} 3 + \textbf{Cython} + \item Continuous integration: \textbf{Travis} CI + \item \textbf{conda}, \textbf{pip} + \item Two (main) developers: + \begin{itemize} + \item Didier Vezinet (creator, Physics) + \item Laura S. Mendoza (since 06.2018, Applied Maths) + \end{itemize} + \item Contributors: + \begin{itemize} + \item Jorge Morales + \item Florian Le Bourdais + \item Arpan Khandelwal + \item Koyo Munechika + \end{itemize} +\end{itemize} +\end{column} +\begin{column}{0.35\textwidth} +\begin{center} + \includegraphics[width=\textwidth]{figures/qr-code.png} +\end{center} +\end{column} +\end{columns} +\begin{center} + \includegraphics[width=\textwidth]{figures/badges.png} +\end{center} + +\end{frame} +% ================================= + +% ================================= +\begin{frame} +\frametitle{Tofu's structure} + +\begin{center} + \includegraphics[width=0.8\linewidth]{figures/tofu.pdf} +\end{center} + +\end{frame} +% ================================= + + +% ================================= +\begin{frame} +\frametitle{Tofu's geometry representation} + +\begin{center} + \includegraphics[width=0.6\linewidth]{figures/ITER.pdf} +\end{center} + +\end{frame} +% ================================= + +% ================================= +\begin{frame} +\frametitle{Tofu's geometry representation} + +\begin{center} + \includegraphics[width=\linewidth]{figures/newiter.png} +\end{center} + +\end{frame} +% ================================= + +% ================================= +\section{Demo} + +% ================================= +{\setbeamercolor{palette primary}{fg=black, bg=yellow} +\begin{frame}[standout] + Demo +\end{frame} +} +% ================================= + +%% ================================= +%\begin{frame} +%\frametitle{What ToFu can do: modeling of simplified geometry} +% \begin{center} +% \includegraphics[width=\textwidth,trim={0 0 4cm 0}]{figures/geom_B3.png} +% \end{center} +%\end{frame} +%% ================================= +% +%% ================================= +%\begin{frame} +%\frametitle{What ToFu can do: 3D modeling of a 1D camera} +% \begin{center} +% \includegraphics[width=\textwidth]{figures/cam1d.png} +% \end{center} +%\end{frame} +%% ================================= +% +%% ================================= +%\begin{frame} +%\frametitle{What ToFu can do: 3D modeling of a 2D camera} +% \begin{center} +% \includegraphics[width=\textwidth]{figures/cam2d.png} +% \end{center} +%\end{frame} +%% ================================= + + +% ================================= +\begin{frame} +\frametitle{tofu.geom: modeling of simplified geometry} +\begin{columns} +% \begin{column}{0.33\textwidth} +% \begin{center} +% \textbf{Geometry configuration} +% \includegraphics[width=\textwidth,trim={0 0 6cm 0}]{figures/geom_B3.png} +% \end{center} +% \pause +% \end{column} + \begin{column}{0.5\textwidth} + \begin{center} + \textbf{1D Camera\\(tofu.geom.CamLOS1D)} + \includegraphics[width=\textwidth]{figures/cam1d_bis.pdf} + \end{center} + \end{column} + \pause + \begin{column}{0.5\textwidth} + \begin{center} + \textbf{2D Camera\\(tofu.geom.CamLOS2D)} + \includegraphics[width=\textwidth]{figures/cam2d_bis.pdf} + \end{center} + \end{column} + \end{columns} +\end{frame} +% ================================= + +% ================================= +\begin{frame} +\frametitle{tofu.geom: handle basic reflexions} +\begin{columns} +% \begin{column}{0.33\textwidth} +% \begin{center} +% \textbf{Geometry configuration} +% \includegraphics[width=\textwidth,trim={0 0 6cm 0}]{figures/geom_B3.png} +% \end{center} +% \pause +% \end{column} + \begin{column}{0.5\textwidth} + \begin{center} + \textbf{1D Camera with reflexions\\(tofu.geom.CamLOS1D)} + \includegraphics[width=\textwidth]{figures/cam1d_bis_ref.pdf} + \end{center} + \end{column} + \begin{column}{0.5\textwidth} + \begin{center} + \textbf{2D Camera with reflexions\\(tofu.geom.CamLOS2D)} + \includegraphics[width=\textwidth]{figures/cam2d_bis_ref.pdf} + \end{center} + \end{column} + \end{columns} +\end{frame} +% ================================= + +%% ================================= +%\begin{frame} +%\frametitle{What ToFu can do: computing synthetic signals} +% +%\end{frame} +% ================================= + +% ================================= +\begin{frame} +\frametitle{What ToFu can do} +\begin{itemize} + \item Model simplified 3D geometry + \item 3D modeling of a 1D and 2D LOS camera + \item Handle basic reflections + \item Computing synthetic signals + \item Native support for IMAS interfacing + \item Data easy interactive visualization and basic treatment + \pause + \item ...and soon (being re-written / developed): + \begin{itemize} + \item finite beam width (VOS, in 1.4.3, mid 2020) + \item meshing and basis functions (mid 2020) + \item tomographic inversion (late 2020 - 2021) + \item dust particle trajectory tracking (new, Arpan) + \item faster Matplotlib + PyQtGraph visualization + \item magnetic field line tracing (new) + \item statistical data analysis (pandas) integrated + \end{itemize} +\end{itemize} +\end{frame} + + +\section{Code Optimization} + + +% ================================= +\begin{frame} +\frametitle{Geometry reconstruction: ray-tracing techniques} + + To reconstruct emissivity we need to take account: + \begin{itemize} + \item Up to hundreds of structural elements in vessel + \item Scale of the vessel: + $10^4$ bigger than smaller structural detail + \setbeamertemplate{itemize item}{$\Rightarrow$} + + \item Geometry defined with minimal data polygon $(R,Z)$\\ extruded along $\varphi$ + \item Symmetry of vessel along $\varphi$ + + \end{itemize} + + \vspace{0.5cm} + \begin{center} + \includegraphics[height=2.3cm]{figures/confB3_wp_view2.png}% + ~ + \includegraphics[height=2.3cm]{figures/confB3_wp_view1.png}% + ~ + \includegraphics[height=2.3cm]{figures/confB3_wp_view3.png}% + \end{center} + +\end{frame} +% ================================= + + + +% ================================= +\begin{frame} +\frametitle{Optimization of ray-tracing algorithm} + +\begin{columns} +\begin{column}{0.65\textwidth} + \begin{itemize} + \item Description of geometry: + \begin{itemize} + \item Vessel and structures: set of 2D polygon $\mathcal{P}_j = \cup_{i=1}^n \overline{{\rm A}_i{\rm B}_i}$ + \item Extruded along $[\varphi_{min}, \varphi_{max}]$ + \item Detectors defined as set of rays (of origin $D$ and direction $u$) + \item[{$\Rightarrow$}] Light memory-wise + \end{itemize} + \item[{$\Rightarrow$}] Equivalent to: set of truncated cones (frustums) of generatrix $A_iB_i$ + + \end{itemize} +\end{column} +\begin{column}{0.35\textwidth} +\begin{center} + \includegraphics[width=\linewidth]{figures/tore_cones1.pdf} +\end{center} +\end{column} +\end{columns} + +\vspace{0.3cm} +\begin{alertblock}{Ray-tracing algorithm on fusion device $\longrightarrow$ Computation of cone-Ray intersection} +\end{alertblock} + +$$ +\hspace{-3cm} +\exists (q,k) \in [0;1]\times [0;\infty[, \; +\left\{\begin{array}{ll} +R-R_A = q(R_B-R_A)\\ +Z-Z_A = q(Z_B-Z_A)\\ +DM = k u +\end{array}\right. +$$ + +\end{frame} +% ================================= + + +% ================================= +\begin{frame} +\frametitle{Optimization of ray-tracing algorithm} + + + \begin{itemize} + \item Algorithmic optimizations: + \begin{itemize} + \item Test intersection \textbf{bounding-box} + \item \textbf{special cases} (ray direction, segment, etc.) + \item Cone-Ray intersection algorithm: solution of a\textbf{ quadratic equation} + \end{itemize} + \item Pre-computation of variables + \item Core functions in \textbf{Cython} + \item OpenMP parallelization (Cython \textbf{prange} loops) + \end{itemize} +\pause +\begin{table}[h] % just use this specifier if really needed. + \centering + \label{tab:LOS_init_sirrah} + \sisetup{round-mode=places} + \begin{tabular}{@{}l*{4}{S[table-format=1.2e-1,scientific-notation=true]}@{}} + \toprule + \textbf{Nb LOS} & {$10^3$} & {$10^4$} & {$10^5$} & {$10^6$}\\ + \midrule + original & 32.569 & 310.23 & 3202.38 & 31723.827 $(~8$h$48)$\\ + optimized & 0.0258 & 0.2720 & 2.73581 & 26.589564 $(<30$s$)$\\ + 32 threads & 0.0136 & 0.0466 & 0.3640 & 2.9188 \\ + \bottomrule + \end{tabular} +\end{table} +\end{frame} +% ================================= + + +% ================================= +\begin{frame} +\frametitle{Tofu's structure} + +\begin{center} + \includegraphics[width=0.8\linewidth]{figures/tofu.pdf} +\end{center} + +\end{frame} +% ================================= + + + +% ================================= +\begin{frame} +\frametitle{Optimization of spatial integration routines} + + $$M_i(t) = \iiint\limits_{V_i} \vv{\varepsilon(x,t)}\cdot\vv{n} \,\Omega_i \;{d}V$$ + +Integration of \textbf{user-defined function $\varepsilon$} along a LOS: + \begin{itemize} + \item Integration of a python function \textbf{func} defined by user by: + \begin{itemize} + \item \textbf{numpy}.sum (quad: \textbf{midpoint}) + \item Cython based sum (quad: \textbf{midpoint}) + \item \textbf{Scipy}.integrate.simps + \item \textbf{Scipy}.integrate.romb + \end{itemize} + \item Optional optimizations: + \begin{itemize} + \item calls to \textbf{func}: avoid Cython-Python conversion, user-defined + \item memory: fine resolutions, high number of LOS + \item hybrid: compromise + \end{itemize} + \end{itemize} + +\end{frame} +% ================================= + + +%% ================================= +%\begin{frame} +%\frametitle{Optimization of spatial integration routines} +% \begin{figure} +% \begin{tikzpicture} +% \begin{axis}[ +% mbarplot, +% xlabel={Number of Lines of Sight}, +% ylabel={Execution time}, +% width=0.9\textwidth, +% height=6cm, +% xticklabels={$0$,,$10$,,$10^2$, ,$10^3$,,$10^4$}] +% ] +% +% \addplot plot coordinates {(1, 0.46) (2, 2.24) (3, 18.1) }; +% \addplot plot coordinates {(1, 18) (2, 24) (3, 23.5) (4, 13.2)}; +% \addplot plot coordinates {(1, 0.2) (2, 0.53) (3, 4.32) (4, 40)}; +% \addplot plot coordinates {(1, 0.08) (2, 0.44) (3, 4.4) (4, 40)}; +% +% \legend{ref, mem, calls, hybd} +% +% \end{axis} +% \end{tikzpicture} +% \end{figure} +% \vspace{-0.2cm} +% \begin{itemize} +% \item Space resolution: $10^{-3}$ +% \item Number of time steps: $10^3$ +% \item Integration method: \textbf{sum} (Cython or numpy) on midpoint +% \end{itemize} +% \end{frame} +% +% +%% ================================= + + + +% ================================= +\begin{frame} +\frametitle{Optimization of spatial integration routines} + +\begin{table}[h] % just use this specifier if really needed. + \centering + \label{tab:LOS_init_sirrah} + \sisetup{round-mode=places} + \begin{tabular}{@{}lllll@{}} + \toprule + \textbf{LOS} & {$10$} & {$10^2$} & {$10^3$} & {$10^4$} \\% & {$10^5$}\\ + \midrule + original & 0.46 & 2.24 & 18.1 & x \\%& x \\ + calls & 0.207 & 0.53 & 4.2 & x \\ + hybrid & 0.08 & 0.44 & 4.32 & 40.3 (32Gb) \\%& 612 (32Gb)\\ + memory & 0.9 & 8.9 & 96 & 945 (6Gb)\\ + \bottomrule + \end{tabular} +\end{table} + + \vspace{-0.2cm} + \begin{itemize} + \item Space resolution: $10^{-3}$ + \item Number of time steps: $10^3$ + \item Integration method: \textbf{sum} (Cython or numpy) on midpoint + \end{itemize} + \end{frame} +% ================================= + + +% ================================= +\begin{frame} +\frametitle{What we've learned on code optimization} + +Importance of \textbf{cross disciplinary} expert collaborations: + + \begin{itemize} + \item Understand and define the problem (physics) + \item Create optimal algorithm (mathematics) + \item Write efficient code (computer science) + \end{itemize} + +...and from there: + + \begin{itemize} + \item \textbf{Profile} your code: {\textcolor{blue}{\textit{"premature optimization is the root of all evil."} - Sir Tony Hoare}} + \item Core functions: \textbf{Cython} (or Numba or Transonic or ...) + \item Cython specific: + \begin{itemize} + \item \textbf{cython --annotate} + \item \textbf{type} your variables and \textbf{inline} your functions! + \item use cython decorators: \textbf{@cython.boundscheck(False)}, \textbf{@cython.wraparound(False)} + \item release the gil (when possible)! + \item see more: \href{https://github.com/ToFuProject/tofu/blob/master/Notebooks/Cython_speedup_notes.ipynb}{\textcolor{blue}{my notes}}, \href{https://github.com/jeremiedbb/tutorial-euroscipy-2019}{\textcolor{blue}{Jeremie du Boisberranger's tutorial}} + + \end{itemize} + \end{itemize} + +\end{frame} +% ================================= + + +\section{What's next} + + +% ================================= +\begin{frame} +\frametitle{Tofu's structure} + +\begin{center} + \includegraphics[width=0.8\linewidth]{figures/tofu.pdf} +\end{center} + +\end{frame} +% ================================= + +% +%% ================================= +%\begin{frame} +%\frametitle{Optimization of ray-tracing algorithm} +% +% +%$$ +%\hspace{-3cm} +%\exists (q,k) \in [0;1]\times [0;\infty[, \; +%\left\{\begin{array}{ll} +%R-R_A = q(R_B-R_A)\\ +%Z-Z_A = q(Z_B-Z_A)\\ +%DM = k u +%\end{array}\right. +%$$ +%\vspace{-0.5cm} +%\begin{columns} +%\begin{column}{0.65\textwidth} +% \begin{itemize} +% \item Vessel and structures defined as polygons +% \item Algorithm based on \textbf{cone-ray intersection} +% \item Core functions written in \textbf{cython} +% \item Parallelization using \textbf{OpenMP} +% \end{itemize} +%\end{column} +%\begin{column}{0.35\textwidth} +%\vspace{-1cm} +%\begin{center} +% \includegraphics[width=\linewidth]{figures/tore_cones1.pdf} +%\end{center} +%\end{column} +%\end{columns} +% +%\begin{table}[h] % just use this specifier if really needed. +% \centering +% \label{tab:LOS_init_sirrah} +% % Visionner le code LaTeX du paragraphe 0 +% \sisetup{round-mode=places} +% \begin{tabular}{@{}l*{4}{S[table-format=1.2e-1,scientific-notation=true]}@{}} +% \toprule +% \textbf{Nb LOS} & {$10^3$} & {$10^4$} & {$10^5$} & {$10^6$}\\ +% \midrule +% original & 32.569 & 310.23 & 3202.38 & 31723.827 $(~8$h$48)$\\ +% optimized & 0.0258 & 0.2720 & 2.73581 & 26.589564 $(<30$s$)$\\ +% 32 threads & 0.0136 & 0.0466 & 0.3640 & 2.9188 \\ +% \bottomrule +% \end{tabular} +%\end{table} +% +% +%\end{frame} +%% ================================= + + + + + + +% ================================= +\begin{frame} +\frametitle{Tofu's main algorithms} + + +\begin{columns} +\begin{column}{0.6\textwidth} +\textbf{Geometry}: + \begin{itemize} + \item Finite beam width (LOS $\Rightarrow$ VOS) + \item More advanced reflections (specular, diffusive, curved-cubic) + \item Thermal heat load computation + \end{itemize} +\end{column} +\begin{column}{0.4\textwidth} + +\begin{center} + \hspace{-0.5cm}\includegraphics[width=\linewidth]{figures/cones.png} +\end{center} +\end{column} +\end{columns} + +\vspace{0.2cm} +\textbf{Meshing and Inversions}: +\begin{itemize} + \item Meshing and Basis functions (local and global) with visualization + + \item Geometry matrix (fast) computation +\end{itemize} + +%\begin{center} +% \includegraphics[width=0.6\linewidth]{figures/meshes.png} +%\end{center} +\end{frame} +% ================================= + + + +% ================================= +\begin{frame} +\frametitle{On geometry discretization: meshing} + + +\begin{columns} +\begin{column}{0.6\textwidth} +Several options for poloidal cut meshing: + +\begin{itemize} + \item Cartesian mesh + \item (Adaptive) polar mesh + \item Hexagonal mesh + \item Triangular mesh +\end{itemize} + +\end{column} +\begin{column}{0.4\textwidth} + +For basis functions: + +\begin{itemize} + \item B-splines + \item NURBS + \item Box-splines +\end{itemize} +\end{column} +\end{columns} + + + +\begin{center} + \includegraphics[width=0.75\linewidth]{figures/meshes.png} + \includegraphics[width=0.15\linewidth]{figures/maillage3.png} + +\end{center} +\vspace{-0.5cm} + +\begin{alertblock}{$\longrightarrow$ Collaboration with IPP} +\end{alertblock} +\end{frame} +% ================================= + + + +% ================================= +\begin{frame} +\frametitle{Tofu's main algorithms} + + +\begin{columns} +\begin{column}{0.6\textwidth} +\textbf{Geometry}: + \begin{itemize} + \item Finite beam width (LOS $\Rightarrow$ VOS) + \item More advanced reflections + \item Thermal heat load computation + \end{itemize} +\end{column} +\begin{column}{0.4\textwidth} + +\begin{center} + \hspace{-0.5cm}\includegraphics[width=\linewidth]{figures/cones.png} +\end{center} +\end{column} +\end{columns} + +\vspace{0.2cm} +\textbf{Meshing and Inversions}: +\begin{itemize} + \item Meshing and Basis functions (local and global) with visualization + + \item Geometry matrix (fast) computation and introspection plots + \item \textcolor{myblue}{Multiple Inversion-Regularization (linear and non-linear) and visualization + \item pre- and post-inversion tools} +\end{itemize} +\begin{alertblock}{$\longrightarrow$ CEMRACS 2020 project (Machine Learning?)} +\end{alertblock} + +%\begin{center} +% \includegraphics[width=0.6\linewidth]{figures/meshes.png} +%\end{center} +\end{frame} +% ================================= + + +% ================================= +\begin{frame} +\frametitle{Tofu's main algorithms} + + +\begin{columns} +\begin{column}{0.6\textwidth} +\textbf{Geometry}: + \begin{itemize} + \item Finite beam width (LOS $\Rightarrow$ VOS) + \item More advanced reflections + \item Thermal heat load computation + \end{itemize} +\end{column} +\begin{column}{0.4\textwidth} + +\begin{center} + \hspace{-0.5cm}\includegraphics[width=\linewidth]{figures/cones.png} +\end{center} +\end{column} +\end{columns} + +\vspace{0.2cm} +\textbf{Meshing and Inversions}: +\begin{itemize} + \item Meshing and Basis functions (local and global) with visualization + + \item Geometry matrix (fast) computation and introspection plots + \item Multiple Inversion-Regularization (linear and non-linear) and visualization + \item post-inversion analysis tools +\end{itemize} + +\vspace{0.2cm} +\textbf{On the side}: +\begin{itemize} + \item Statistical data analysis (tofu / pandas interface) + \item Basic magnetic field line and particle trajectory tracing + \item continued IMAS support +\end{itemize} + +%\begin{center} +% \includegraphics[width=0.6\linewidth]{figures/meshes.png} +%\end{center} +\end{frame} +% ================================= + + + + + +% ================================= +\begin{frame} +\frametitle{} +{\begin{center} +{\Huge Merci !} +\end{center}} +\vspace{1cm} + +{\footnotesize \emph{ This work has been carried out within the framework of the EUROfusion Consortium and has received funding from the Euratom research and training programme 2014-2018 and 2019-2020 under grant agreement No 633053. The views and opinions expressed herein do not necessarily reflect those of the European Commission.}} +\end{frame} +% ================================= + +% +%% ================================= +%\subsection*{Splines interpolation} +%\begin{frame} +% \frametitle{B(asis)-Splines basis*} +% +% B-Splines of degree $d$ are defined by the \textbf{recursion} formula: +% +% \begin{equation} +% B_j^{d+1}(x)= \dfrac{x - x_j}{x_{j+d}-x_j} B_j^d(x)+ \dfrac{x_{j+1} - x}{x_{j+d+1} - x_{j+1}} B_{j+1}^d (x) +% \end{equation} +% +% Some important properties about B-splines: +% +% \begin{itemize} + +\end{itemize} +% \item Piece-wise polynomials of degree $d \quad \Rightarrow$ \textbf{smoothness} +% \item Compact support $\Rightarrow$ \textbf{sparse matrix system} +% \item Partition of unity $\sum_j Bj (x) = 1$, $\forall x \quad \Rightarrow$ \textbf{conservation laws} +% \end{itemize} +% +% \begin{center} +% \includegraphics[width = 0.3\textwidth]{\figurespath/bsplines1.png} +% \includegraphics[width = 0.3\textwidth]{\figurespath/bsplines3.png} +% \includegraphics[width = 0.3\textwidth]{\figurespath/bsplines5.png} +% \end{center} +% +%\end{frame} +%% ================================= + + +\end{document} diff --git a/Notes_Upgrades/Communications/SeminaireStras20/figures b/Notes_Upgrades/Communications/SeminaireStras20/figures new file mode 120000 index 000000000..b8d0ba1c6 --- /dev/null +++ b/Notes_Upgrades/Communications/SeminaireStras20/figures @@ -0,0 +1 @@ +../Euroscipy2019/figures \ No newline at end of file diff --git a/anaconda_upload.sh b/anaconda_upload.sh index e0f5a9a66..c5d368920 100644 --- a/anaconda_upload.sh +++ b/anaconda_upload.sh @@ -1,8 +1,11 @@ #!/bin/bash set -e -#echo "Converting conda package..." -#conda convert --platform all $PKG_DIR/tofu-*.tar.bz2 --output-dir $PKG_DIR +conda config --set anaconda_upload no +conda install anaconda-client conda-build +conda build conda_recipe +export PKG_REAL=$(conda build . --output | tail -1) +echo $PKG_REAL echo "Deploying to anaconda.org..." export USER=ToFuProject diff --git a/doc/source/index.rst b/doc/source/index.rst index bbcc751dc..6c3086703 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -65,6 +65,7 @@ Contents A guide to contributing to tofu A list of all different device configurations available (ITER, WEST, JET, ...) Tutorials and examples + Using tofu from a bash terminal * How to create / handle a diagnostic geometry - Visit the basic_ tutorial for getting started: create, plot and save diff --git a/doc/source/tofu.data.rst b/doc/source/tofu.data.rst index 952cfb5df..1aeeec85e 100644 --- a/doc/source/tofu.data.rst +++ b/doc/source/tofu.data.rst @@ -41,14 +41,6 @@ tofu.data.\_def module :undoc-members: :show-inheritance: -tofu.data.\_physics module --------------------------- - -.. automodule:: tofu.data._physics - :members: - :undoc-members: - :show-inheritance: - tofu.data.\_plot module ----------------------- diff --git a/doc/source/tofu.geom.rst b/doc/source/tofu.geom.rst index 713d9d717..8a72dec6f 100644 --- a/doc/source/tofu.geom.rst +++ b/doc/source/tofu.geom.rst @@ -9,6 +9,14 @@ tofu.geom package Submodules ---------- +tofu.geom.\_GG.cpython\-36m\-darwin module +------------------------------------------ + +.. automodule:: tofu.geom._GG.cpython-36m-darwin + :members: + :undoc-members: + :show-inheritance: + tofu.geom.\_GG module --------------------- @@ -17,6 +25,14 @@ tofu.geom.\_GG module :undoc-members: :show-inheritance: +tofu.geom.\_basic\_geom\_tools.cpython\-36m\-darwin module +---------------------------------------------------------- + +.. automodule:: tofu.geom._basic_geom_tools.cpython-36m-darwin + :members: + :undoc-members: + :show-inheritance: + tofu.geom.\_basic\_geom\_tools module ------------------------------------- @@ -65,6 +81,14 @@ tofu.geom.\_def module :undoc-members: :show-inheritance: +tofu.geom.\_distance\_tools.cpython\-36m\-darwin module +------------------------------------------------------- + +.. automodule:: tofu.geom._distance_tools.cpython-36m-darwin + :members: + :undoc-members: + :show-inheritance: + tofu.geom.\_distance\_tools module ---------------------------------- @@ -89,6 +113,14 @@ tofu.geom.\_plot\_optics module :undoc-members: :show-inheritance: +tofu.geom.\_raytracing\_tools.cpython\-36m\-darwin module +--------------------------------------------------------- + +.. automodule:: tofu.geom._raytracing_tools.cpython-36m-darwin + :members: + :undoc-members: + :show-inheritance: + tofu.geom.\_raytracing\_tools module ------------------------------------ @@ -97,6 +129,14 @@ tofu.geom.\_raytracing\_tools module :undoc-members: :show-inheritance: +tofu.geom.\_sampling\_tools.cpython\-36m\-darwin module +------------------------------------------------------- + +.. automodule:: tofu.geom._sampling_tools.cpython-36m-darwin + :members: + :undoc-members: + :show-inheritance: + tofu.geom.\_sampling\_tools module ---------------------------------- @@ -105,6 +145,14 @@ tofu.geom.\_sampling\_tools module :undoc-members: :show-inheritance: +tofu.geom.\_vignetting\_tools.cpython\-36m\-darwin module +--------------------------------------------------------- + +.. automodule:: tofu.geom._vignetting_tools.cpython-36m-darwin + :members: + :undoc-members: + :show-inheritance: + tofu.geom.\_vignetting\_tools module ------------------------------------ diff --git a/doc/source/tofu.rst b/doc/source/tofu.rst index cf238008d..40e4fdbe0 100644 --- a/doc/source/tofu.rst +++ b/doc/source/tofu.rst @@ -15,10 +15,20 @@ Subpackages tofu.dumpro tofu.dust tofu.geom + tofu.scripts + tofu.tests Submodules ---------- +tofu.\_physics module +--------------------- + +.. automodule:: tofu._physics + :members: + :undoc-members: + :show-inheritance: + tofu.\_plot module ------------------ diff --git a/doc/source/tofu.scripts.rst b/doc/source/tofu.scripts.rst new file mode 100644 index 000000000..ce5c6da07 --- /dev/null +++ b/doc/source/tofu.scripts.rst @@ -0,0 +1,35 @@ +tofu.scripts package +==================== + +.. automodule:: tofu.scripts + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +tofu.scripts.tofucalc module +---------------------------- + +.. automodule:: tofu.scripts.tofucalc + :members: + :undoc-members: + :show-inheritance: + +tofu.scripts.tofucustom module +------------------------------ + +.. automodule:: tofu.scripts.tofucustom + :members: + :undoc-members: + :show-inheritance: + +tofu.scripts.tofuplot module +---------------------------- + +.. automodule:: tofu.scripts.tofuplot + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/tofu.tests.rst b/doc/source/tofu.tests.rst new file mode 100644 index 000000000..07f57ad83 --- /dev/null +++ b/doc/source/tofu.tests.rst @@ -0,0 +1,17 @@ +tofu.tests package +================== + +.. automodule:: tofu.tests + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + tofu.tests.tests00_root + tofu.tests.tests01_geom + tofu.tests.tests02_data + tofu.tests.tests09_tutorials diff --git a/doc/source/tofu.tests.tests00_root.rst b/doc/source/tofu.tests.tests00_root.rst new file mode 100644 index 000000000..92f06c1f1 --- /dev/null +++ b/doc/source/tofu.tests.tests00_root.rst @@ -0,0 +1,19 @@ +tofu.tests.tests00\_root package +================================ + +.. automodule:: tofu.tests.tests00_root + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +tofu.tests.tests00\_root.tests03\_plot module +--------------------------------------------- + +.. automodule:: tofu.tests.tests00_root.tests03_plot + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/tofu.tests.tests01_geom.rst b/doc/source/tofu.tests.tests01_geom.rst new file mode 100644 index 000000000..348a2d23e --- /dev/null +++ b/doc/source/tofu.tests.tests01_geom.rst @@ -0,0 +1,50 @@ +tofu.tests.tests01\_geom package +================================ + +.. automodule:: tofu.tests.tests01_geom + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + tofu.tests.tests01_geom.tests03_core_data + +Submodules +---------- + +tofu.tests.tests01\_geom.test\_get\_sample module +------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.test_get_sample + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests01\_GG module +------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests01_GG + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests02\_compute module +------------------------------------------------ + +.. automodule:: tofu.tests.tests01_geom.tests02_compute + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core module +--------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/tofu.tests.tests01_geom.tests03_core_data.rst b/doc/source/tofu.tests.tests01_geom.tests03_core_data.rst new file mode 100644 index 000000000..d233b3786 --- /dev/null +++ b/doc/source/tofu.tests.tests01_geom.tests03_core_data.rst @@ -0,0 +1,139 @@ +tofu.tests.tests01\_geom.tests03\_core\_data package +==================================================== + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_CoilPF\_Notes module +----------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_CoilPF_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_Baffle\_Notes module +---------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_Baffle_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_BumperInner\_Notes module +--------------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_BumperInner_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_BumperOuter\_Notes module +--------------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_BumperOuter_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_DivLowGC\_Notes module +------------------------------------------------------------------------------ + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_DivLowGC_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_DivLowITER\_Notes module +-------------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_DivLowITER_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_DivUp\_Notes module +--------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_DivUp_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_IC1\_Notes module +------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_IC1_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_IC2\_Notes module +------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_IC2_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_IC3\_Notes module +------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_IC3_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_LH1\_Notes module +------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_LH1_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_LH2\_Notes module +------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_LH2_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PFC\_Ripple\_Notes module +---------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PFC_Ripple_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_PlasmaDomain\_Standard\_Notes module +--------------------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_PlasmaDomain_Standard_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_Ves\_VesIn\_Notes module +--------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_Ves_VesIn_Notes + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests01\_geom.tests03\_core\_data.WEST\_Ves\_VesOut\_Notes module +---------------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests01_geom.tests03_core_data.WEST_Ves_VesOut_Notes + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/tofu.tests.tests02_data.rst b/doc/source/tofu.tests.tests02_data.rst new file mode 100644 index 000000000..3d0617c9d --- /dev/null +++ b/doc/source/tofu.tests.tests02_data.rst @@ -0,0 +1,19 @@ +tofu.tests.tests02\_data package +================================ + +.. automodule:: tofu.tests.tests02_data + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +tofu.tests.tests02\_data.tests03\_core module +--------------------------------------------- + +.. automodule:: tofu.tests.tests02_data.tests03_core + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/tofu.tests.tests09_tutorials.rst b/doc/source/tofu.tests.tests09_tutorials.rst new file mode 100644 index 000000000..ec8d47a07 --- /dev/null +++ b/doc/source/tofu.tests.tests09_tutorials.rst @@ -0,0 +1,43 @@ +tofu.tests.tests09\_tutorials package +===================================== + +.. automodule:: tofu.tests.tests09_tutorials + :members: + :undoc-members: + :show-inheritance: + +Submodules +---------- + +tofu.tests.tests09\_tutorials.tests01\_runall module +---------------------------------------------------- + +.. automodule:: tofu.tests.tests09_tutorials.tests01_runall + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests09\_tutorials.tuto\_plot\_basic module +------------------------------------------------------ + +.. automodule:: tofu.tests.tests09_tutorials.tuto_plot_basic + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests09\_tutorials.tuto\_plot\_create\_geometry module +----------------------------------------------------------------- + +.. automodule:: tofu.tests.tests09_tutorials.tuto_plot_create_geometry + :members: + :undoc-members: + :show-inheritance: + +tofu.tests.tests09\_tutorials.tuto\_plot\_custom\_emissivity module +------------------------------------------------------------------- + +.. automodule:: tofu.tests.tests09_tutorials.tuto_plot_custom_emissivity + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/source/tofu_from_bash.rst b/doc/source/tofu_from_bash.rst new file mode 100644 index 000000000..6e620455e --- /dev/null +++ b/doc/source/tofu_from_bash.rst @@ -0,0 +1,181 @@ +.. _command_line: + +Bash access to tofu +=================== + +tofu is a python library, and it through a python console that you'll get the most of it. +However, it also provides a few bash commands to be used straight form the terminal. +These commands provide a quick and simple access to a few very common features of tofu. +So far they include: + +- :ref:`tofuplot`: for interactive plotting of data from IMAS +- :ref:`tofucalc`: for computing and interactive plotting of synthetic data from IMAS +- :ref:`tofu-custom`: for setting-up your own tofu preferences + + +.. _tofuplot: + +tofuplot +-------- + +tofuplot is available only if IMAS is also installed on your environment. +In that case, the sub-package imas2tofu will be operational. +This sub-package provides an interface between tofu and IMAS, and allows, +among other things, to use tofu to plot experimental data stored in IMAS in +interactive figures. + +This feature is typically used as follows: + +:: + + $ tofuplot -s 54178 -i ece + +The line above calls tofuplot with the following arguments: + +- -s / --shot : the shot number of the imas data entry (here 54178) +- -i / --ids : the name of the ids we want to get data from (here ece) + +The ids names that can be used are diagnostic ids, they include: + +- soft_x_rays +- bolometer +- interferometer +- polarimeter +- reflectometer_profile +- barometry +- spectrometer_visible +- bremsstrahlung_visible + +Note that you can combine the plots from several ids in the same figure by +simply adding more ids (they will have common time axis): + +:: + + $ tofuplot -s 54178 -i ece soft_x_rays interferometer + + +In all cases, what tofuplot does is simply: + +- read the tokamak geometry from ids wall +- read the diagnostic geometry from the provided ids +- compute the Lines of Sight (LOS) +- read the diagnostic experimental data from the provided ids +- display the data (time traces per LOS) and the geometry (tokamak + LOS) in +an interactive figure + +There are many other parameters that can be specified, like in particular: + +- -tok / --tokamak: the name of the tokamak of the imas data entry +- -u / --user : the user of the imas data entry +- -t0 / --t0 : the name of the time event used as origin (can be a float) + +When a parameter is not specified, a default value is used. + +For help on the other parameters, type: + +:: + + $ tofuplot --help + +Here is an example of the interactive figure + + + +.. _tofucalc: + +tofucalc +-------- + +tofuplot simply reads and plot data. + +By comparison, tofucalc also reads the diagnostics geometry, +but more importantly, it reads plasma profiles (1d radial profiles or 2d maps) +of the quantity of interest for the chosen diagnostic +(i.e.: electron density for interferometer, total radiated power for +bolometer...) and calculates the synthetic data of the +diagnostic (it performs the Line Of Sight integrationi of the quantity). +Finally, it displays the result in the same interactive figure as tofuplot. + +Once you have understood the parameters of tofuplot, tofucalc is intuitive +as it requires the same input: + +:: + + $ tofucalc -s 54178 -i interferometer + +Note however, that tofucalc is available for a limited number of diagnostics. +Indeed, it requires pre-tabulating the quantity of interest for each ids and +implementing proper 2D interpolation methods for each type of profile. +So far, it is available for: + +- interefreometer +- polarimeter +- bolometer +- bremsstrhalung_visible + +Other diagnostics / ids will be added to this list as tofu is developped and +tested on WEST. + +Also note that there is a default profile tabulated for each diagostics. +For example interferometer used the 1d electron density profile stored in ids +core_profiles. +But one could of course object that electron density can be stored as a 2D map +in another ids, produced, for example by a plasma edge code. +Computing synthetic data from an alternative source than the tabulated default +o ne is possible, but through the python console only as it requires a more +advanced use of the features offered by tofu. + +For some specific users, an alternative way of providing the profiles has been +implemented: it is possible to pass the ids not by its identification +parameters (shot, user, tokamak, ids name...), but via an input file saved with +matlab (.mat). +The input file shall contain an exact representation of the ids. +Likewise, the result can be saved into a .mat output file. +This feature is only available for ids bremsstrahlung_visible so far and it +only the 1d radiation profile that is passed throught the input file. +The equilibrium (for interpolation) and diagnostic geometry are still read from +regular IMAS ids. + +:: + + $ tofucalc -s 54178 -i interferometer + + + + +.. _tofu-custom: + +tofu-custom +----------- + +tofu used a lot of default parameters, such that providing no parameter at all +is ok when using most method / functions. +However, you may want to customize some of these default parameters to better +suit your usage or liking. + +If tofu is installed on a shared cluster, you can't access tofu's default +parametersas modifying them would also affect everybody else's usage. + +In order to allow for user-specific customization, run: + +:: + + $ tofu-custom + + +This will create a .tofu/ directory in your home (~/), in which tofu will copy +default parameters and data that you are free to edit. +This local copy is thus user-specific and will always have precedence when +importing tofu. + +Not all parameters can be customized, and this effort is on-going, but to, +as of tofu 1.4.3, you can edit: + +- the imas shortcuts in _imas2tofu-def.py +- the default parameters of tofuplot and tofcalc in _scripts_def.py + +Other parameters will be available for customization in future versions. + +This hidden directory also holds a openadas2tofu/ sub-directory where all data +downloaded by tofu from `openadas `__ +(a free online atomic database) is stored. diff --git a/pyproject.toml b/pyproject.toml index 914ec2419..daf04b8b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ [build-system] -requires = ["setuptools>=40.8.0", "wheel", "cython>0.26", "numpy"] -build-backend = "setuptools.build_meta" \ No newline at end of file +requires = ["setuptools>=40.8.0", + "wheel", + "cython>0.26", + "numpy", + ] diff --git a/release_notes/release_notes_143.rst b/release_notes/release_notes_143.rst new file mode 100644 index 000000000..6030aa54b --- /dev/null +++ b/release_notes/release_notes_143.rst @@ -0,0 +1,133 @@ +==================== +What's new in 1.4.3 +==================== + +Main changes: +============= + +- Python 2 not supported anymore +- Several changes to try and make installation cleaner / easier +- More robust IMAS interface #280, #317, #319, #336, #343, #350 +- Better documentation, more ressources +- More informative error messages +- Better PEP8 compliance + +New features: +~~~~~~~~~~~~~ +- New bash commands tofuplot, tofucalc and tofu-custom #345, #352, #353, #360 +- New **AUG configuration** available! #333 +- More complete CrystalBragg class #320 + + +Detailed changes: +================= + +Installation / portability: +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- travis test matrix #328 +- Skipped deployment if package already detected, deploy bdist for Mac #293 +- Bug corrected Cython vs cython #274 +- pip install working, cleaned up setup.py, requirements.txt included #283 #299 #302 +- More general path delimiters, corrected path in update_version #296 #315 +- Bug fixed on Mac (due to wrong modulo operation) #323 +- Debugged deployment and updating of version number #348 + + +scripts: +~~~~~~~~ +- Introduction of bash command line tools: tofuplot (for plotting IMAS data) + tofucalc (for calculating synthetic signal) and tofu-custom for user-specific + customization of tofu default parameters #345 +- tofucalc also handles input_file and output_file in .mat format #352 +- tofu-custom allows for cutomization of IMAS shortcuts #353 +- tofu-custom allows for cutomization of script parameters #360 + + +Exception / warnings: +~~~~~~~~~~~~~~~~~~~~~ +- More details exception msg when loading wrong Config #327 +- Better warning when a sub-package is not available #269 #270 +- Warning added when gcc compiler < 8 used on Mac #268 +- Better (more concise) warnings when unable to load data from IMAS #280 +- Now warns if IMAS version loaded is not the latest available on the system #289 +- Exception raised when poly ill-defined #332 + + +Default configurations: +~~~~~~~~~~~~~~~~~~~~~~~ +- Asdex Upgrade added (AUG) #333 + + +Geom classes (Struct, Config, CamLOS1D, CamLOS2D...): +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- Reformatted _GG.Poly_Order by _GG.format_poly #338 +- get_summary() routine implemented for Struct and CamLOS2D #305 +- Faster synthetic signal cmputation (calc_signal()) using method='sum' #308 #322 + + +Plasma2D: +~~~~~~~~~ +- Bug fixed in interpolation routine #279 +- Now handles rectangular meshes #280 +- Can now save/load Config to/from IMAS and indch_auto propagated #325 + + +2D XRay spectrometer: +~~~~~~~~~~~~~~~~~~~~~ +- CrystalBragg ow has several geometry and data plotting methods useful for geometrical adjustments and spectrum visualization #320 + + +IMAS interfacing: +~~~~~~~~~~~~~~~~~ +- More robust MultiIDSLoader vs one-time-step signal #280 +- Now loads rectangular meshes #280 +- add_ids() now also works with idd handle #317 +- It is now possible to add the ids corresponding to the synthetic diagnostics of an ids, and optionally to add them from a diffrent idd #319 +- description_2d input argument propagated #336 +- Automated selection of valiud channels (indch_auto=True) now more robust #343 +- Updated signal for relectometer_profile #350 + + +Documentation: +~~~~~~~~~~~~~~ +- New tutorials and link to release notes #273 #297 #300 #330 +- Better badges on Github #278 +- Better release notes #284 +- Documentation page on bash command lines + + +Miscellaneous: +~~~~~~~~~~~~~~ +- Better PEP8 compliance #285 +- Removed useless files #290 #324 +- Minutes from meeting on 27.11.2019 added #291 +- allow_pickle available to users #358 + + +Contributors: +============= +Many thanks to all developpers: +- Didier Vezinet (@Didou09) +- Laura Mendoza (@lasofivec) +- Florian Le Bourdais (@flothesof) +- Jack Hare (@harej) + + +What's next (indicative): +========================= +- Migrating from nosetests (ongoing for @lasofivec : issues #95 and #232 ) +- Easier binary and source installs from pip and conda for all platforms, including unit tests on alla platforms (ongoing for @lasofivec and @flothesof : issue #92 and #259 ) +- Solid angles for Volume-Of-Sight and radiative heat loads computation (ongoing for @lasofivec : Issues #71, #72, #73, #74, #75, #76, #77, #78 ) +- Tools and classes to handle 2D Bragg X-Ray crystal spectrometer (ongoing for @Didou09 : Issues #202 and #263) +- Generic data class to incorporate plateau-finding, data analysis and 1d Bayesian fitting routines and classes (ongoing for @Didou09 and @jmoralesFusion and @MohammadKozeiha: issues #208, #260 and #262) +- More general magnetic field line tracing workflow +- Better unit tests coverage +- More complete documentation + + +List of PR merged into this release: +==================================== +- PR: #268, #269, #270, #273, #274, #278, #279, #280, #282, #283, #284, #285, + #289, #290, #291, #293, #296, #297, #299, #300, #302, #305, #308, #309, #315, + #317, #319, #320, #322, #323, #324, #325, #327, #328, #330, #332, #333, #336, + #338, #343, #345, #348, #352, #353, #358, #360 diff --git a/setup.py b/setup.py index 443fdd0f2..5784c3292 100644 --- a/setup.py +++ b/setup.py @@ -16,11 +16,8 @@ # ... packages that need to be in pyproject.toml from Cython.Distutils import build_ext import numpy as np -# ... -# Remove _updateversion import due to build-time dependency -# => updateversion defined in here for commodity -# => find a cleaner solution later -# import _updateversion as up +# ... local script +import _updateversion as up # ... for `clean` command from distutils.command.clean import clean as Clean @@ -149,25 +146,6 @@ def check_for_openmp(cc_var): _HERE = os.path.abspath(os.path.dirname(__file__)) -def updateversion(path=_HERE): - # Fetch version from git tags, and write to version.py - # Also, when git is not available (PyPi package), use stored version.py - version_py = os.path.join(path, 'tofu', 'version.py') - try: - version_git = subprocess.check_output(["git", - "describe"]).rstrip().decode() - except subprocess.CalledProcessError: - with open(version_py, 'r') as fh: - version_git = fh.read().strip().split("=")[-1].replace("'", '') - version_git = version_git.lower().replace('v', '') - - version_msg = "# Do not edit, pipeline versioning governed by git tags!" - with open(version_py, "w") as fh: - msg = "{0}__version__ = '{1}'{0}".format(os.linesep, version_git) - fh.write(version_msg + msg) - return version_git - - def get_version_tofu(path=_HERE): # Try from git @@ -186,8 +164,9 @@ def get_version_tofu(path=_HERE): .rstrip() .decode() ) - if git_branch in ["master", "deploy-test"]: - version_tofu = updateversion() + deploy_branches = ["master", "deploy-test"] + if (git_branch in deploy_branches or "TRAVIS_TAG" in os.environ): + version_tofu = up.updateversion() else: isgit = False except Exception: @@ -301,7 +280,7 @@ def get_version_tofu(path=_HERE): # The project's main homepage. url="https://github.com/ToFuProject/tofu", # Author details - author="Didier VEZINET", + author="Didier VEZINET and Laura MENDOZA", author_email="didier.vezinet@gmail.com", # Choose your license @@ -395,14 +374,19 @@ def get_version_tofu(path=_HERE): # installing-additional-files # noqa # In this case, 'data_file' will be installed into '/my_data' # data_files=[('my_data', ['data/data_file'])], + # To provide executable scripts, use entry points in preference to the # "scripts" keyword. Entry points provide cross-platform support and allow # pip to create the appropriate form of executable for the target platform. - # entry_points={ - # 'console_scripts': [ - # 'sample=sample:main', - # ], - # }, + entry_points={ + 'console_scripts': [ + 'tofuplot=tofu.scripts.tofuplot:main', + 'tofucalc=tofu.scripts.tofucalc:main', + 'tofu-custom=tofu.scripts.tofucustom:main', + ], + }, + + # Extensions and commands ext_modules=extensions, cmdclass={"build_ext": build_ext, "clean": CleanCommand}, diff --git a/tofu/__init__.py b/tofu/__init__.py index 4ef97c747..7bf166d8c 100644 --- a/tofu/__init__.py +++ b/tofu/__init__.py @@ -74,7 +74,7 @@ # ------------------------------------- msg = None -dsub = dict.fromkeys(['imas2tofu', 'mag']) +dsub = dict.fromkeys(['imas2tofu', 'openadas2tofu', 'mag']) for sub in dsub.keys(): try: exec('import tofu.{0} as {0}'.format(sub)) diff --git a/tofu/data/_physics.py b/tofu/_physics.py similarity index 100% rename from tofu/data/_physics.py rename to tofu/_physics.py diff --git a/tofu/data/_core.py b/tofu/data/_core.py index a4f45c47e..0529c28ec 100644 --- a/tofu/data/_core.py +++ b/tofu/data/_core.py @@ -24,13 +24,13 @@ import tofu.data._comp as _comp import tofu.data._plot as _plot import tofu.data._def as _def - import tofu.data._physics as _physics + import tofu._physics as _physics import tofu.data._spectrafit2d as _spectrafit2d except Exception: from . import _comp as _comp from . import _plot as _plot from . import _def as _def - from . import _physics as _physics + from .. import _physics as _physics from . import _spectrafit2d as _spectrafit2d __all__ = ['DataCam1D','DataCam2D', diff --git a/tofu/data/_core_new.py b/tofu/data/_core_new.py index 6c9b97937..0b2b49d33 100644 --- a/tofu/data/_core_new.py +++ b/tofu/data/_core_new.py @@ -24,12 +24,12 @@ import tofu.data._comp as _comp import tofu.data._plot as _plot import tofu.data._def as _def - import tofu.data._physics as _physics + import tofu._physics as _physics except Exception: from . import _comp as _comp from . import _plot as _plot from . import _def as _def - from . import _physics as _physics + from .. import _physics as _physics __all__ = ['DataHolder'] # , 'Plasma0D'] diff --git a/tofu/geom/_core.py b/tofu/geom/_core.py index fc4c2c8bf..cf70a7227 100644 --- a/tofu/geom/_core.py +++ b/tofu/geom/_core.py @@ -3594,7 +3594,7 @@ def __init__(self, dgeom=None, lOptics=None, Etendues=None, Surfaces=None, config=None, dchans=None, dX12='geom', Id=None, Name=None, Exp=None, shot=None, Diag=None, sino_RefPt=None, fromdict=None, sep=None, method='optimized', - SavePath=os.path.abspath('./'), color=None, plotdebug=True): + SavePath=os.path.abspath('./'), color=None): # Create a dplot at instance level self._dplot = copy.deepcopy(self.__class__._dplot) @@ -3901,20 +3901,31 @@ def _checkformat_dOptics(lOptics=None): @staticmethod def _checkformat_inputs_dconfig(config=None): - C0 = isinstance(config, Config) - msg = "Arg config must be a Config instance !" - msg += "\n expected : {0}".format(str(Config)) - msg += "\n obtained : {0}".format(str(config.__class__)) - assert C0, msg + # Check config has proper class + if not isinstance(config, Config): + msg = ("Arg config must be a Config instance!\n" + + "\t- expected: {}".format(str(Config)) + + "\t- provided: {}".format(str(config.__class__))) + raise Exception(msg) + + # Check all structures lS = config.lStruct - lC = [ - hasattr(ss, "_InOut") and ss._InOut in ["in", "out"] for ss in lS - ] - msg = "All Struct in config must have self._InOut in ['in','out']" - assert all(lC), msg + lC = [hasattr(ss, "_InOut") and ss._InOut in ["in", "out"] + for ss in lS] + if not all(lC): + msg = "All Struct in config must have self._InOut in ['in','out']" + raise Exception(msg) + + # Check there is at least one struct which is a subclass of StructIn lSIn = [ss for ss in lS if ss._InOut == "in"] - msg = "Arg config must have at least a StructIn subclass !" - assert len(lSIn) > 0, msg + if len(lSIn) == 0: + lclsnames = ['{}; {}'.format(ss.Id.Name, ss.Id.Cls, ss._InOut) + for ss in lS] + msg = ("Config {} is missing a StructIn!\n".format(config.Id.Name) + + "\t- " + "\n\t- ".join(lclsnames)) + raise Exception(msg) + + # Add 'compute' parameter if not present if "compute" not in config._dextraprop["lprop"]: config = config.copy() config.add_extraprop("compute", True) @@ -4436,10 +4447,20 @@ def _compute_kInOut(self, largs=None, dkwd=None, indStruct=None): indout[0, :] = indStruct[indout[0, :]] return kIn, kOut, vperp, indout, indStruct - def compute_dgeom(self, extra=True, plotdebug=True): + def compute_dgeom(self, extra=True, show_debug_plot=True): + """ Compute dictionnary of geometrical attributes (dgeom) + + Parameters + ---------- + show_debug_plot: bool + In case some lines of sight have no visibility inside the tokamak, + they will be considered invalid. tofu will issue a warning with + their indices and if show_debug_plot is True, try to plot a 3d + figure to help understand why these los have no visibility + """ # Can only be computed if config if provided if self._dconfig["Config"] is None: - msg = "Attribute dgeom cannot be computed without a config !" + msg = "Attribute dgeom cannot be computed without a config!" warnings.warn(msg) return @@ -4450,36 +4471,39 @@ def compute_dgeom(self, extra=True, plotdebug=True): # Perform computation of kIn and kOut kIn, kOut, vperp, indout, indStruct = self._compute_kInOut() - # Clean up (in case of nans) + # Check for LOS that have no visibility inside the plasma domain (nan) ind = np.isnan(kIn) kIn[ind] = 0.0 ind = np.isnan(kOut) | np.isinf(kOut) if np.any(ind): - kOut[ind] = np.nan - msg = "Some LOS have no visibility inside the plasma domain !\n" - msg += "Nb. of LOS concerned: %s out of %s\n" % ( - str(ind.sum()), - str(kOut.size), - ) - msg += "Indices of LOS ok:\n" - msg += repr((~ind).nonzero()[0]) - msg += "\nIndices of LOS with no visibility:\n" - msg += repr(ind.nonzero()[0]) - warnings.warn(msg) - if plotdebug: + msg = ("Some LOS have no visibility inside the plasma domain!\n" + + "Nb. of LOS concerned: {} / {}\n".format(ind.sum(), + kOut.size) + + "Indices of LOS ok:\n" + + repr((~ind).nonzero()[0]) + + "\nIndices of LOS with no visibility:\n" + + repr(ind.nonzero()[0])) + if show_debug_plot is True: PIn = self.D[:, ind] + kIn[None, ind] * self.u[:, ind] POut = self.D[:, ind] + kOut[None, ind] * self.u[:, ind] - # To be updated - _plot._LOS_calc_InOutPolProj_Debug( - self.config, - self.D[:, ind], - self.u[:, ind], - PIn, - POut, - nptstot=kOut.size, - Lim=[np.pi / 4.0, 2.0 * np.pi / 4], - Nstep=50, - ) + msg2 = ("\n\tD = {}\n".format(self.D[:, ind]) + + "\tu = {}\n".format(self.u[:, ind]) + + "\tPIn = {}\n".format(PIn) + + "\tPOut = {}".format(POut)) + warnings.warn(msg2) + # plot 3d debug figure + # _plot._LOS_calc_InOutPolProj_Debug( + # self.config, + # self.D[:, ind], + # self.u[:, ind], + # PIn, + # POut, + # nptstot=kOut.size, + # Lim=[np.pi / 4.0, 2.0 * np.pi / 4], + # Nstep=50, + # ) + kOut[ind] = np.nan + raise Exception(msg) # Handle particular cases with kIn > kOut ind = np.zeros(kIn.shape, dtype=bool) diff --git a/tofu/geom/_def.py b/tofu/geom/_def.py index 2119c8236..0e0bca89d 100644 --- a/tofu/geom/_def.py +++ b/tofu/geom/_def.py @@ -164,7 +164,7 @@ def Plot_3D_plt_Tor_DefAxes(fs=None, wintit='tofu'): ax.set_xlabel(r"X (m)") ax.set_ylabel(r"Y (m)") ax.set_zlabel(r"Z (m)") - ax.set_aspect(aspect="equal", adjustable='datalim') + # ax.set_aspect(aspect="equal", adjustable='datalim') return ax diff --git a/tofu/geom/_plot.py b/tofu/geom/_plot.py index 88235b1a3..b01c29665 100644 --- a/tofu/geom/_plot.py +++ b/tofu/geom/_plot.py @@ -788,19 +788,14 @@ def _LOS_calc_InOutPolProj_Debug(config, Ds, us ,PIns, POuts, msg = '_LOS_calc_InOutPolProj - Debugging %s / %s pts'%(str(nP),str(nptstot)) ax.set_title(msg) ax.plot(pts[0,:], pts[1,:], pts[2,:], c='k', lw=1, ls='-') - ax.plot(PIns[0,:],PIns[1,:],PIns[2,:], c='b', ls='None', marker='o', label=r"PIn") - ax.plot(POuts[0,:],POuts[1,:],POuts[2,:], c='r', ls='None', marker='x', label=r"POut") - #ax.legend(**_def.TorLegd) + # ax.plot(PIns[0,:],PIns[1,:],PIns[2,:], + # c='b', ls='None', marker='o', label=r"PIn") + # ax.plot(POuts[0,:],POuts[1,:],POuts[2,:], + # c='r', ls='None', marker='x', label=r"POut") + # ax.legend(**_def.TorLegd) if draw: ax.figure.canvas.draw() - msg = "\nDebugging %s / %s pts with no visibility:\n"%(str(nP),str(nptstot)) - msg += " D = %s\n"%str(Ds) - msg += " u = %s\n"%str(us) - msg += " PIn = %s\n"%str(PIns) - msg += " POut = %s\n"%str(POuts) - print(msg) - def _get_LLOS_Leg(GLLOS, Leg=None, ind=None, Val=None, Crit='Name', PreExp=None, PostExp=None, Log='any', InOut='In'): diff --git a/tofu/imas2tofu/__init__.py b/tofu/imas2tofu/__init__.py index 842ecaef4..5df50ae23 100644 --- a/tofu/imas2tofu/__init__.py +++ b/tofu/imas2tofu/__init__.py @@ -11,8 +11,10 @@ try: try: from tofu.imas2tofu._core import * + from tofu.imas2tofu._mat2ids2calc import * except Exception: from ._core import * + from ._mat2ids2calc import * except Exception as err: if str(err) == 'imas not available': msg = "" diff --git a/tofu/imas2tofu/_core.py b/tofu/imas2tofu/_core.py index 2b47dfb5b..12c39fc37 100644 --- a/tofu/imas2tofu/_core.py +++ b/tofu/imas2tofu/_core.py @@ -23,6 +23,25 @@ import matplotlib as mpl import datetime as dtm +# tofu +pfe = os.path.join(os.path.expanduser('~'), '.tofu', '_imas2tofu_def.py') +if os.path.isfile(pfe): + # Make sure we load the user-specific file + # sys.path method + # sys.path.insert(1, os.path.join(os.path.expanduser('~'), '.tofu')) + # import _scripts_def as _defscripts + # _ = sys.path.pop(1) + # importlib method + import importlib.util + spec = importlib.util.spec_from_file_location("_defimas2tofu", pfe) + _defimas2tofu = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_defimas2tofu) +else: + try: + import tofu.imas2tofu._def as _defimas2tofu + except Exception as err: + from . import _def as _defimas2tofu + # imas try: import imas @@ -47,9 +66,10 @@ _IMAS_VERSION = '3' _IMAS_SHOTR = -1 _IMAS_RUNR = -1 -_IMAS_DIDD = {'shot':_IMAS_SHOT, 'run':_IMAS_RUN, - 'refshot':_IMAS_SHOTR, 'refrun':_IMAS_RUNR, - 'user':_IMAS_USER, 'tokamak':_IMAS_TOKAMAK, 'version':_IMAS_VERSION} +_IMAS_DIDD = {'shot': _IMAS_SHOT, 'run': _IMAS_RUN, + 'refshot': _IMAS_SHOTR, 'refrun': _IMAS_RUNR, + 'user': _IMAS_USER, 'tokamak': _IMAS_TOKAMAK, + 'version': _IMAS_VERSION} # Root tofu path (for saving repo in IDS) _ROOT = os.path.abspath(os.path.dirname(__file__)) @@ -89,377 +109,8 @@ class MultiIDSLoader(object): 'shot', 'run', 'refshot', 'refrun'] # Known short version of signal str - - _dshort = { - 'wall': - {'wallR':{'str':'description_2d[0].limiter.unit[0].outline.r'}, - 'wallZ':{'str':'description_2d[0].limiter.unit[0].outline.z'}}, - - 'pulse_schedule': - {'events_times':{'str':'event[].time_stamp'}, - 'events_names':{'str':'event[].identifier'}}, - - 'equilibrium': - {'t':{'str':'time'}, - 'ip':{'str':'time_slice[time].global_quantities.ip', - 'dim':'current', 'quant':'Ip', 'units':'A'}, - 'q0':{'str':'time_slice[time].global_quantities.q_axis'}, - 'qmin':{'str':'time_slice[time].global_quantities.q_min.value'}, - 'q95':{'str':'time_slice[time].global_quantities.q_95'}, - 'volume':{'str':'time_slice[time].global_quantities.volume', - 'dim':'volume', 'quant':'pvol', 'units':'m3'}, - 'psiaxis':{'str':'time_slice[time].global_quantities.psi_axis', - 'dim':'B flux', 'quant':'psi', 'units':'Wb'}, - 'psisep':{'str':'time_slice[time].global_quantities.psi_boundary', - 'dim':'B flux', 'quant':'psi', 'units':'Wb'}, - 'BT0':{'str':'time_slice[time].global_quantities.magnetic_axis.b_field_tor', - 'dim':'B', 'quant':'BT', 'units':'T'}, - 'axR':{'str':'time_slice[time].global_quantities.magnetic_axis.r', - 'dim':'distance', 'quant':'R', 'units':'m'}, - 'axZ':{'str':'time_slice[time].global_quantities.magnetic_axis.z', - 'dim':'distance', 'quant':'Z', 'units':'m'}, - 'x0R':{'str':'time_slice[time].boundary.x_point[0].r'}, - 'x0Z':{'str':'time_slice[time].boundary.x_point[0].z'}, - 'x1R':{'str':'time_slice[time].boundary.x_point[1].r'}, - 'x1Z':{'str':'time_slice[time].boundary.x_point[1].z'}, - 'strike0R':{'str':'time_slice[time].boundary.strike_point[0].r'}, - 'strike0Z':{'str':'time_slice[time].boundary.strike_point[0].z'}, - 'strike1R':{'str':'time_slice[time].boundary.strike_point[1].r'}, - 'strike1Z':{'str':'time_slice[time].boundary.strike_point[1].z'}, - 'sepR':{'str':'time_slice[time].boundary_separatrix.outline.r'}, - 'sepZ':{'str':'time_slice[time].boundary_separatrix.outline.z'}, - - '1drhotn':{'str':'time_slice[time].profiles_1d.rho_tor_norm', - 'dim':'rho', 'quant':'rhotn', 'units':'adim.'}, - '1dphi':{'str':'time_slice[time].profiles_1d.phi', - 'dim':'B flux', 'quant':'phi', 'units':'Wb'}, - '1dpsi':{'str':'time_slice[time].profiles_1d.psi', - 'dim':'B flux', 'quant':'psi', 'units':'Wb'}, - '1dq':{'str':'time_slice[time].profiles_1d.q', - 'dim':'safety factor', 'quant':'q', 'units':'adim.'}, - '1dpe':{'str':'time_slice[time].profiles_1d.pressure', - 'dim':'pressure', 'quant':'pe', 'units':'Pa'}, - '1djT':{'str':'time_slice[time].profiles_1d.j_tor', - 'dim':'vol. current dens.', 'quant':'jT', 'units':'A/m^2'}, - - '2dphi':{'str':'time_slice[time].ggd[0].phi[0].values', - 'dim':'B flux', 'quant':'phi', 'units':'Wb'}, - '2dpsi':{'str':'time_slice[time].ggd[0].psi[0].values', - 'dim':'B flux', 'quant':'psi', 'units':'Wb'}, - '2djT':{'str':'time_slice[time].ggd[0].j_tor[0].values', - 'dim':'vol. current dens.', 'quant':'jT', 'units':'A/m^2'}, - '2dBR':{'str':'time_slice[time].ggd[0].b_field_r[0].values', - 'dim':'B', 'quant':'BR', 'units':'T'}, - '2dBT':{'str':'time_slice[time].ggd[0].b_field_tor[0].values', - 'dim':'B', 'quant':'BT', 'units':'T'}, - '2dBZ':{'str':'time_slice[time].ggd[0].b_field_z[0].values', - 'dim':'B', 'quant':'BZ', 'units':'T'}, - '2dmeshNodes': {'str': ('grids_ggd[0].grid[0].space[0]' - + '.objects_per_dimension[0]' - + '.object[].geometry')}, - '2dmeshFaces': {'str': ('grids_ggd[0].grid[0].space[0]' - + '.objects_per_dimension[2]' - + '.object[].nodes')}, - '2dmeshR': {'str': 'time_slice[0].profiles_2d[0].r'}, - '2dmeshZ': {'str': 'time_slice[0].profiles_2d[0].z'}}, - - 'core_profiles': - {'t':{'str':'time'}, - 'ip':{'str':'global_quantities.ip', - 'dim':'current', 'quant':'Ip', 'units':'A'}, - 'vloop':{'str':'global_quantities.v_loop', - 'dim':'voltage', 'quant':'Vloop', 'units':'V/m'}, - - '1dTe':{'str':'profiles_1d[time].electrons.temperature', - 'dim':'temperature', 'quant':'Te', 'units':'eV'}, - '1dne':{'str':'profiles_1d[time].electrons.density', - 'dim':'density', 'quant':'ne', 'units':'/m^3'}, - '1dzeff':{'str':'profiles_1d[time].zeff', - 'dim':'charge', 'quant':'zeff', 'units':'adim.'}, - '1dphi':{'str':'profiles_1d[time].grid.phi', - 'dim':'B flux', 'quant':'phi', 'units':'Wb'}, - '1dpsi':{'str':'profiles_1d[time].grid.psi', - 'dim':'B flux', 'quant':'psi', 'units':'Wb'}, - '1drhotn':{'str':'profiles_1d[time].grid.rho_tor_norm', - 'dim':'rho', 'quant':'rhotn', 'units':'adim.'}, - '1drhopn':{'str':'profiles_1d[time].grid.rho_pol_norm', - 'dim':'rho', 'quant':'rhopn', 'units':'adim.'}, - '1dnW':{'str':'profiles_1d[time].ions[identifier.label=W].density', - 'dim':'density', 'quant':'nI', 'units':'/m^3'}}, - - 'edge_profiles': - {'t':{'str':'time'}}, - - 'core_sources': - {'t':{'str':'time'}, - '1dpsi':{'str':'source[identifier.name=lineradiation].profiles_1d[time].grid.psi', - 'dim':'B flux', 'quant':'psi', 'units':'Wb'}, - '1drhotn':{'str':'source[identifier.name=lineradiation].profiles_1d[time].grid.rho_tor_norm', - 'dim':'rho', 'quant':'rhotn', 'units':'Wb'}, - '1dbrem':{'str':"source[identifier.name=bremsstrahlung].profiles_1d[time].electrons.energy", - 'dim':'vol.emis.', 'quant':'brem.', 'units':'W/m^3'}, - '1dline':{'str':"source[identifier.name=lineradiation].profiles_1d[time].electrons.energy", - 'dim':'vol. emis.', 'quant':'lines', 'units':'W/m^3'}}, - - 'edge_sources': - {'t':{'str':'time'}, - '2dmeshNodes':{'str':'grid_ggd[0].space[0].objects_per_dimension[0].object[].geometry'}, - '2dmeshFaces':{'str':'grid_ggd[0].space[0].objects_per_dimension[2].object[].nodes'}, - '2dradiation':{'str':'source[13].ggd[0].electrons.energy[0].values', - 'dim':'vol. emis.', 'quant':'vol.emis.', - 'name':'tot. vol. emis.','units':'W/m^3'}}, - - 'lh_antennas': - {'t':{'str':'antenna[chan].power_launched.time'}, - 'power0':{'str':'antenna[0].power_launched.data', - 'dim':'power', 'quant':'lh power', 'units':'W', - 'pos':True}, - 'power1':{'str':'antenna[1].power_launched.data', - 'dim':'power', 'quant':'lh power', 'units':'W', - 'pos':True}, - 'power':{'str':'antenna[chan].power_launched.data', - 'dim':'power', 'quant':'lh power', 'units':'W', - 'pos':True}, - 'R':{'str':'antenna[chan].position.r.data', - 'dim':'distance', 'quant':'R', 'units':'m'}}, - - 'ic_antennas': - {'t':{'str':'antenna[chan].module[0].power_forward.time'}, - 'power0mod_fwd':{'str':'antenna[0].module[].power_forward.data', - 'dim':'power', 'quant':'ic power', 'units':'W'}, - 'power0mod_reflect':{'str':'antenna[0].module[].power_reflected.data', - 'dim':'power', 'quant':'ic power', 'units':'W'}, - 'power1mod_fwd':{'str':'antenna[1].module[].power_forward.data', - 'dim':'power', 'quant':'ic power', 'units':'W'}, - 'power1mod_reflect':{'str':'antenna[1].module[].power_reflected.data', - 'dim':'power', 'quant':'ic power', 'units':'W'}, - 'power2mod_fwd':{'str':'antenna[2].module[].power_forward.data', - 'dim':'power', 'quant':'ic power', 'units':'W'}, - 'power2mod_reflect':{'str':'antenna[2].module[].power_reflected.data', - 'dim':'power', 'quant':'ic power', 'units':'W'}, - }, - - 'magnetics': - {'t':{'str':'time'}, - 'ip':{'str':'method[0].ip.data'}, - 'diamagflux':{'str':'method[0].diamagnetic_flux.data'}, - 'bpol_B':{'str':'bpol_probe[chan].field.data', - 'dim':'B', 'quant':'Bpol', 'units':'T'}, - 'bpol_name':{'str':'bpol_probe[chan].name'}, - 'bpol_R':{'str':'bpol_probe[chan].position.r', - 'dim':'distance', 'quant':'R', 'units':'m'}, - 'bpol_Z':{'str':'bpol_probe[chan].position.z', - 'dim':'distance', 'quant':'Z', 'units':'m'}, - 'bpol_angpol':{'str':'bpol_probe[chan].poloidal_angle', - 'dim':'angle', 'quant':'angle_pol', 'units':'rad'}, - 'bpol_angtor':{'str':'bpol_probe[chan].toroidal_angle', - 'dim':'angle', 'quant':'angle_tor', 'units':'rad'}, - 'floop_flux':{'str':'flux_loop[chan].flux.data', - 'dim':'B flux', 'quant':'B flux', 'units':'Wb'}, - 'floop_name':{'str':'flux_loop[chan].name'}, - 'floop_R': {'str': 'flux_loop[chan].position.r', - 'dim': 'distance', 'quant': 'R', 'units': 'm'}, - 'floop_Z': {'str': 'flux_loop[chan].position.z', - 'dim': 'distance', 'quant': 'Z', 'units': 'm'}}, - - 'barometry': - {'t': {'str': 'gauge[chan].pressure.time'}, - 'names': {'str': 'gauge[chan].name'}, - 'p': {'str': 'gauge[chan].pressure.data', - 'dim': 'pressure', 'quant': 'p', 'units': 'Pa?'}}, - - 'neutron_diagnostic': - {'t':{'str':'time', 'units':'s'}, - 'flux_total':{'str':'synthetic_signals.total_neutron_flux', - 'dim':'particle flux', 'quant':'particle flux', 'units':'Hz'}}, - - 'ece': - {'t':{'str':'time', - 'quant':'t', 'units':'s'}, - 'freq':{'str':'channel[chan].frequency.data', - 'dim':'freq', 'quant':'freq', 'units':'Hz'}, - 'Te': {'str':'channel[chan].t_e.data', - 'dim':'temperature', 'quant':'Te', 'units':'eV'}, - 'R': {'str':'channel[chan].position.r.data', - 'dim':'distance', 'quant':'R', 'units':'m'}, - 'rhotn':{'str':'channel[chan].position.rho_tor_norm.data', - 'dim':'rho', 'quant':'rhotn', 'units':'adim.'}, - 'theta':{'str':'channel[chan].position.theta.data', - 'dim':'angle', 'quant':'theta', 'units':'rad.'}, - 'tau1keV':{'str':'channel[chan].optical_depth.data', - 'dim':'optical_depth', 'quant':'tau', 'units':'adim.'}, - 'validity_timed': {'str':'channel[chan].t_e.validity_timed'}, - 'names': {'str': 'channel[chan].name'}, - 'Te0': {'str':'t_e_central.data', - 'dim':'temperature', 'quant':'Te', 'units':'eV'}}, - - 'reflectometer_profile': - {'t':{'str':'time'}, - 'ne':{'str':'channel[chan].n_e.data', - 'dim':'density', 'quant':'ne', 'units':'/m^3'}, - 'R':{'str':'channel[chan].position.r.data', - 'dim':'distance', 'quant':'R', 'units':'m'}, - 'Z':{'str':'channel[chan].position.z.data', - 'dim':'distance', 'quant':'Z', 'units':'m'}, - 'phi':{'str':'channel[chan].position.phi.data', - 'dim':'angle', 'quant':'phi', 'units':'rad'}, - 'names': {'str': 'channel[chan].name'}, - 'mode': {'str': 'mode'}, - 'sweep': {'str': 'sweep_time'}}, - - 'interferometer': - {'t': {'str': 'time', - 'quant': 't', 'units': 's'}, - 'names': {'str': 'channel[chan].name'}, - 'ne_integ': {'str': 'channel[chan].n_e_line.data', - 'dim': 'ne_integ', 'quant': 'ne_integ', - 'units': '/m2', 'Brightness': True}}, - - 'polarimeter': - {'t': {'str': 'time', - 'quant': 't', 'units': 's'}, - 'lamb': {'str': 'channel[chan].wavelength', - 'dim': 'distance', 'quant': 'wavelength', - 'units': 'm'}, - 'fangle': {'str': 'channel[chan].faraday_angle.data', - 'dim': 'angle', 'quant': 'faraday angle', - 'units': 'rad', 'Brightness': True}, - 'names': {'str': 'channel[chan].name'}}, - - 'bolometer': - {'t': {'str': 'channel[chan].power.time', - 'quant': 't', 'units': 's'}, - 'power': {'str': 'channel[chan].power.data', - 'dim': 'power', 'quant': 'power radiative', - 'units': 'W', 'Brightness': False}, - 'etendue': {'str': 'channel[chan].etendue', - 'dim': 'etendue', 'quant': 'etendue', - 'units': 'm2.sr'}, - 'names': {'str': 'channel[chan].name'}, - 'tpower': {'str': 'time', 'quant': 't', 'units': 's'}, - 'prad': {'str': 'power_radiated_total', - 'dim': 'power', 'quant': 'power radiative', - 'units': 'W'}, - 'pradbulk': {'str': 'power_radiated_inside_lcfs', - 'dim': 'power', 'quant': 'power radiative', - 'units': 'W'}}, - - 'soft_x_rays': - {'t': {'str': 'time', - 'quant': 't', 'units': 's'}, - 'power': {'str': 'channel[chan].power.data', - 'dim': 'power', 'quant': 'power radiative', - 'units': 'W', 'Brightness': False}, - 'brightness': {'str': 'channel[chan].brightness.data', - 'dim': 'brightness', 'quant': 'brightness', - 'units': 'W/(m2.sr)', 'Brightness': True}, - 'names': {'str': 'channel[chan].name'}, - 'etendue': {'str': 'channel[chan].etendue', - 'dim': 'etendue', 'quant': 'etendue', - 'units': 'm2.sr'}}, - - 'spectrometer_visible': - {'t':{'str':'channel[chan].grating_spectrometer.radiance_spectral.time', - 'quant':'t', 'units':'s'}, - 'spectra': {'str': ('channel[chan].grating_spectrometer' - + '.radiance_spectral.data'), - 'dim': 'radiance_spectral', - 'quant': 'radiance_spectral', - 'units': 'ph/s/(m2.sr)/m', 'Brightness': True}, - 'names': {'str': 'channel[chan].name'}, - 'lamb':{'str':'channel[chan].grating_spectrometer.wavelengths', - 'dim':'wavelength', 'quant':'wavelength', 'units':'m'}}, - - 'bremsstrahlung_visible': - {'t': {'str': 'time', - 'quant': 't', 'units': 's'}, - 'radiance': {'str': 'channel[chan].radiance_spectral.data', - 'dim': 'radiance_spectral', - 'quant': 'radiance_spectral', - 'units': 'ph/s/(m2.sr)/m', - 'Brightness': True}, - 'names': {'str': 'channel[chan].name'}, - 'lamb_up': {'str':'channel[chan].filter.wavelength_upper'}, - 'lamb_lo': {'str':'channel[chan].filter.wavelength_lower'}}, - } - - _didsdiag = {'magnetics': {'datacls':'DataCam1D', - 'geomcls':False, - 'sig':{'t':'t', - 'data':'bpol_B'}}, - 'barometry':{'datacls':'DataCam1D', - 'geomcls':False, - 'sig':{'t':'t', - 'data':'p'}}, - 'ece':{'datacls':'DataCam1D', - 'geomcls':False, - 'sig':{'t':'t', - 'X':'rhotn_sign', - 'data':'Te'}}, - 'neutron_diagnostic':{'datacls':'DataCam1D', - 'geomcls':False, - 'sig':{'t':'t', - 'data':'flux_total'}}, - 'reflectometer_profile':{'datacls':'DataCam1D', - 'geomcls':False, - 'sig':{'t':'t', - 'X':'R', - 'data':'ne'}}, - 'interferometer':{'datacls':'DataCam1D', - 'geomcls':'CamLOS1D', - 'sig':{'t':'t', - 'data':'ne_integ'}, - 'synth':{'dsynth':{'quant':'core_profiles.1dne', - 'ref1d':'core_profiles.1drhotn', - 'ref2d':'equilibrium.2drhotn'}, - 'dsig':{'core_profiles':['t'], - 'equilibrium':['t']}, - 'Brightness':True}}, - 'polarimeter':{'datacls':'DataCam1D', - 'geomcls':'CamLOS1D', - 'sig':{'t':'t', - 'data':'fangle'}, - 'synth':{'dsynth':{'fargs':['core_profiles.1dne', - 'equilibrium.2dBR', - 'equilibrium.2dBT', - 'equilibrium.2dBZ', - 'core_profiles.1drhotn', - 'equilibrium.2drhotn']}, - 'dsig':{'core_profiles':['t'], - 'equilibrium':['t']}, - 'Brightness':True}}, - 'bolometer':{'datacls':'DataCam1D', - 'geomcls':'CamLOS1D', - 'sig':{'t':'t', - 'data':'power'}, - 'synth':{'dsynth':{'quant':'core_sources.1dprad', - 'ref1d':'core_sources.1drhotn', - 'ref2d':'equilibrium.2drhotn'}, - 'dsig':{'core_sources':['t'], - 'equilibrium':['t']}, - 'Brightness':True}}, - 'soft_x_rays':{'datacls':'DataCam1D', - 'geomcls':'CamLOS1D', - 'sig':{'t':'t', - 'data':'power'}}, - 'spectrometer_visible':{'datacls':'DataCam1DSpectral', - 'geomcls':'CamLOS1D', - 'sig':{'t':'t', - 'lamb':'lamb', - 'data':'spectra'}}, - 'bremsstrahlung_visible':{'datacls':'DataCam1D', - 'geomcls':'CamLOS1D', - 'sig':{'t':'t', - 'data':'radiance'}, - 'synth':{'dsynth':{'quant':['core_profiles.1dTe', - 'core_profiles.1dne', - 'core_profiles.1dzeff'], - 'ref1d':'core_profiles.1drhotn', - 'ref2d':'equilibrium.2drhotn'}, - 'dsig':{'core_profiles':['t'], - 'equilibrium':['t']}, - 'Brightness':True}}} - + _dshort = _defimas2tofu._dshort + _didsdiag = _defimas2tofu._didsdiag _lidsplasma = ['equilibrium', 'core_profiles', 'core_sources', 'edge_profiles', 'edge_sources'] @@ -495,22 +146,41 @@ class MultiIDSLoader(object): dlos['los_pt2Phi'] = {'str':'channel[chan].%s.second_point.phi'%strlos} _dshort[ids].update( dlos ) - # Computing functions - _events = lambda names, t: np.array([(nn,tt) - for nn,tt in zip(*[np.char.strip(names),t])], - dtype=[('name','U%s'%str(np.nanmax(np.char.str_len(np.char.strip(names))))), - ('t',np.float)]) - _RZ2array = lambda ptsR, ptsZ: np.array([ptsR,ptsZ]).T - _losptsRZP = lambda *pt12RZP: np.swapaxes([pt12RZP[:3], pt12RZP[3:]],0,1).T - _add = lambda a0, a1: np.abs(a0 + a1) - _icmod = lambda al, ar, axis=0: np.sum(al - ar, axis=axis) - _eqB = lambda BT, BR, BZ: np.sqrt(BT**2 + BR**2 + BZ**2) + + def _events(names, t): + ustr = 'U{}'.format(np.nanmax(np.char.str_len(np.char.strip(names)))) + return np.array([(nn, tt) + for nn, tt in zip(*[np.char.strip(names), t])], + dtype=[('name', ustr), ('t', np.float)]) + + def _RZ2array(ptsR, ptsZ): + return np.array([ptsR, ptsZ]).T + + def _losptsRZP(*pt12RZP): + return np.swapaxes([pt12RZP[:3], pt12RZP[3:]], 0, 1).T + + def _add(a0, a1): + return np.abs(a0 + a1) + + def _eqB(BT, BR, BZ): + return np.sqrt(BT**2 + BR**2 + BZ**2) + + def _icmod(al, ar, axis=0): + return np.sum(al - ar, axis=axis) + + def _icmodadd(al0, ar0, al1, ar1, al2, ar2, axis=0): + return (np.sum(al0 - ar0, axis=axis) + + np.sum(al1 - ar1, axis=axis) + + np.sum(al2 - ar2, axis=axis)) + def _rhopn1d(psi): return np.sqrt((psi - psi[:, 0:1]) / (psi[:, -1] - psi[:, 0])[:, None]) + def _rhopn2d(psi, psi0, psisep): return np.sqrt( (psi - psi0[:, None]) / (psisep[:, None] - psi0[:, None])) + def _rhotn2d(phi): return np.sqrt(np.abs(phi) / np.nanmax(np.abs(phi), axis=1)[:, None]) @@ -536,6 +206,8 @@ def _rhosign(rho, theta): rho[ind] = -rho[ind] return rho + def _lamb(lamb_up, lamb_lo): + return 0.5*(lamb_up + lamb_lo) _dcomp = { 'pulse_schedule': @@ -577,21 +249,28 @@ def _rhosign(rho, theta): {'bpol_pos':{'lstr':['bpol_R', 'bpol_Z'], 'func':_RZ2array}, 'floop_pos':{'lstr':['floop_R', 'floop_Z'], 'func':_RZ2array}}, - 'ic_antennas': - {'power0': {'lstr':['power0mod_fwd', 'power0mod_reflect'], - 'func': _icmod, 'kargs':{'axis':0}, 'pos':True}, - 'power1': {'lstr':['power1mod_fwd', 'power1mod_reflect'], - 'func': _icmod, 'kargs':{'axis':0}, 'pos':True}, - 'power2': {'lstr':['power2mod_fwd', 'power2mod_reflect'], - 'func': _icmod, 'kargs':{'axis':0}, 'pos':True}}, + 'ic_antennas': { + 'power0': {'lstr': ['power0mod_fwd', 'power0mod_reflect'], + 'func': _icmod, 'kargs': {'axis': 0}, 'pos': True}, + 'power1': {'lstr': ['power1mod_fwd', 'power1mod_reflect'], + 'func': _icmod, 'kargs': {'axis': 0}, 'pos': True}, + 'power2': {'lstr': ['power2mod_fwd', 'power2mod_reflect'], + 'func': _icmod, 'kargs': {'axis': 0}, 'pos': True}, + 'power': {'lstr': ['power0mod_fwd', 'power0mod_reflect', + 'power1mod_fwd', 'power1mod_reflect', + 'power2mod_fwd', 'power2mod_reflect'], + 'func': _icmodadd, 'kargs': {'axis': 0}, + 'pos': True}}, 'ece': {'rhotn_sign':{'lstr':['rhotn','theta'], 'func':_rhosign, 'units':'adim.'}}, 'bremsstrahlung_visible': - {'lamb':{'lstr':['lamb_up','lamb_lo'], 'func':np.mean, - 'dim':'distance', 'quantity':'wavelength', 'units':'m'}} + {'lamb': {'lstr': ['lamb_up', 'lamb_lo'], 'func': _lamb, + 'dim': 'distance', + 'quantity': 'wavelength', + 'units': 'm'}} } _lstr = ['los_pt1R', 'los_pt1Z', 'los_pt1Phi', @@ -902,13 +581,25 @@ def refidd(self): @staticmethod def _getcharray(ar, col=None, sep=' ', line='-', just='l', msg=True): - + c0 = ar is None or len(ar) == 0 + if c0: + return '' ar = np.array(ar, dtype='U') + if ar.ndim == 1: + ar = ar.reshape((1, ar.size)) + # Get just len nn = np.char.str_len(ar).max(axis=0) if col is not None: - assert len(col) == ar.shape[1] + if len(col) not in ar.shape: + msg = ("len(col) should be in np.array(ar, dtype='U').shape:\n" + + "\t- len(col) = {}\n".format(len(col)) + + "\t- ar.shape = {}".format(ar.shape)) + raise Exception(msg) + if len(col) != ar.shape[1]: + ar = ar.T + nn = np.char.str_len(ar).max(axis=0) nn = np.fmax(nn, [len(cc) for cc in col]) # Apply to array @@ -925,13 +616,14 @@ def _getcharray(ar, col=None, sep=' ', line='-', just='l', msg=True): out = '\n'.join(out) return out - @classmethod - def _shortcuts(cls, obj=None, ids=None, return_=False, + @staticmethod + def _shortcuts(obj, ids=None, return_=False, verb=True, sep=' ', line='-', just='l'): - if obj is None: - obj = cls if ids is None: - lids = list(obj._dids.keys()) + if hasattr(obj, '_dids'): + lids = list(obj._dids.keys()) + else: + lids = list(obj._dshort.keys()) elif ids == 'all': lids = list(obj._dshort.keys()) else: @@ -942,7 +634,6 @@ def _shortcuts(cls, obj=None, ids=None, return_=False, lids = [ids] else: lids = ids - lids = sorted(set(lids).intersection(obj._dshort.keys())) short = [] @@ -967,6 +658,22 @@ def _shortcuts(cls, obj=None, ids=None, return_=False, if return_: return short + @classmethod + def get_shortcutsc(cls, ids=None, return_=False, + verb=True, sep=' ', line='-', just='l'): + """ Display and/or return the builtin shortcuts for imas signal names + + By default (ids=None), only display shortcuts for stored ids + To display all possible shortcuts, use ids='all' + To display shortcuts for a specific ids, use ids= + + These shortcuts can be customized (with self.set_shortcuts()) + They are useful for use with self.get_data() + + """ + return cls._shortcuts(cls, ids=ids, return_=return_, verb=verb, + sep=sep, line=line, just=just) + def get_shortcuts(self, ids=None, return_=False, verb=True, sep=' ', line='-', just='l'): """ Display and/or return the builtin shortcuts for imas signal names @@ -979,7 +686,7 @@ def get_shortcuts(self, ids=None, return_=False, They are useful for use with self.get_data() """ - return self._shortcuts(obj=self, ids=ids, return_=return_, verb=verb, + return self._shortcuts(self, ids=ids, return_=return_, verb=verb, sep=sep, line=line, just=just) def set_shortcuts(self, dshort=None): @@ -2485,21 +2192,22 @@ def to_Config(self, Name=None, occ=None, mobile = True elif nmob == 0 and nlim > 0: mobile = False - elif nmob == nlim: + elif nmob > nlim: msgw = 'wall.description_2[{}]'.format(description_2d) - msg = ("\nids wall has same number of limiter / mobile units\n" + msg = ("\nids wall has less limiter than mobile units\n" + "\t- len({}.limiter.unit) = {}\n".format(msgw, nlim) + "\t- len({}.mobile.unit) = {}\n".format(msgw, nmob) - + " => Choosing limiter by default") + + " => Choosing mobile by default") warnings.warn(msg) - mobile = False - else: + mobile = True + elif nmob <= nlim: msgw = 'wall.description_2[{}]'.format(description_2d) - msg = ("Can't decide automatically whether to choose" - + " limiter or mobile!\n" + msg = ("\nids wall has more limiter than mobile units\n" + "\t- len({}.limiter.unit) = {}\n".format(msgw, nlim) - + "\t- len({}.mobile.unit) = {}".format(msgw, nmob)) - raise Exception(msg) + + "\t- len({}.mobile.unit) = {}\n".format(msgw, nmob) + + " => Choosing limiter by default") + warnings.warn(msg) + mobile = False assert isinstance(mobile, bool) # Get PFC @@ -3309,8 +3017,14 @@ class with which the output shall be returned def _checkformat_Cam_geom(self, ids=None, geomcls=None, indch=None): # Check ids + idsok = set(self._lidsdiag).intersection(self._dids.keys()) + if ids is None and len(idsok) == 1: + ids = next(iter(idsok)) + if ids not in self._dids.keys(): - msg = "Provided ids should be available as a self.dids.keys() !" + msg = ("Provided ids should be available as a self.dids.keys()!\n" + + "\t- provided: {}\n".format(str(ids)) + + "\t- available: {}".format(sorted(self._dids.keys()))) raise Exception(msg) if ids not in self._lidsdiag: @@ -3329,123 +3043,175 @@ def _checkformat_Cam_geom(self, ids=None, geomcls=None, indch=None): msg = "Arg geomcls must be in {}".format([False]+lgeom) raise Exception(msg) + if geomcls is False: + msg = "ids {} does not seem to be a ids with a camera".format(ids) + raise Exception(msg) + return geomcls - def _get_indch_geomtdata(self, indch=None, indch_auto=None, - dgeom=None, t=None, - ids=None, out=None, dsig=None, kk=None): - nch = 0 if indch is None else len(indch) - # Get from geometry of LOS consistency - if dgeom is not None: - indnan = np.logical_or(np.any(np.isnan(dgeom[0]), axis=0), - np.any(np.isnan(dgeom[1]), axis=0)) - if np.any(indnan) and not np.all(indnan): - if indch_auto is not True: - dmsg = {True: 'not available', False: 'ok'} - ls = ['index {} los {}'.format(ii, dmsg[indnan[ii]]) - for ii in range(0, dgeom[0].shape[1])] - msg = ("The geometry of all channels is not available !\n" - + "Please choose indch to get all LOS!\n" - + "Currently:\n" - + "\n ".join(ls) - + "\n\n => Solution: choose indch accordingly !") - raise Exception(msg) - else: - msg = ("Geometry missing for some los !\n" - + " => indch automatically set to:\n" - + " {}".format(indch)) - warnings.warn(msg) - if indch is None: - indch = (~indnan).nonzero()[0] - else: - indch = set(indch).intersection((~indnan).nonzero()[0]) - indch = np.array(indch, dtype=int) - - # Get from time vectors consistency - if t is not None: - if indch_auto is not True: - if indch is None: - ls = [ - 'index {} {}.shape {}'.format(ii, kk, - out[dsig[kk]][ii].shape) - for ii in range(0, len(out[dsig[kk]])) - ] - else: - ls = [ - 'index {} {}.shape {}'.format(indch[ii], kk, - out[dsig[kk]][ii].shape) - for ii in range(0, len(out[dsig[kk]])) - ] - msg = ("The following is supposed to be a np.ndarray:\n" - + " - diag: {}\n".format(ids) - + " - shortcut: {}\n".format(dsig[kk]) - + " - used as: {} input\n".format(kk) - + " Observed type: {}\n".format(type(out[dsig[kk]])) - + " Probable cause: non-uniform shape (vs channels)\n" - + " => shapes :\n " - + "\n ".join(ls) - + "\n => Solution: choose indch accordingly !") - raise Exception(msg) - ls = [t[ii].shape for ii in range(0, len(t))] - lsu = list(set([ssu for ssu in ls if 0 not in ssu])) - su = lsu[np.argmax([ls.count(ssu) for ssu in lsu])] - msg = ("indch set automatically for {}\n".format(ids) - + " (due to inhomogenous time shapes)\n" - + " - main shape: {}\n".format(su) - + " - nb. chan. selected: {}\n".format(len(indch)) - + " - indch: {}".format(indch)) - warnings.warn(msg) + def inspect_channels(self, ids=None, occ=None, indch=None, geom=None, + dsig=None, data=None, X=None, datacls=None, + geomcls=None, return_dict=None, return_ind=None, + return_msg=None, verb=None): + # ------------------ + # Preliminary checks + if return_dict is None: + return_dict = False + if return_ind is None: + return_ind = False + if return_msg is None: + return_msg = False + if verb is None: + verb = True + if occ is None: + occ = 0 + if geom is None: + geom = True + compute_ind = return_ind or return_msg or verb + + # Check ids is relevant + idsok = set(self._lidsdiag).intersection(self._dids.keys()) + if ids is None and len(idsok) == 1: + ids = next(iter(idsok)) + + # Check ids has channels (channel, gauge, ...) + lch = ['channel', 'gauge', 'group', 'antenna', + 'pipe', 'reciprocating', 'bpol_probe'] + ind = [ii for ii in range(len(lch)) + if hasattr(self._dids[ids]['ids'][occ], lch[ii])] + if len(ind) == 0: + msg = "ids {} has no attribute with '[chan]' index!".format(ids) + raise Exception(msg) + nch = len(getattr(self._dids[ids]['ids'][0], lch[ind[0]])) - if indch is None: - indch = [ii for ii in range(0, len(t)) if ls[ii] == su] + datacls, geomcls, dsig = self._checkformat_Data_dsig(ids, dsig, + data=data, X=X, + datacls=datacls, + geomcls=geomcls) + if geomcls is False: + geom = False + + # ------------------ + # Extract sig and shapes / values + if geom == 'only': + lsig = [] + else: + lsig = sorted(dsig.values()) + lsigshape = list(lsig) + if geom in ['only', True] and 'LOS' in geomcls: + lkok = set(self._dshort[ids].keys()).union(self._dcomp[ids].keys()) + lsig += [ss for ss in ['los_ptsRZPhi', 'etendue', + 'surface', 'names'] + if ss in lkok] + + out = self.get_data(ids, sig=lsig, + isclose=False, stack=False, nan=True, + pos=False) + dout = {} + for k0, v0 in out.items(): + if len(v0) != nch: + if len(v0) != 1: + import pdb # DB + pdb.set_trace() # DB + continue + if isinstance(v0[0], np.ndarray): + dout[k0] = {'shapes': np.array([vv.shape for vv in v0]), + 'isnan': np.array([np.any(np.isnan(vv)) + for vv in v0])} + if k0 == 'los_ptsRZPhi': + dout[k0]['equal'] = np.array([np.allclose(vv[0, ...], + vv[1, ...]) + for vv in v0]) + elif type(v0[0]) in [int, float, np.int, np.float, str]: + dout[k0] = {'value': np.asarray(v0).ravel()} else: - indch = [indch[ii] for ii in range(0, len(indch)) - if ls[indch[ii]] == su] - - # Get from data consistency - if all([ss is not None for ss in [out, kk, dsig]]): - if indch_auto: - ls = [out[dsig[kk]][ii].shape - for ii in range(0, len(out[dsig[kk]]))] - lsu = list(set([ssu for ssu in ls if 0 not in ssu])) - su = lsu[np.argmax([ls.count(ssu) for ssu in lsu])] - if indch is None: - indch = [ii for ii in range(0, len(out[dsig[kk]])) - if ls[ii] == su] - else: - indch = [indch[ii] for ii in range(0, len(out[dsig[kk]])) - if ls[ii] == su] - msg = ("indch set automatically for {}\n".format(ids) - + " (due to inhomogeneous data shapes)\n" - + " - main shape: {}\n".format(su) - + " - nb. chan. selected: {}\n".format(len(indch)) - + " - indch: {}".format(indch)) + typv = type(v0[0]) + k0str = (self._dshort[ids][k0]['str'] + if k0 in self._dshort[ids].keys() else k0) + msg = ("\nUnknown data type:\n" + + "\ttype({}) = {}".format(k0str, typv)) + raise Exception(msg) + + lsig = sorted(set(lsig).intersection(dout.keys())) + lsigshape = sorted(set(lsigshape).intersection(dout.keys())) + + # -------------- + # Get ind, msg + ind, msg = None, None + if compute_ind: + if geom in ['only', True] and 'los_ptsRZPhi' in out.keys(): + indg = ((np.prod(dout['los_ptsRZPhi']['shapes'], axis=1) == 0) + | dout['los_ptsRZPhi']['isnan'] + | dout['los_ptsRZPhi']['equal']) + if geom == 'only': + indok = ~indg + indchout = indok.nonzero()[0] + if geom != 'only': + shapes0 = np.concatenate([np.prod(dout[k0]['shapes'], + axis=1, keepdims=True) + for k0 in lsigshape], axis=1) + indok = np.all(shapes0 != 0, axis=1) + if geom is True and 'los_ptsRZPhi' in out.keys(): + indok[indg] = False + if not np.any(indok): + indchout = np.array([], dtype=int) + elif geom != 'only': + indchout = (np.arange(0, nch)[indok] + if indch is None else np.r_[indch][indok]) + lshapes = [dout[k0]['shapes'][indchout, :] for k0 in lsigshape] + lshapesu = [np.unique(ss, axis=0) for ss in lshapes] + if any([ss.shape[0] > 1 for ss in lshapesu]): + for ii in range(len(lshapesu)): + if lshapesu[ii].shape[0] > 1: + _, inv, counts = np.unique(lshapes[ii], axis=0, + return_counts=True, + return_inverse=True) + indchout = indchout[inv == np.argmax(counts)] + lshapes = [dout[k0]['shapes'][indchout, :] + for k0 in lsigshape] + lshapesu = [np.unique(ss, axis=0) + for ss in lshapes] + + if return_msg is True or verb is True: + col = ['index'] + [k0 for k0 in dout.keys()] + ar = ([np.arange(nch)] + + [['{} {}'.format(tuple(v0['shapes'][ii]), 'nan') + if v0['isnan'][ii] else str(tuple(v0['shapes'][ii])) + for ii in range(nch)] + if 'shapes' in v0.keys() + else v0['value'].astype(str) for v0 in dout.values()]) + msg = self._getcharray(ar, col, msg=True) + if verb is True: + indstr = ', '.join(map(str, indchout)) + msg += "\n\n => recommended indch = [{}]".format(indstr) + print(msg) + + # ------------------ + # Return + lv = [(dout, return_dict), (indchout, return_ind), (msg, return_msg)] + lout = [vv[0] for vv in lv if vv[1] is True] + if len(lout) == 1: + return lout[0] + elif len(lout) > 1: + return lout + + @staticmethod + def _compare_indch_indchr(indch, indchr, nch, indch_auto=None): + if indch_auto is None: + indch_auto = True + if indch is None: + indch = np.arange(0, nch) + if not np.all(np.in1d(indch, indchr)): + msg = ("indch has to be changed, some data may be missing\n" + + "\t- indch: {}\n".format(indch) + + "\t- indch recommended: {}".format(indchr) + + "\n\n => check self.inspect_channels() for details") + if indch_auto is True: + indch = indchr warnings.warn(msg) else: - if indch is None: - ls = [ - 'index {} {}.shape {}'.format(ii, kk, - out[dsig[kk]][ii].shape) - for ii in range(0, len(out[dsig[kk]])) - ] - else: - ls = [ - 'index {} {}.shape {}'.format(indch[ii], kk, - out[dsig[kk]][ii].shape) - for ii in range(0, len(out[dsig[kk]])) - ] - msg = ("The following is supposed to be a np.ndarray:\n" - + " - diag: {}\n".format(ids) - + " - shortcut: {}\n".format(dsig[kk]) - + " - used as: {} input\n".format(kk) - + " Observed type: {}\n".format(type(out[dsig[kk]])) - + " Probable cause: non-uniform shape (vs channels)\n" - + " => shapes :\n " - + "\n ".join(ls) - + "\n => Solution: choose indch accordingly !") raise Exception(msg) - nchout = 0 if indch is None else len(indch) - return indch, nchout != nch + return indch def _to_Cam_Du(self, ids, lk, indch, nan=None, pos=None): Etendues, Surfaces, names = None, None, None @@ -3538,6 +3304,11 @@ def to_Cam(self, ids=None, indch=None, indch_auto=False, """ + # Check ids + idsok = set(self._lidslos).intersection(self._dids.keys()) + if ids is None and len(idsok) == 1: + ids = next(iter(idsok)) + # dsig geom = self._checkformat_Cam_geom(ids) if Name is None: @@ -3569,6 +3340,14 @@ def to_Cam(self, ids=None, indch=None, indch_auto=False, raise Exception(msg) if 'LOS' in geom: + # Check channel indices + indchr = self.inspect_channels(ids, indch=indch, + geom='only', return_ind=True, + verb=False) + indch = self._compare_indch_indchr(indch, indchr, nchMax, + indch_auto=indch_auto) + + # Load geometrical data lk = ['los_ptsRZPhi', 'etendue', 'surface', 'names'] lkok = set(self._dshort[ids].keys()) lkok = lkok.union(self._dcomp[ids].keys()) @@ -3577,16 +3356,6 @@ def to_Cam(self, ids=None, indch=None, indch_auto=False, nan=nan, pos=pos) - # Check all channels can be used, reset indch if necessary - indch, modif = self._get_indch_geomtdata(indch=indch, - indch_auto=indch_auto, - dgeom=dgeom) - if modif is True: - dgeom, Etendues, Surfaces, names = self._to_Cam_Du(ids, lk, - indch, - nan=nan, - pos=pos) - if names is not None: dchans['names'] = names @@ -3606,6 +3375,10 @@ def _checkformat_Data_dsig(self, ids=None, dsig=None, data=None, X=None, datacls=None, geomcls=None): # Check ids + idsok = set(self._lidsdiag).intersection(self._dids.keys()) + if ids is None and len(idsok) == 1: + ids = next(iter(idsok)) + if ids not in self._dids.keys(): msg = "Provided ids should be available as a self.dids.keys() !" raise Exception(msg) @@ -3775,6 +3548,11 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, return_indch = True """ + # Check ids + idsok = set(self._lidsdiag).intersection(self._dids.keys()) + if ids is None and len(idsok) == 1: + ids = next(iter(idsok)) + # dsig datacls, geomcls, dsig = self._checkformat_Data_dsig(ids, dsig, data=data, X=X, @@ -3807,8 +3585,17 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, indchanstr = self._dshort[ids][dsig['data']]['str'].index('[chan]') chanstr = self._dshort[ids][dsig['data']]['str'][:indchanstr] nchMax = len(getattr(self._dids[ids]['ids'][0], chanstr)) - dgeom = None - if geomcls != False: + + # Check channel indices + indchr = self.inspect_channels(ids, indch=indch, + geom=(geomcls is not False), + return_ind=True, + verb=False) + indch = self._compare_indch_indchr(indch, indchr, nchMax, + indch_auto=indch_auto) + + dgeom, names = None, None + if geomcls is not False: Etendues, Surfaces = None, None if config is None: msg = "A config must be provided to compute the geometry !" @@ -3833,23 +3620,16 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, msg += " - 't' = %s"%str(t) raise Exception(msg) - # ----------- - # Check indch - if type(t) is list: - indch, modif = self._get_indch_geomtdata(indch=indch, - indch_auto=indch_auto, - dgeom=dgeom, t=t) - assert modif is True - else: - indch, modif = self._get_indch_geomtdata(indch=indch, - indch_auto=indch_auto, - dgeom=dgeom) - if modif is True: - if geomcls is not False: - dgeom, Etendues, Surfaces, names = self._to_Cam_Du( - ids, lk_geom, indch, nan=nan, pos=pos) - t = self.get_data(ids, sig='t', indch=indch)['t'] - modif = False + # ---------- + # Get data + out = self.get_data(ids, sig=dsig['data'], + indch=indch, nan=nan, pos=pos) + if len(out[dsig['data']]) == 0: + msgstr = self._dshort[ids]['data']['str'] + msg = ("The data array is not available for {}:\n".format(ids) + + " - 'data' <=> {}.{}\n".format(ids, msgstr) + + " - 'data' = {}".format(out[dsig['data']])) + raise Exception(msg) if names is not None: dchans['names'] = names @@ -3865,21 +3645,9 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, out = self.get_data(ids, sig=[dsig[k] for k in lk], indt=indt, indch=indch, nan=nan, pos=pos) for kk in set(lk).difference('t'): - if not isinstance(out[dsig[kk]], np.ndarray): - indch, modifk = self._get_indch_geomtdata( - indch=indch, indch_auto=indch_auto, - out=out, dsig=dsig, kk=kk) - if modifk is True: - out = self.get_data(ids, sig=[dsig[k] for k in lk], - indt=indt, indch=indch, - nan=nan, pos=pos) - modif = True - # Arrange depending on shape and field if type(out[dsig[kk]]) is not np.ndarray: msg = "BEWARE : non-conform data !" - import ipdb - ipdb.set_trace() raise Exception(msg) if out[dsig[kk]].size == 0 or out[dsig[kk]].ndim not in [1, 2, 3]: @@ -3894,7 +3662,6 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, if out[dsig[kk]].ndim == 2: if dsig[kk] in ['X','lamb']: if np.allclose(out[dsig[kk]], out[dsig[kk]][:,0:1]): - import pdb; pdb.set_trace() # DB dins[kk] = out[dsig[kk]][:,0] else: dins[kk] = out[dsig[kk]] @@ -3905,13 +3672,6 @@ def to_Data(self, ids=None, dsig=None, data=None, X=None, tlim=None, assert kk == 'data' dins[kk] = np.swapaxes(out[dsig[kk]].T, 1,2) - # Update dgeom if necessary - if modif is True and geomcls is not False: - dgeom, Etendues, Surfaces, names = self._to_Cam_Du( - ids, lk_geom, indch, - nan=nan, pos=pos) - modif = False - # -------------------------- # Format special ids cases if ids == 'reflectometer_profile': @@ -4152,7 +3912,7 @@ def calc_signal(self, ids=None, dsig=None, tlim=None, t=None, res=None, return_indch=True, plot=False) # Get camera - cam = self.to_Cam(ids=ids, indch=indch, + cam = self.to_Cam(ids=ids, indch=indch, indch_auto=indch_auto, Name=None, occ=occ_cam, config=config, description_2d=description_2d, plot=False, nan=True, pos=None) diff --git a/tofu/imas2tofu/_def.py b/tofu/imas2tofu/_def.py new file mode 100644 index 000000000..58ac31c8c --- /dev/null +++ b/tofu/imas2tofu/_def.py @@ -0,0 +1,451 @@ +""" +That's where the default shortcuts for imas2tofu are defined +Shortcuts allow you (via the class MulTiIDSLoader) to define short str + for some important data contained in IMAS ids +The are stored as a large dict, with: + {ids: { + shortcut1: long_version1, + ... + shortcutN: long_version1} + } + +There is a default copy of this file in the tfu install, but each user can +(re)define his/her own shortcuts using a local copy that will take precedence +at import time. + +To customize your tofu install with user-specific parameters, run in terminal: + + tofu-custom + +This will create a .tofu/ in your home (~) with a local copy that you can edit +To see the shortcuts available from your running ipython console do: + + > import tof as tf + > tf.imas2tofu.MultiIDSLoader.get_shortcuts() + +Available since tofu 1.4.3 +""" + +# ############################################################################ +# +# shortcuts for imas2tofu interface (MultiIDSLoader class) +# +# ############################################################################ + +_dshort = { + 'wall': { + 'wallR': {'str': 'description_2d[0].limiter.unit[0].outline.r'}, + 'wallZ': {'str': 'description_2d[0].limiter.unit[0].outline.z'}}, + + 'pulse_schedule': { + 'events_times': {'str': 'event[].time_stamp'}, + 'events_names': {'str': 'event[].identifier'}}, + + 'equilibrium': { + 't': {'str': 'time'}, + 'ip': {'str': 'time_slice[time].global_quantities.ip', + 'dim': 'current', 'quant': 'Ip', 'units': 'A'}, + 'q0': {'str': 'time_slice[time].global_quantities.q_axis'}, + 'qmin': {'str': 'time_slice[time].global_quantities.q_min.value'}, + 'q95': {'str': 'time_slice[time].global_quantities.q_95'}, + 'volume': {'str': 'time_slice[time].global_quantities.volume', + 'dim': 'volume', 'quant': 'pvol', 'units': 'm3'}, + 'psiaxis': {'str': 'time_slice[time].global_quantities.psi_axis', + 'dim': 'B flux', 'quant': 'psi', 'units': 'Wb'}, + 'psisep': {'str': 'time_slice[time].global_quantities.psi_boundary', + 'dim': 'B flux', 'quant': 'psi', 'units': 'Wb'}, + 'BT0': {'str': ('time_slice[time].global_quantities' + + '.magnetic_axis.b_field_tor'), + 'dim': 'B', 'quant': 'BT', 'units': 'T'}, + 'axR': {'str': 'time_slice[time].global_quantities.magnetic_axis.r', + 'dim': 'distance', 'quant': 'R', 'units': 'm'}, + 'axZ': {'str': 'time_slice[time].global_quantities.magnetic_axis.z', + 'dim': 'distance', 'quant': 'Z', 'units': 'm'}, + 'x0R': {'str': 'time_slice[time].boundary.x_point[0].r'}, + 'x0Z': {'str': 'time_slice[time].boundary.x_point[0].z'}, + 'x1R': {'str': 'time_slice[time].boundary.x_point[1].r'}, + 'x1Z': {'str': 'time_slice[time].boundary.x_point[1].z'}, + 'strike0R': {'str': 'time_slice[time].boundary.strike_point[0].r'}, + 'strike0Z': {'str': 'time_slice[time].boundary.strike_point[0].z'}, + 'strike1R': {'str': 'time_slice[time].boundary.strike_point[1].r'}, + 'strike1Z': {'str': 'time_slice[time].boundary.strike_point[1].z'}, + 'sepR': {'str': 'time_slice[time].boundary_separatrix.outline.r'}, + 'sepZ': {'str': 'time_slice[time].boundary_separatrix.outline.z'}, + + '1drhotn': {'str': 'time_slice[time].profiles_1d.rho_tor_norm', + 'dim': 'rho', 'quant': 'rhotn', 'units': 'adim.'}, + '1dphi': {'str': 'time_slice[time].profiles_1d.phi', + 'dim': 'B flux', 'quant': 'phi', 'units': 'Wb'}, + '1dpsi': {'str': 'time_slice[time].profiles_1d.psi', + 'dim': 'B flux', 'quant': 'psi', 'units': 'Wb'}, + '1dq': {'str': 'time_slice[time].profiles_1d.q', + 'dim': 'safety factor', 'quant': 'q', 'units': 'adim.'}, + '1dpe': {'str': 'time_slice[time].profiles_1d.pressure', + 'dim': 'pressure', 'quant': 'pe', 'units': 'Pa'}, + '1djT': {'str': 'time_slice[time].profiles_1d.j_tor', + 'dim': 'vol. current dens.', 'quant': 'jT', 'units': 'A/m^2'}, + + '2dphi': {'str': 'time_slice[time].ggd[0].phi[0].values', + 'dim': 'B flux', 'quant': 'phi', 'units': 'Wb'}, + '2dpsi': {'str': 'time_slice[time].ggd[0].psi[0].values', + 'dim': 'B flux', 'quant': 'psi', 'units': 'Wb'}, + '2djT': {'str': 'time_slice[time].ggd[0].j_tor[0].values', + 'dim': 'vol. current dens.', 'quant': 'jT', 'units': 'A/m^2'}, + '2dBR': {'str': 'time_slice[time].ggd[0].b_field_r[0].values', + 'dim': 'B', 'quant': 'BR', 'units': 'T'}, + '2dBT': {'str': 'time_slice[time].ggd[0].b_field_tor[0].values', + 'dim': 'B', 'quant': 'BT', 'units': 'T'}, + '2dBZ': {'str': 'time_slice[time].ggd[0].b_field_z[0].values', + 'dim': 'B', 'quant': 'BZ', 'units': 'T'}, + '2dmeshNodes': {'str': ('grids_ggd[0].grid[0].space[0]' + + '.objects_per_dimension[0]' + + '.object[].geometry')}, + '2dmeshFaces': {'str': ('grids_ggd[0].grid[0].space[0]' + + '.objects_per_dimension[2]' + + '.object[].nodes')}, + '2dmeshR': {'str': 'time_slice[0].profiles_2d[0].r'}, + '2dmeshZ': {'str': 'time_slice[0].profiles_2d[0].z'}}, + + 'core_profiles': { + 't': {'str': 'time'}, + 'ip': {'str': 'global_quantities.ip', + 'dim': 'current', 'quant': 'Ip', 'units': 'A'}, + 'vloop': {'str': 'global_quantities.v_loop', + 'dim': 'voltage', 'quant': 'Vloop', 'units': 'V/m'}, + + '1dTe': {'str': 'profiles_1d[time].electrons.temperature', + 'dim': 'temperature', 'quant': 'Te', 'units': 'eV'}, + '1dne': {'str': 'profiles_1d[time].electrons.density', + 'dim': 'density', 'quant': 'ne', 'units': '/m^3'}, + '1dzeff': {'str': 'profiles_1d[time].zeff', + 'dim': 'charge', 'quant': 'zeff', 'units': 'adim.'}, + '1dphi': {'str': 'profiles_1d[time].grid.phi', + 'dim': 'B flux', 'quant': 'phi', 'units': 'Wb'}, + '1dpsi': {'str': 'profiles_1d[time].grid.psi', + 'dim': 'B flux', 'quant': 'psi', 'units': 'Wb'}, + '1drhotn': {'str': 'profiles_1d[time].grid.rho_tor_norm', + 'dim': 'rho', 'quant': 'rhotn', 'units': 'adim.'}, + '1drhopn': {'str': 'profiles_1d[time].grid.rho_pol_norm', + 'dim': 'rho', 'quant': 'rhopn', 'units': 'adim.'}, + '1dnW': {'str': 'profiles_1d[time].ions[identifier.label=W].density', + 'dim': 'density', 'quant': 'nI', 'units': '/m^3'}}, + + 'edge_profiles': { + 't': {'str': 'time'}}, + + 'core_sources': { + 't': {'str': 'time'}, + '1dpsi': {'str': ('source[identifier.name=lineradiation]' + + '.profiles_1d[time].grid.psi'), + 'dim': 'B flux', 'quant': 'psi', 'units': 'Wb'}, + '1drhotn': {'str': ('source[identifier.name=lineradiation]' + + '.profiles_1d[time].grid.rho_tor_norm'), + 'dim': 'rho', 'quant': 'rhotn', 'units': 'Wb'}, + '1dbrem': {'str': ('source[identifier.name=bremsstrahlung]' + + '.profiles_1d[time].electrons.energy'), + 'dim': 'vol.emis.', 'quant': 'brem.', 'units': 'W/m^3'}, + '1dline': {'str': ('source[identifier.name=lineradiation]' + + '.profiles_1d[time].electrons.energy'), + 'dim': 'vol. emis.', 'quant': 'lines', 'units': 'W/m^3'}}, + + 'edge_sources': { + 't': {'str': 'time'}, + '2dmeshNodes': {'str': ('grid_ggd[0].space[0].objects_per_dimension[0]' + + '.object[].geometry')}, + '2dmeshFaces': {'str': ('grid_ggd[0].space[0].objects_per_dimension[2]' + + '.object[].nodes')}, + '2dradiation': {'str': 'source[13].ggd[0].electrons.energy[0].values', + 'dim': 'vol. emis.', 'quant': 'vol.emis.', + 'name': 'tot. vol. emis.', 'units': 'W/m^3'}}, + + 'lh_antennas': { + 't': {'str': 'antenna[chan].power_launched.time'}, + 'power0': {'str': 'antenna[0].power_launched.data', + 'dim': 'power', 'quant': 'lh power', 'units': 'W', + 'pos': True}, + 'power1': {'str': 'antenna[1].power_launched.data', + 'dim': 'power', 'quant': 'lh power', 'units': 'W', + 'pos': True}, + 'power': {'str': 'antenna[chan].power_launched.data', + 'dim': 'power', 'quant': 'lh power', 'units': 'W', + 'pos': True}, + 'R': {'str': 'antenna[chan].position.r.data', + 'dim': 'distance', 'quant': 'R', 'units': 'm'}}, + + 'ic_antennas': { + 't': {'str': 'antenna[chan].module[0].power_forward.time'}, + 'power0mod_fwd': {'str': 'antenna[0].module[].power_forward.data', + 'dim': 'power', 'quant': 'ic power', 'units': 'W'}, + 'power0mod_reflect': {'str': ('antenna[0].module[]' + + '.power_reflected.data'), + 'dim': 'power', 'quant': 'ic power', + 'units': 'W'}, + 'power1mod_fwd': {'str': 'antenna[1].module[].power_forward.data', + 'dim': 'power', 'quant': 'ic power', 'units': 'W'}, + 'power1mod_reflect': {'str': ('antenna[1].module[]' + + '.power_reflected.data'), + 'dim': 'power', 'quant': 'ic power', + 'units': 'W'}, + 'power2mod_fwd': {'str': 'antenna[2].module[].power_forward.data', + 'dim': 'power', 'quant': 'ic power', 'units': 'W'}, + 'power2mod_reflect': {'str': ('antenna[2].module[]' + + '.power_reflected.data'), + 'dim': 'power', 'quant': 'ic power', + 'units': 'W'}}, + + 'magnetics': { + 't': {'str': 'time'}, + 'ip': {'str': 'method[0].ip.data'}, + 'diamagflux': {'str': 'method[0].diamagnetic_flux.data'}, + 'bpol_B': {'str': 'bpol_probe[chan].field.data', + 'dim': 'B', 'quant': 'Bpol', 'units': 'T'}, + 'bpol_name': {'str': 'bpol_probe[chan].name'}, + 'bpol_R': {'str': 'bpol_probe[chan].position.r', + 'dim': 'distance', 'quant': 'R', 'units': 'm'}, + 'bpol_Z': {'str': 'bpol_probe[chan].position.z', + 'dim': 'distance', 'quant': 'Z', 'units': 'm'}, + 'bpol_angpol': {'str': 'bpol_probe[chan].poloidal_angle', + 'dim': 'angle', 'quant': 'angle_pol', 'units': 'rad'}, + 'bpol_angtor': {'str': 'bpol_probe[chan].toroidal_angle', + 'dim': 'angle', 'quant': 'angle_tor', 'units': 'rad'}, + 'floop_flux': {'str': 'flux_loop[chan].flux.data', + 'dim': 'B flux', 'quant': 'B flux', 'units': 'Wb'}, + 'floop_name': {'str': 'flux_loop[chan].name'}, + 'floop_R': {'str': 'flux_loop[chan].position.r', + 'dim': 'distance', 'quant': 'R', 'units': 'm'}, + 'floop_Z': {'str': 'flux_loop[chan].position.z', + 'dim': 'distance', 'quant': 'Z', 'units': 'm'}}, + + 'barometry': { + 't': {'str': 'gauge[chan].pressure.time'}, + 'names': {'str': 'gauge[chan].name'}, + 'p': {'str': 'gauge[chan].pressure.data', + 'dim': 'pressure', 'quant': 'p', 'units': 'Pa?'}}, + + 'calorimetry': { + 't': {'str': 'group[chan].component[0].power.time'}, + 'names': {'str': 'group[chan].name'}, + 'power': {'str': 'group[chan].component[0].power.data', + 'dim': 'power', 'quant': 'extracted power', + 'units': 'W'}}, + + 'neutron_diagnostic': { + 't': {'str': 'time', 'units': 's'}, + 'flux_total': {'str': 'synthetic_signals.total_neutron_flux', + 'dim': 'particle flux', 'quant': 'particle flux', + 'units': 'Hz'}}, + + 'ece': { + 't': {'str': 'time', + 'quant': 't', 'units': 's'}, + 'freq': {'str': 'channel[chan].frequency.data', + 'dim': 'freq', 'quant': 'freq', 'units': 'Hz'}, + 'Te': {'str': 'channel[chan].t_e.data', + 'dim': 'temperature', 'quant': 'Te', 'units': 'eV'}, + 'R': {'str': 'channel[chan].position.r', + 'dim': 'distance', 'quant': 'R', 'units': 'm'}, + 'rhotn': {'str': 'channel[chan].position.rho_tor_norm', + 'dim': 'rho', 'quant': 'rhotn', 'units': 'adim.'}, + 'theta': {'str': 'channel[chan].position.theta', + 'dim': 'angle', 'quant': 'theta', 'units': 'rad.'}, + 'tau1keV': {'str': 'channel[chan].optical_depth.data', + 'dim': 'optical_depth', 'quant': 'tau', 'units': 'adim.'}, + 'validity_timed': {'str': 'channel[chan].t_e.validity_timed'}, + 'names': {'str': 'channel[chan].name'}, + 'Te0': {'str': 't_e_central.data', + 'dim': 'temperature', 'quant': 'Te', 'units': 'eV'}}, + + 'reflectometer_profile': { + 't': {'str': 'time'}, + 'ne': {'str': 'channel[chan].n_e.data', + 'dim': 'density', 'quant': 'ne', 'units': '/m^3'}, + 'R': {'str': 'channel[chan].position.r', + 'dim': 'distance', 'quant': 'R', 'units': 'm'}, + 'Z': {'str': 'channel[chan].position.z', + 'dim': 'distance', 'quant': 'Z', 'units': 'm'}, + 'phi': {'str': 'channel[chan].position.phi', + 'dim': 'angle', 'quant': 'phi', 'units': 'rad'}, + 'names': {'str': 'channel[chan].name'}, + 'mode': {'str': 'mode'}, + 'sweep': {'str': 'sweep_time'}}, + + 'interferometer': { + 't': {'str': 'time', + 'quant': 't', 'units': 's'}, + 'names': {'str': 'channel[chan].name'}, + 'ne_integ': {'str': 'channel[chan].n_e_line.data', + 'dim': 'ne_integ', 'quant': 'ne_integ', + 'units': '/m2', 'Brightness': True}}, + + 'polarimeter': { + 't': {'str': 'time', + 'quant': 't', 'units': 's'}, + 'lamb': {'str': 'channel[chan].wavelength', + 'dim': 'distance', 'quant': 'wavelength', + 'units': 'm'}, + 'fangle': {'str': 'channel[chan].faraday_angle.data', + 'dim': 'angle', 'quant': 'faraday angle', + 'units': 'rad', 'Brightness': True}, + 'names': {'str': 'channel[chan].name'}}, + + 'bolometer': { + 't': {'str': 'channel[chan].power.time', + 'quant': 't', 'units': 's'}, + 'power': {'str': 'channel[chan].power.data', + 'dim': 'power', 'quant': 'power radiative', + 'units': 'W', 'Brightness': False}, + 'etendue': {'str': 'channel[chan].etendue', + 'dim': 'etendue', 'quant': 'etendue', + 'units': 'm2.sr'}, + 'names': {'str': 'channel[chan].name'}, + 'tpower': {'str': 'time', 'quant': 't', 'units': 's'}, + 'prad': {'str': 'power_radiated_total', + 'dim': 'power', 'quant': 'power radiative', + 'units': 'W'}, + 'pradbulk': {'str': 'power_radiated_inside_lcfs', + 'dim': 'power', 'quant': 'power radiative', + 'units': 'W'}}, + + 'soft_x_rays': { + 't': {'str': 'time', + 'quant': 't', 'units': 's'}, + 'power': {'str': 'channel[chan].power.data', + 'dim': 'power', 'quant': 'power radiative', + 'units': 'W', 'Brightness': False}, + 'brightness': {'str': 'channel[chan].brightness.data', + 'dim': 'brightness', 'quant': 'brightness', + 'units': 'W/(m2.sr)', 'Brightness': True}, + 'names': {'str': 'channel[chan].name'}, + 'etendue': {'str': 'channel[chan].etendue', + 'dim': 'etendue', 'quant': 'etendue', + 'units': 'm2.sr'}}, + + 'spectrometer_visible': { + 't': {'str': ('channel[chan].grating_spectrometer' + + '.radiance_spectral.time'), + 'quant': 't', 'units': 's'}, + 'spectra': {'str': ('channel[chan].grating_spectrometer' + + '.radiance_spectral.data'), + 'dim': 'radiance_spectral', + 'quant': 'radiance_spectral', + 'units': 'ph/s/(m2.sr)/m', 'Brightness': True}, + 'names': {'str': 'channel[chan].name'}, + 'lamb': {'str': 'channel[chan].grating_spectrometer.wavelengths', + 'dim': 'wavelength', 'quant': 'wavelength', 'units': 'm'}}, + + 'bremsstrahlung_visible': { + 't': {'str': 'time', + 'quant': 't', 'units': 's'}, + 'radiance': {'str': 'channel[chan].radiance_spectral.data', + 'dim': 'radiance_spectral', + 'quant': 'radiance_spectral', + 'units': 'ph/s/(m2.sr)/m', + 'Brightness': True}, + 'names': {'str': 'channel[chan].name'}, + 'lamb_up': {'str': 'channel[chan].filter.wavelength_upper'}, + 'lamb_lo': {'str': 'channel[chan].filter.wavelength_lower'}} + } + + +# ############################################################################ +# +# default data for each ids (not used yet) +# +# ############################################################################ + + +_didsdiag = { + 'lh_antennas': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'data': 'power', + 't': 't'}}, + 'ic_antennas': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'data': 'power', + 't': 't'}}, + 'magnetics': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'data': 'bpol_B', + 't': 't'}}, + 'barometry': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'data': 'p', + 't': 't'}}, + 'calorimetry': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'data': 'power', + 't': 't'}}, + 'ece': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'t': 't', + 'X': 'rhotn_sign', + 'data': 'Te'}}, + 'neutron_diagnostic': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'t': 't', + 'data': 'flux_total'}}, + 'reflectometer_profile': {'datacls': 'DataCam1D', + 'geomcls': False, + 'sig': {'t': 't', + 'X': 'R', + 'data': 'ne'}}, + 'interferometer': {'datacls': 'DataCam1D', + 'geomcls': 'CamLOS1D', + 'sig': {'t': 't', + 'data': 'ne_integ'}, + 'synth': {'dsynth': { + 'quant': 'core_profiles.1dne', + 'ref1d': 'core_profiles.1drhotn', + 'ref2d': 'equilibrium.2drhotn'}, + 'dsig': {'core_profiles': ['t'], + 'equilibrium': ['t']}, + 'Brightness': True}}, + 'polarimeter': {'datacls': 'DataCam1D', + 'geomcls': 'CamLOS1D', + 'sig': {'t': 't', + 'data': 'fangle'}, + 'synth': {'dsynth': { + 'fargs': ['core_profiles.1dne', + 'equilibrium.2dBR', + 'equilibrium.2dBT', + 'equilibrium.2dBZ', + 'core_profiles.1drhotn', + 'equilibrium.2drhotn']}, + 'dsig': {'core_profiles': ['t'], + 'equilibrium': ['t']}, + 'Brightness': True}}, + 'bolometer': {'datacls': 'DataCam1D', + 'geomcls': 'CamLOS1D', + 'sig': {'t': 't', + 'data': 'power'}, + 'synth': {'dsynth': { + 'quant': 'core_sources.1dprad', + 'ref1d': 'core_sources.1drhotn', + 'ref2d': 'equilibrium.2drhotn'}, + 'dsig': {'core_sources': ['t'], + 'equilibrium': ['t']}, + 'Brightness': True}}, + 'soft_x_rays': {'datacls': 'DataCam1D', + 'geomcls': 'CamLOS1D', + 'sig': {'t': 't', + 'data': 'power'}}, + 'spectrometer_visible': {'datacls': 'DataCam1DSpectral', + 'geomcls': 'CamLOS1D', + 'sig': {'data': 'spectra', + 't': 't', + 'lamb': 'lamb'}}, + 'bremsstrahlung_visible': { + 'datacls': 'DataCam1D', + 'geomcls': 'CamLOS1D', + 'sig': {'t': 't', + 'data': 'radiance'}, + 'synth': { + 'dsynth': { + 'quant': ['core_profiles.1dTe', + 'core_profiles.1dne', + 'core_profiles.1dzeff'], + 'ref1d': 'core_profiles.1drhotn', + 'ref2d': 'equilibrium.2drhotn'}, + 'dsig': {'core_profiles': ['t'], + 'equilibrium': ['t']}, + 'Brightness': True}}} diff --git a/tofu/imas2tofu/_mat2ids2calc.py b/tofu/imas2tofu/_mat2ids2calc.py new file mode 100644 index 000000000..1f490b813 --- /dev/null +++ b/tofu/imas2tofu/_mat2ids2calc.py @@ -0,0 +1,205 @@ + +# Built-in +import os + +# Common +import scipy.io as scpio +import numpy as np + +# tofu-specific +from .. import _physics + + +__all__ = ['get_data_from_matids'] +_LIDSOK = ['core_profiles'] +_DRETURN = {'core_profiles': ['rhotn', 'ne', 'Te', 'zeff', 't', 'brem']} +_MSG0 = ("The input file structure is not as expected !\n" + + " => Maybe file structure changed ?\n" + + " => Maybe corrupted data ?\n") + + +# #################################################### +# Utility +# #################################################### + +def _get_indtlim(t, tlim=None, shot=None, out=bool): + """ """ + c0 = tlim is None + c1 = type(tlim) in [list, tuple, np.ndarray] + assert c0 or c1 + assert type(t) is np.ndarray + + if c0: + tlim = [-np.inf, np.inf] + else: + assert len(tlim) == 2 + ls = [int, float, np.int64, np.float64] # , str + assert all([tt is None or type(tt) in ls for tt in tlim]) + tlim = list(tlim) + for (ii, sgn) in [(0, -1.), (1, 1.)]: + if tlim[ii] is None: + tlim[ii] = sgn*np.inf + # elif type(tlim[ii]) is str and 'ign' in tlim[ii].lower(): + # tlim[ii] = get_t0(shot) + + assert tlim[0] < tlim[1] + indt = (t >= tlim[0]) & (t <= tlim[1]) + if out is int: + indt = indt.nonzero()[0] + return indt + + +# #################################################### +# Main function +# #################################################### + +def get_data_from_matids(input_pfe=None, tlim=None, + return_fields=None, lamb=None): + """ Extract tofu-compatible from an ids saved as a mat file + + Assumes that the mat file contains the ids data + Only the following ids are handled: + {} + + """.format(_LIDSOK) + + # --------------- + # Check + if not os.path.isfile(input_pfe): + msg = ("Provided file does not seem to exist:\n" + + "\t- {}".format(input_pfe)) + raise Exception(msg) + lc = [return_fields is None, + isinstance(return_fields, str), + isinstance(return_fields, list) + and all([isinstance(ss, str) for ss in return_fields])] + if not any(lc): + msg = "return_fields must be a str or a list of str " + raise Exception(msg) + if lc[1]: + return_fields = [return_fields] + + # --------------- + # Load and check / extract ids + mat = scpio.loadmat(input_pfe) + ids = [k0 for k0 in mat.keys() if '__' not in k0] + if len(ids) != 1 or ids[0] not in _LIDSOK: + msg = ("The file does not seem to contain a known ids:\n" + + "\t- file: {}\n".format(input_pfe) + + "\t- keys: {}\n".format(sorted(mat.keys())) + + "\t- known ids: {}".format(_LIDSOK)) + raise Exception(msg) + ids = ids[0] + data = mat[ids] + + if return_fields is None: + return_fields = _DRETURN[ids] + notok = [ss for ss in return_fields if ss not in _DRETURN[ids]] + if len(notok) > 0: + msg = ("Some requested fields are not available:\n" + + "\t- requested: {}\n".format(notok) + + "\t- available: {}".format(_DRETURN[ids])) + raise Exception(msg) + + # --------------- + # Get inside ids and extract data + if ids == 'core_profiles': + # --------------- + # Check expected structure + if not (data.shape == (1, 1) and data[0, 0].size == 1): + msg = ("\t{}.shape = {}\n".format(ids, data.shape) + + "\t{}.size = {}".format(ids, data[0, 0].size)) + raise Exception(_MSG0 + msg) + data = data[0, 0].tolist() + if not (isinstance(data, tuple) and len(data) == 6): + msg = ("\ttype({}[0, 0].tolist()) = {}\n".format(ids, type(data)) + + "\tlen({}[0, 0].tolist()) = {}".format(ids, len(data))) + raise Exception(_MSG0 + msg) + + ls = [pp.shape for pp in data] + c0 = [len(ss) == 2 for ss in ls] + c1 = np.sum([ss == (1, 1) for ss in ls]) == 4 + c2 = np.sum([(ss[1] == 1 and ss[0] > ss[1]) for ss in ls]) == 1 + c3 = np.sum([(ss[0] == 1 and ss[1] > ss[0]) for ss in ls]) == 1 + if c0 and c1 and c2 and c3: + indt = [ii for ii in range(len(ls)) if ls[ii][0] > ls[ii][1]] + indp = [ii for ii in range(len(ls)) if ls[ii][0] < ls[ii][1]] + else: + if np.sum([ss == (1, 1) for ss in ls]) == 6: + warnings.warn("There seems to be only one time step...") + indt = [ii for ii in range(len(ls)) if data[ii].dtype == ' download the data with tf.openadas2tofu.download()") + raise Exception(msg) + return os.path.join(path, ld[0]) + + +# ############################################################################# +# Main functions +# ############################################################################# + + +def read(adas_path, **kwdargs): + """ Read openadas-formatted files and return a dict with the data + + Povide the full adas file name + The result is returned as a dict + + example + ------- + >>> import tofu as tf + >>> fn = '/adf11/scd74/scd74_ar.dat' + >>> out = tf.openadas2tofu.read(fn) + >>> fn = '/adf15/pec40][ar/pec40][ar_ca][ar16.dat' + >>> out = tf.openadas2tofu.read(fn) + """ + + path_local = _get_PATH_LOCAL() + + # Check whether the local .tofu repo exists, if not recommend tofu-custom + if path_local is None: + path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + msg = ("You do not seem to have a local ./tofu repository\n" + + "tofu uses that local repository to store all user-specific " + + "data and downloads\n" + + "In particular, openadas files are downloaded and saved in:\n" + + "\t{}\n".format(path) + + " => to set-up your local .tofu repo, run in a terminal:\n" + + "\ttofu-custom") + raise Exception(msg) + + # Determine whether adas_path is an absolute path or an adas full name + if os.path.isfile(adas_path): + pfe = adas_path + else: + # make sure adas_path is not understood as absolute local path + if adas_path[0] == '/': + adas_path = adas_path[1:] + + # Check file was downloaded locally + pfe = os.path.join(path_local, adas_path) + if not os.path.isfile(pfe): + msg = ("Provided file does not seem to exist:\n" + + "\t{}\n".format(pfe) + + " => Search it online with tofu.openadas2tofu.search()\n" + + " => Download it with tofu.openadas2tofu.download()") + raise FileNotFoundError(msg) + + lc = [ss for ss in _DTYPES.keys() if ss in pfe] + if not len(lc) == 1: + msg = ("File type could not be derived from absolute path:\n" + + "\t- provided: {}\n".format(pfe) + + "\t- supported: {}".format(sorted(_DTYPES.keys()))) + raise Exception(msg) + + func = eval('_read_{}'.format(lc[0])) + return func(pfe, **kwdargs) + + +def read_all(element=None, charge=None, typ1=None, typ2=None, + verb=None, **kwdargs): + """ Read all relevant openadas files for chosen typ1 + + Please specify: + - typ1: 'adf11' or 'adf15' + - element: the symbol of the element + + If typ1 = 'adf11', you can also provide typ2 to specify the coefficients: + - 'scd': effective ionisation coefficients + - 'acd': effective electron-impact recombination coefficients + - 'ccd': effective hydrogen-impact recombination coefficients + - 'plt': line power due to electron-impact excitation + - 'prc': line power due to hydrogen-impact excitation + - 'prb': rad. recombination and bremmstrahlung due to electron-impact + + If typ1 = 'adf15', you can optioanlly provide a min/max wavelength + + The result is returned as a dict + + examples + -------- + >>> import tofu as tf + >>> dout = tf.openadas2tofu.read_all(element='ar', typ1='adf11') + >>> dout = tf.openadas2tofu.read_all(element='ar', typ1='adf15', + charge=16, + lambmin=3.94e-10, lambmax=4.e-10) + """ + + # -------------------- + # Check whether the local .tofu repo exists, if not recommend tofu-custom + path_local = _get_PATH_LOCAL() + if path_local is None: + path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + msg = ("You do not seem to have a local ./tofu repository\n" + + "tofu uses that local repository to store all user-specific " + + "data and downloads\n" + + "In particular, openadas files are downloaded and saved in:\n" + + "\t{}\n".format(path) + + " => to set-up your local .tofu repo, run in a terminal:\n" + + "\ttofu-custom") + raise Exception(msg) + + # -------------------- + # Check / format input + if typ1 is None: + typ1 = 'adf15' + if not isinstance(typ1, str) or typ1.lower() not in _DTYPES.keys(): + msg = ("Please choose a valid adas file type:\n" + + "\t- allowed: {}\n".format(_DTYPES.keys()) + + "\t- provided: {}".format(typ1)) + raise Exception(msg) + typ1 = typ1.lower() + if typ1 == 'adf11' and typ2 is None: + typ2 = _DTYPES[typ1] + fd = os.listdir(os.path.join(path_local, typ1)) + typ2 = [sorted([ss for ss in fd if tt in ss])[-1] for tt in typ2] + if isinstance(typ2, str): + typ2 = [typ2] + if (_DTYPES[typ1] is not None + and (not isinstance(typ2, list) + or not all([any([s1 in ss for s1 in _DTYPES[typ1]]) + for ss in typ2]))): + msg = ("typ2 must be a list of valid openadas file types for typ1:\n" + + "\t- provided: {}\n".format(typ2) + + "\t- available for {}: {}".format(typ1, _DTYPES[typ1])) + raise Exception(msg) + + if not isinstance(element, str): + msg = "Please choose an element!" + raise Exception(msg) + element = element.lower() + if charge is not None and not isinstance(charge, int): + msg = "charge must be a int!" + raise Exception(msg) + + if verb is None: + verb = True + + # -------------------- + # Get list of relevant directories + + # Level 1: Type + path = _get_subdir_from_pattern(path_local, typ1) + # Level 2: element or typ2 + if typ1 == 'adf11': + lpath = [_get_subdir_from_pattern(path, tt) for tt in typ2] + elif typ1 == 'adf15': + lpath = [_get_subdir_from_pattern(path, element)] + + # -------------------- + # Get list of relevant files pfe + lpfe = list(itt.chain.from_iterable( + [[os.path.join(path, ff) for ff in os.listdir(path) + if (os.path.isfile(os.path.join(path, ff)) + and ff[-4:] == '.dat' + and element in ff)] + for path in lpath])) + + if charge is not None and typ1 == 'adf15': + lpfe = [ff for ff in lpfe if str(charge) in ff] + + # -------------------- + # Extract data from each file + func = eval('_read_{}'.format(typ1)) + dout = {} + for pfe in lpfe: + if verb is True: + msg = "\tLoading data from {}".format(pfe) + print(msg) + out = func(pfe, dout=dout, **kwdargs) + return out + + +# ############################################################################# +# Specialized functions for ADF 11 +# ############################################################################# + + +def _read_adf11(pfe, deg=None, dout=None): + if deg is None: + deg = _DEG + if dout is None: + dout = {} + + # Get second order file type + typ1 = [vv for vv in _DTYPES['adf11'] if vv in pfe] + if len(typ1) != 1: + msg = ("Second order file type culd not be inferred from file name!\n" + + "\t- available: {}\n".format(_DTYPES['adf11']) + + "\t- provided: {}".format(pfe)) + raise Exception(msg) + typ1 = typ1[0] + + # Get element + elem = pfe[:-4].split('_')[1] + comline = '-'*60 + comline2 = 'C'+comline + + if typ1 in ['acd', 'ccd', 'scd', 'plt', 'prb']: + + # read blocks + with open(pfe) as search: + for ii, line in enumerate(search): + + if comline2 in line: + break + + # Get atomic number (transitions) stored in this file + if ii == 0: + lstr = line.split('/') + lin = [ss for ss in lstr[0].strip().split(' ') + if ss.strip() != ''] + lc = [len(lin) == 5 and all([ss.isdigit() for ss in lin]), + elem.upper() in lstr[1], + 'ADF11' in lstr[2]] + if not all(lc): + msg = ("File header format seems to have changed!\n" + + "\t- lc = {}".format(lc)) + raise Exception(msg) + Z, nne, nte, q0, qend = map(int, lin) + nelog10 = np.array([]) + telog10 = np.array([]) + in_ne = True + continue + + if comline in line: + continue + + # Get nelog10 + if in_ne: + li = [ss for ss in line.strip().split(' ') + if ss.strip() != ''] + nelog10 = np.append(nelog10, np.array(li, dtype=float)) + if nelog10.size == nne: + in_ne = False + in_te = True + + # Get telog10 + elif in_te is True: + li = [ss for ss in line.strip().split(' ') + if ss.strip() != ''] + telog10 = np.append(telog10, np.array(li, dtype=float)) + if telog10.size == nte: + in_te = False + in_ion = True + + # Get ion block + elif (in_ion is True and 'Z1=' in line + and ('------/' in line and 'DATE=' in line)): + nion = int( + line[line.index('Z1=')+len('Z1='):].split('/')[0]) + if typ1 in ['scd', 'plt']: + charge = nion - 1 + else: + charge = nion + coefslog10 = np.array([]) + elif in_ion is True and charge is not None: + li = [ss for ss in line.strip().split(' ') + if ss.strip() != ''] + coefslog10 = np.append(coefslog10, + np.array(li, dtype=float)) + if coefslog10.size == nne*nte: + key = '{}{}'.format(elem, charge) + tkv = [('element', elem), ('Z', Z), ('charge', charge)] + if key in dout.keys(): + assert all([dout[key][ss] == vv for ss, vv in tkv]) + else: + dout[key] = {ss: vv for ss, vv in tkv} + if typ1 == 'scd': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10-6 to convert cm3/s -> m3/s + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))-6, + kx=deg, ky=deg) + dout[key]['ionis'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(m3/s)', + 'source': pfe} + elif typ1 == 'acd': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10-6 to convert cm3/s -> m3/s + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))-6, + kx=deg, ky=deg) + dout[key]['recomb'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(m3/s)', + 'source': pfe} + elif typ1 == 'ccd': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10-6 to convert cm3/s -> m3/s + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))-6, + kx=deg, ky=deg) + dout[key]['recomb_ce'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(m3/s)', + 'source': pfe} + elif typ1 == 'plt': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10+6 to convert W.cm3 -> W.m3 + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))+6, + kx=deg, ky=deg) + dout[key]['rad_bb'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(W.m3)', + 'source': pfe} + elif typ1 == 'prb': + # nelog10+6 to convert /cm3 -> /m3 + # coefslog10+6 to convert W.cm3 -> W.m3 + func = scpRectSpl(nelog10+6, telog10, + coefslog10.reshape((nne, nte))+6, + kx=deg, ky=deg) + dout[key]['rad_fffb'] = {'func': func, + 'type': 'log10_nete', + 'units': 'log10(W.m3)', + 'source': pfe} + if nion == Z: + break + + return dout + + +# ############################################################################# +# Specialized functions for ADF 15 +# ############################################################################# + + +def _get_adf15_key(elem, charge, isoel, typ0, typ1): + return '{}{}_{}_openadas_{}_{}'.format(elem, charge, isoel, + typ0, typ1) + + +def _read_adf15(pfe, dout=None, + lambmin=None, + lambmax=None, + deg=None): + + if deg is None: + deg = _DEG + if dout is None: + dout = {} + + # Get summary of transitions + flagblock = '/isel =' + flag0 = 'superstage partition information' + + # Get file markers from name (elem, charge, typ0, typ1) + typ0, typ1, elemq = pfe.split('][')[1:] + ind = re.search(r'\d', elemq).start() + elem = elemq[:ind].title() + charge = int(elemq[ind:-4]) + assert elem.lower() in typ0[:2] + assert elem.lower() == typ1.split('_')[0] + typ0 = typ0[len(elem)+1:] + typ1 = typ1.split('_')[1] + + # Extract data from file + nlines, nblock = None, 0 + in_ne, in_te, in_pec, in_tab, itab = False, False, False, False, np.inf + skip = False + with open(pfe) as search: + for ii, line in enumerate(search): + + # Get number of lines (transitions) stored in this file + if ii == 0: + lstr = line.split('/') + nlines = int(lstr[0].replace(' ', '')) + continue + + # Get info about the transition being scanned (block) + if flagblock in line and 'C' not in line and nblock < nlines: + lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] + lamb = float(lstr[0])*1.e-10 + isoel = nblock + 1 + nblock += 1 + c0 = ((lambmin is not None and lamb < lambmin) + or (lambmax is not None and lamb > lambmax)) + if c0: + skip = True + continue + skip = False + nne, nte = int(lstr[1]), int(lstr[2]) + typ = [ss[ss.index('type=')+len('type='):ss.index('/ispb')] + for ss in lstr[3:] if 'type=' in ss] + assert len(typ) == 1 + # To be updated : proper rezading from line + in_ne = True + ne = np.array([]) + te = np.array([]) + pec = np.full((nne*nte,), np.nan) + ind = 0 + continue + + if 'root partition information' in line and skip is True: + skip = False + + # Check lamb is ok + if skip is True: + continue + + # Get ne for the transition being scanned (block) + if in_ne is True: + ne = np.append(ne, + np.array(line.rstrip().strip().split(' '), + dtype=float)) + if ne.size == nne: + in_ne = False + in_te = True + + # Get te for the transition being scanned (block) + elif in_te is True: + te = np.append(te, + np.array(line.rstrip().strip().split(' '), + dtype=float)) + if te.size == nte: + in_te = False + in_pec = True + + # Get pec for the transition being scanned (block) + elif in_pec is True: + data = np.array(line.rstrip().strip().split(' '), + dtype=float) + pec[ind:ind+data.size] = data + ind += data.size + if ind == pec.size: + in_pec = False + key = _get_adf15_key(elem, charge, isoel, typ0, typ1) + # log(ne)+6 to convert /cm3 -> /m3 + # log(pec)+6 to convert cm3/s -> m3/s + func = scpRectSpl(np.log(ne)+6, np.log(te), + np.log(pec).reshape((nne, nte))+6, + kx=deg, ky=deg) + dout[key] = {'lambda': lamb, + 'ION': '{}{}+'.format(elem, charge), + 'symbol': '{}{}-{}'.format(typ0, typ1, isoel), + 'origin': pfe, + 'type': typ[0], + 'ne': ne, 'te': te, + 'pec': {'func': func, + 'type': 'log_nete', + 'units': 'log(m3/s)', + 'source': 'pfe'}} + + # Get transitions from table at the end + if 'photon emissivity atomic transitions' in line: + itab = ii + 6 + if ii == itab: + in_tab = True + if in_tab is True: + lstr = [kk for kk in line.rstrip().split(' ') if len(kk) > 0] + isoel = int(lstr[1]) + lamb = float(lstr[2])*1.e-10 + key = _get_adf15_key(elem, charge, isoel, typ0, typ1) + c0 = ((lambmin is None or lambmin < lamb) + and (lambmax is None or lambmax > lamb)) + if c0 and key not in dout.keys(): + msg = ("Inconsistency in file {}:\n".format(pfe) + + "\t- line should be present".format(key)) + raise Exception(msg) + if key in dout.keys(): + if dout[key]['lambda'] != lamb: + msg = "Inconsistency in file {}".format(pfe) + raise Exception(msg) + c0 = (dout[key]['type'] not in lstr + or lstr.index(dout[key]['type']) < 4) + if c0: + msg = ("Inconsistency in table, type not found:\n" + + "\t- expected: {}\n".format(dout[key]['type']) + + "\t- line: {}".format(line)) + raise Exception(msg) + trans = lstr[3:lstr.index(dout[key]['type'])] + dout[key]['transition'] = ''.join(trans) + if isoel == nlines: + in_tab = False + assert all(['transition' in vv.keys() for vv in dout.values()]) + return dout diff --git a/tofu/openadas2tofu/_requests.py b/tofu/openadas2tofu/_requests.py new file mode 100644 index 000000000..ab1f24ebb --- /dev/null +++ b/tofu/openadas2tofu/_requests.py @@ -0,0 +1,486 @@ + +# Built-in +import os +import shutil +import requests +import warnings + +# Common +import numpy as np + + +__all__ = ['search_online', 'search_online_by_wavelengthA', + 'download', 'download_all', 'clean_downloads'] + + +# Check whether a local .tofu/ repo exists +_URL = 'https://open.adas.ac.uk' +_URL_SEARCH = _URL + '/freeform?searchstring=' +_URL_SEARCH_WAVL = _URL + '/wavelength?' +_URL_ADF15 = _URL + '/adf15' +_URL_DOWNLOAD = _URL + '/download' + +_INCLUDE_PARTIAL = True + + +# ############################################################################# +# Utility functions +# ############################################################################# + + +def _get_PATH_LOCAL(): + pfe = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + if os.path.isdir(pfe): + return pfe + else: + return None + + +def _getcharray(ar, col=None, sep=' ', line='-', just='l', + verb=True, returnas=str): + """ Format and return char array (for pretty printing) """ + c0 = ar is None or len(ar) == 0 + if c0: + return '' + ar = np.array(ar, dtype='U') + + if ar.ndim == 1: + ar = ar.reshape((1, ar.size)) + + # Get just len + nn = np.char.str_len(ar).max(axis=0) + if col is not None: + if len(col) not in ar.shape: + msg = ("len(col) should be in np.array(ar, dtype='U').shape:\n" + + "\t- len(col) = {}\n".format(len(col)) + + "\t- ar.shape = {}".format(ar.shape)) + raise Exception(msg) + if len(col) != ar.shape[1]: + ar = ar.T + nn = np.char.str_len(ar).max(axis=0) + nn = np.fmax(nn, [len(cc) for cc in col]) + + # Apply to array + fjust = np.char.ljust if just == 'l' else np.char.rjust + out = np.array([sep.join(v) for v in fjust(ar, nn)]) + + # Apply to col + if col is not None: + arcol = np.array([col, [line*n for n in nn]], dtype='U') + arcol = np.array([sep.join(v) for v in fjust(arcol, nn)]) + out = np.append(arcol, out) + + if verb is True: + print('\n'.join(out)) + if returnas is str: + return '\n'.join(out) + elif returnas is np.ndarray: + return ar + + +# ############################################################################# +# online search +# ############################################################################# + + +class FileAlreayExistsException(Exception): + pass + + +def search_online(searchstr=None, returnas=None, + include_partial=None, verb=None): + """ Perform an online freeform search on https://open.adas.ac.uk + + Pass searchstr to the online freeform search + Prints the results (if verb=True) + Optionally return the result as: + - a char array (returnas = np.ndarray) + - a formatted str (returnas = str) + + example + ------- + >>> import tofu as tf + >>> tf.openadas2tofu.search_online('ar+16 ADF15') + """ + + # Check input + if returnas is None: + returnas = False + if verb is None: + verb = True + if include_partial is None: + include_partial = _INCLUDE_PARTIAL + if searchstr is None: + searchstr = '' + searchurl = '+'.join([requests.utils.quote(kk) + for kk in searchstr.split(' ')]) + + total_url = '{}{}&{}'.format(_URL_SEARCH, searchurl, 'searching=1') + resp = requests.get(total_url) + + # Extract response from html + out = resp.text.split('\n') + flag0 = '' + flag1 = '
' + ind0 = [ii for ii, vv in enumerate(out) if flag0 in vv] + ind1 = [ii for ii, vv in enumerate(out) if flag1 in vv] + if len(ind0) != 1 or len(ind1) == 0: + msg = ("Format of html response seems to have changed!\n" + + "Cannot find flags:\n" + + "\t- {}\n".format(flag0) + + "\t- {}\n".format(flag1) + + "in requests.get({}).text".format(total_url)) + raise Exception(msg) + ind1 = np.min([ii for ii in ind1 if ii > ind0[0]]) + out = out[ind0[0] + 1:ind1-1] + nresults = len(out) - 1 + + # Get columns + heads = [str.replace(kk.replace('', '').replace('', ''), + '', '').replace('', '') + for kk in out[0].split('')] + nhead = len(heads) + + # Get results + lout = [] + for ii in range(0, nresults): + if 'Partial results are listed below' in out[ii+1]: + if include_partial is False: + break + lout.append(['-', '-', '----- (partial results) -----', '-']) + ind = out[ii+1].index('below') + len('below') + out[ii+1] = out[ii+1][ind:] + lstri = out[ii+1].split('', '') + if '' in elmq: + elm = elmq[:elmq.index('')] + charge = elmq[elmq.index('')+len(''):] + charge = charge.replace('', '') + if charge == '+': + charge = '1+' + else: + charge = '' + elm = elmq + typ = lstri[2][1:] + fil = lstri[3][lstri[3].index('detail')+len('detail'):] + fil = fil[:fil.index('.dat')+len('.dat')] + lout.append([elm, charge, typ, fil]) + + # Format output + char = np.array(lout) + col = ['Element', 'charge', 'type of data', 'full file name'] + arr = _getcharray(char, col=col, + sep=' ', line='-', just='l', + returnas=returnas, verb=verb) + return arr + + +def search_online_by_wavelengthA(lambmin=None, lambmax=None, resolveby=None, + element=None, charge=None, + returnas=None, verb=None): + """ Perform an online search by wavelength on https://open.adas.ac.uk + + Pass the min / max wavelength (in Angstrom) to the online wavelength search + Prints the results (if verb=True) + Optionally return the result as: + - a char array (returnas = np.ndarray) + - a formatted str (returnas = str) + + The result can be resolve by transition or by adas file + Optionally filter by element to return only the results of one element + + example + ------- + >>> import tofu as tf + >>> tf.openadas2tofu.search_online_by_wavelengthA(3., 4., element='ar') + """ + + # Check input + if returnas is None: + returnas = False + if verb is None: + verb = True + if lambmin is None: + lambmin = '' + if lambmax is None: + lambmax = '' + if resolveby is None: + resolveby = 'transition' + if resolveby not in ['transition', 'file']: + msg = ("Arg resolveeby must be:\n" + + "\t- 'transition': list all available transitions\n" + + "\t- 'file': list all files containing relevant transitions") + raise Exception(msg) + if element is not None and not isinstance(element, str): + msg = ("Arg element must be a str (e.g.: element='ar')\n" + + "\t- provided: {}".format(element)) + raise Exception(msg) + if charge is not None: + if not isinstance(charge, int): + msg = ("Arg charge must be a int!\n" + + "\t- provided: {}".format(charge)) + raise Exception(msg) + charge = '0' if charge == 0 else '{}+'.format(charge) + + searchurl = '&'.join(['wave_min={}'.format(lambmin), + 'wave_max={}'.format(lambmax), + 'resolveby={}'.format(resolveby)]) + + total_url = '{}{}&{}'.format(_URL_SEARCH_WAVL, searchurl, 'searching=1') + resp = requests.get(total_url) + + # Extract response from html + out = resp.text.split('\n') + flag0 = '' + flag1 = '
' + ind0 = [ii for ii, vv in enumerate(out) if flag0 in vv] + ind1 = [ii for ii, vv in enumerate(out) if flag1 in vv] + if len(ind0) != 1 or len(ind1) != 1: + msg = ("Format of html response seems to have changed!\n" + + "Cannot find flags:\n" + + "\t- {}\n".format(flag0) + + "\t- {}\n".format(flag1) + + "in requests.get({}).text".format(total_url)) + raise Exception(msg) + out = out[ind0[0] + 1].split('') + nresults = len(out) - 1 + + # Get columns + col = [kk.replace('', '').replace('', '').strip() + for kk in out[0].split('')] + ncol = len(col) + colex = ['Wavelength', 'Ion', 'Data Type', 'Transition', 'File Details'] + if col != colex: + msg = ("Format of table columns in html seems to have changed!\n" + + "\t- expected: {}\n".format(colex) + + "\t- observed: {}".format(col)) + raise Exception(msg) + + lout = [] + for ii in range(0, nresults): + lstri = out[ii+1].split('') + assert len(lstri) == ncol + lamb = lstri[0].replace('Å', '') + elm, charg = lstri[1].replace('', '').split('') + if charg == '+': + charg = '1+' + if element is not None and elm.lower() != element.lower(): + continue + if charge is not None and charg != charge: + continue + typ = lstri[2].replace('', '').split('>')[1] + trans = lstri[3].replace(' ', ' ') + trans = trans.replace('', '^{').replace('', '}') + trans = trans.replace('', '_{').replace('', '}') + trans = trans.replace('→', '->') + fil = lstri[4][lstri[4].index('detail')+len('detail'):] + fil = fil[:fil.index('.dat')+len('.dat')] + lout.append([lamb, elm, charg, typ, trans, fil]) + + # Format output + char = np.array(lout) + col = ['Wavelength', 'Element', 'Charge', 'Data Type', + 'Transition', 'Full file name'] + arr = _getcharray(char, col=col, + sep=' ', line='-', just='l', + returnas=returnas, verb=verb) + return arr + + +# ############################################################################# +# Download +# ############################################################################# + + +def _check_exists(filename, update=None): + + # In case a small modification becomes necessary later + target = filename + path_local = _get_PATH_LOCAL() + + # Check whether the local .tofu repo exists, if not recommend tofu-custom + if path_local is None: + path = os.path.join(os.path.expanduser('~'), '.tofu', 'openadas2tofu') + msg = ("You do not seem to have a local ./tofu repository\n" + + "tofu uses that local repository to store all user-specific " + + "data and downloads\n" + + "In particular, openadas files are downloaded and saved in:\n" + + "\t{}\n".format(path) + + " => to set-up your local .tofu repo, run in a terminal:\n" + + "\ttofu-custom") + raise Exception(msg) + + # Parse intermediate repos and create if necessary + lrep = target.split('/')[1:-1] + for ii in range(len(lrep)): + repo = os.path.join(path_local, *lrep[:ii+1]) + if not os.path.isdir(repo): + os.mkdir(repo) + + # Check if file already exists + path = os.path.join(path_local, *lrep) + pfe = os.path.join(path, target.split('/')[-1]) + if os.path.isfile(pfe): + if update is False: + msg = ("File already exists in your local repo:\n" + + "\t{}\n".format(pfe) + + " => if you want to force download, use update=True" + + " (local file will be overwritten)") + raise FileAlreayExistsException(msg) + else: + return True, pfe + else: + return False, pfe + + +def download(filename=None, + update=None, verb=None, returnas=None): + """ Download desired file from https://open.adas.ac.uk + + All downloaded files are stored in your local tofu directory (~/.tofu/) + + example + ------- + >>> import tofu as tf + >>> filename = '/adf15/pec40][ar/pec40][ar_ls][ar16.dat' + >>> tf.openadas2tofu.download(filename) + """ + + # --------------------------- + # Check + if verb is None: + verb = True + if update is None: + update = False + if returnas is None: + returnas = False + + c0 = (not isinstance(filename, str) + or filename[:4] != '/adf' or filename[-4:] != '.dat') + if c0: + msg = ("filename must be a str (full file name) of the form:\n" + + "\t/adf.../.../....dat") + raise Exception(msg) + url = _URL_DOWNLOAD + filename + + exists, pfe = _check_exists(filename, update=update) + if exists is True and verb is True: + msg = ("File already exists, will be downloaded and overwritten:\n" + + "\t{}".format(pfe)) + warnings.warn(msg) + + # --------------------------- + # Download + # Note the stream=True parameter below + with requests.get(url, stream=True) as rr: + rr.raise_for_status() + with open(pfe, 'wb') as ff: + for chunk in rr.iter_content(chunk_size=8192): + # filter-out keep-alive new chunks + if chunk: + ff.write(chunk) + # ff.flush() + + if verb is True: + msg = ("file {} was copied to:\n".format(filename) + + "\t{}".format(pfe)) + print(msg) + if returnas is str: + return pfe + + +def download_all(files=None, searchstr=None, + lambmin=None, lambmax=None, element=None, + include_partial=None, update=None, verb=None): + """ Download all desired files from https://open.adas.ac.uk + + The files to download can be provided either as: + - a list of full openadas file names (files) + - the result of an online freeform search + (searchstr fed to search_online) + - the result of an online search by wavelength + (lambmin, lambmax and element fed to search_online_by_wavelengthA) + + All downloaded files are stored in your local tofu directory (~/.tofu/) + + example + ------- + >>> import tofu as tf + >>> tf.openadas2tofu.download_all(lambmin=3., lambmax=4., element='ar') + """ + # Check + if include_partial is None: + include_partial = False + if update is None: + update = False + if verb is None: + verb = True + lc = [files is not None, + searchstr is not None, + any([ss is not None for ss in [lambmin, lambmax, element]])] + if np.sum(lc) != 1: + msg = ( + "Please either searchstr xor (lambmin, lambmax, element)\n" + + "\t- files: list of full file names to be downloaded\n" + + "\t- searchstr: uses search_online()\n" + + "\t- lambmin, lambmax, element: search_online_by_wavelengthA") + raise Exception(msg) + + # Get list of files + if lc[0]: + if isinstance(files, str): + files = [files] + if not (isinstance(files, list) + and all([isinstance(ss, str) for ss in files])): + msg = "files must be a list of full openadas file names!" + raise Exception(msg) + elif lc[1]: + arr = search_online(searchstr=searchstr, + include_partial=include_partial, + verb=False, returnas=np.ndarray) + files = arr[:, -1] + elif lc[2]: + arr = search_online_by_wavelengthA(lambmin=lambmin, + lambmax=lambmax, + element=element, verb=False, + returnas=np.ndarray) + files = np.unique(arr[:, -1]) + + # Download + if verb is True: + msg = "Downloading from {} into {}:".format(_URL, _get_PATH_LOCAL()) + print(msg) + for ii in range(len(files)): + try: + exists, pfe = _check_exists(files[ii], update=update) + pfe = download(filename=files[ii], update=update, + verb=False, returnas=str) + if exists is True: + msg = "\toverwritten: \t{}".format(files[ii]) + else: + msg = "\tdownloaded: \t{}".format(files[ii]) + except FileAlreayExistsException: + msg = "\talready exists: {}".format(files[ii]) + except Exception as err: + msg = (str(err) + + "\n\nCould not download file {}".format(files[ii])) + raise err + + if verb is True: + print(msg) + + +def clean_downloads(): + """ Delete all openadas files downloaded in your ~/.tofu/ directory """ + path_local = _get_PATH_LOCAL() + if path_local is None: + return + lf = [ff for ff in os.listdir(path_local) + if os.path.isfile(os.path.join(path_local, ff))] + ld = [ff for ff in os.listdir(path_local) + if os.path.isdir(os.path.join(path_local, ff))] + for ff in lf: + os.remove(ff) + for dd in ld: + shutil.rmtree(os.path.join(path_local, dd)) diff --git a/tofu/scripts/__init__.py b/tofu/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tofu/scripts/_def.py b/tofu/scripts/_def.py new file mode 100644 index 000000000..b779d9bf4 --- /dev/null +++ b/tofu/scripts/_def.py @@ -0,0 +1,29 @@ + + +# ############################################################################# +# tofuplot parameters +# ############################################################################# + +_TFPLOT_RUN = 0 +_TFPLOT_USER = 'imas_public' +_TFPLOT_TOKAMAK = 'west' +_TFPLOT_VERSION = '3' +_TFPLOT_T0 = 'IGNITRON' +_TFPLOT_SHAREX = False +_TFPLOT_BCK = True +_TFPLOT_EXTRA = True +_TFPLOT_INDCH_AUTO = True + + +# ############################################################################# +# tofucalc parameters +# ############################################################################# +_TFCALC_RUN = 0 +_TFCALC_USER = 'imas_public' +_TFCALC_TOKAMAK = 'west' +_TFCALC_VERSION = '3' +_TFCALC_T0 = 'IGNITRON' +_TFCALC_SHAREX = False +_TFCALC_BCK = True +_TFCALC_EXTRA = None +_TFCALC_INDCH_AUTO = True diff --git a/tofucalc.py b/tofu/scripts/tofucalc.py similarity index 74% rename from tofucalc.py rename to tofu/scripts/tofucalc.py index 79dc306a9..521e6880b 100755 --- a/tofucalc.py +++ b/tofu/scripts/tofucalc.py @@ -13,8 +13,9 @@ # tofu # test if in a tofu git repo _HERE = os.path.abspath(os.path.dirname(__file__)) +_HERE = os.path.dirname(os.path.dirname(_HERE)) istofugit = False -if '.git' in _HERE and 'tofu' in _HERE: +if '.git' in os.listdir(_HERE) and 'tofu' in _HERE: istofugit = True if istofugit: @@ -26,12 +27,29 @@ else: import tofu as tf from tofu.imas2tofu import MultiIDSLoader + +# default parameters +pfe = os.path.join(os.path.expanduser('~'), '.tofu', '_scripts_def.py') +if os.path.isfile(pfe): + # Make sure we load the user-specific file + # sys.path method + # sys.path.insert(1, os.path.join(os.path.expanduser('~'), '.tofu')) + # import _scripts_def as _defscripts + # _ = sys.path.pop(1) + # importlib method + import importlib.util + spec = importlib.util.spec_from_file_location("_defscripts", pfe) + _defscripts = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_defscripts) +else: + try: + import tofu.scripts._def as _defscripts + except Exception as err: + from . import _def as _defscripts + tforigin = tf.__file__ tfversion = tf.__version__ - -# if tf.__version__ < '1.4.1': - # msg = "tofuplot only works with tofu >= 1.4.1" - # raise Exception(msg) +print(tforigin, tfversion) if 'imas2tofu' not in dir(tf): msg = "imas does not seem to be available\n" @@ -46,16 +64,22 @@ # default values ################################################### -_RUN = 0 -_USER = 'imas_public' -_TOKAMAK = 'west' -_VERSION = '3' + +# User-customizable +_RUN = _defscripts._TFCALC_RUN +_USER = _defscripts._TFCALC_USER +_TOKAMAK = _defscripts._TFCALC_TOKAMAK +_VERSION = _defscripts._TFCALC_VERSION +_T0 = _defscripts._TFCALC_T0 +_SHAREX = _defscripts._TFCALC_SHAREX +_BCK = _defscripts._TFCALC_BCK +_EXTRA = _defscripts._TFCALC_EXTRA +_INDCH_AUTO = _defscripts._TFCALC_INDCH_AUTO + +# Non user-customizable _LIDS_DIAG = MultiIDSLoader._lidsdiag _LIDS = _LIDS_DIAG -_T0 = 'IGNITRON' -_SHAREX = False -_BCK = True -_EXTRA = True + ################################################### ################################################### @@ -79,7 +103,8 @@ def call_tfcalcimas(shot=None, run=_RUN, user=_USER, ids=None, t0=_T0, extra=_EXTRA, plot_compare=True, Brightness=None, res=None, interp_t=None, - sharex=_SHAREX, indch=None, indch_auto=None, + sharex=_SHAREX, indch=None, indch_auto=_INDCH_AUTO, + input_file=None, output_file=None, background=_BCK): if t0.lower() == 'none': @@ -90,7 +115,8 @@ def call_tfcalcimas(shot=None, run=_RUN, user=_USER, ids=ids, indch=indch, indch_auto=indch_auto, plot_compare=plot_compare, extra=extra, Brightness=Brightness, res=res, interp_t=interp_t, - t0=t0, plot=True, sharex=sharex, bck=background) + input_file=input_file, output_file=output_file, + t0=t0, plot=None, sharex=sharex, bck=background) plt.show(block=True) @@ -112,8 +138,8 @@ def _str2bool(v): raise argparse.ArgumentTypeError('Boolean value expected !') -if __name__ == '__main__': - +# if __name__ == '__main__': +def main(): # Parse input arguments msg = """Fast interactive visualization tool for diagnostics data in imas @@ -153,13 +179,19 @@ def _str2bool(v): nargs='+', default=None) parser.add_argument('-ichauto', '--indch_auto', type=bool, required=False, help='automatically determine indices of channels to be loaded', - default=True) + default=_INDCH_AUTO) parser.add_argument('-e', '--extra', type=_str2bool, required=False, help='If True loads separatrix and heating power', default=_EXTRA) parser.add_argument('-sx', '--sharex', type=_str2bool, required=False, help='Should X axis be shared between diagnostics ids ?', default=_SHAREX, const=True, nargs='?') + parser.add_argument('-if', '--input_file', type=str, required=False, + help='mat file from which to load core_profiles', + default=None) + parser.add_argument('-of', '--output_file', type=str, required=False, + help='mat file into which to save synthetic signal', + default=None) parser.add_argument('-bck', '--background', type=_str2bool, required=False, help='Plot data enveloppe as grey background ?', default=_BCK, const=True, nargs='?') @@ -168,3 +200,8 @@ def _str2bool(v): # Call wrapper function call_tfcalcimas(**dict(args._get_kwargs())) + + +# Add this to make sure it remains executable even without install +if __name__ == '__main__': + main() diff --git a/tofu/scripts/tofucustom.py b/tofu/scripts/tofucustom.py new file mode 100755 index 000000000..04201c98b --- /dev/null +++ b/tofu/scripts/tofucustom.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python + +# Built-in +import os +import getpass +from shutil import copyfile +import argparse + + +################################################### +################################################### +# default values +################################################### + + +_SOURCE = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +_USER = getpass.getuser() +_USER_HOME = os.path.expanduser('~') +_TARGET = os.path.join(_USER_HOME, '.tofu') +_LF = ['_imas2tofu_def.py', '_scripts_def.py'] +_LD = ['openadas2tofu'] + + +################################################### +################################################### +# function +################################################### + + +def custom(target=_TARGET, source=_SOURCE, + files=_LF, directories=_LD): + + # Caveat (up to now only relevant for _TARGET) + if target != _TARGET: + msg = "" + raise Exception(msg) + + # Check files + if isinstance(files, str): + files = [files] + if not isinstance(files, list) or any([ff not in _LF for ff in files]): + msg = "All files should be in {}".format(_LF) + raise Exception(msg) + + # Try creating directory and copying modules + try: + # Create .tofu/ if non-existent + if not os.path.isdir(target): + os.mkdir(target) + + # Create directories + for dd in directories: + if not os.path.isdir(os.path.join(target, dd)): + os.mkdir(os.path.join(target, dd)) + + # Copy files + for ff in files: + mod, f0 = ff.split('_')[1:] + copyfile(os.path.join(source, mod, '_'+f0), + os.path.join(target, ff)) + + msg = ("A local copy of default tofu parameters is now in:\n" + + "\t{}/\n".format(target) + + "You can edit it to spice up your tofu") + print(msg) + + except Exception as err: + msg = (str(err) + '\n\n' + + "A problem occured\n" + + "tofu-custom tried to create a local directory .tofu/ in " + + "your home {}\n".format(target) + + "But it could not, check the error message above to debug\n" + + "Most frequent cause is a permission issue") + raise Exception(msg) + + +################################################### +################################################### +# bash call (main) +################################################### + +def main(): + # Parse input arguments + msg = """ Create a local copy of tofu default parameters + + This creates a local copy, in your home, of tofu default parameters + A directory .tofu is created in your home directory + In this directory, modules containing default parameters are copied + You can then customize them without impacting other users + + """ + + # Instanciate parser + parser = argparse.ArgumentParser(description=msg) + + # Define input arguments + parser.add_argument('-s', '--source', + type=str, + help='tofu source directory', + required=False, + default=_SOURCE) + parser.add_argument('-t', '--target', + type=str, + help=('directory where .tofu/ should be created' + + ' (default: {})'.format(_TARGET)), + required=False, + default=_TARGET) + parser.add_argument('-f', '--files', + type=str, + help='list of files to be copied', + required=False, + nargs='+', + default=_LF, + choices=_LF) + + # Parse arguments + args = parser.parse_args() + + # Call function + custom(**dict(args._get_kwargs())) + + +if __name__ == '__main__': + main() diff --git a/tofuplot.py b/tofu/scripts/tofuplot.py similarity index 81% rename from tofuplot.py rename to tofu/scripts/tofuplot.py index 5259d3854..8fb72297d 100755 --- a/tofuplot.py +++ b/tofu/scripts/tofuplot.py @@ -13,8 +13,9 @@ # tofu # test if in a tofu git repo _HERE = os.path.abspath(os.path.dirname(__file__)) +_HERE = os.path.dirname(os.path.dirname(_HERE)) istofugit = False -if '.git' in _HERE and 'tofu' in _HERE: +if '.git' in os.listdir(_HERE) and 'tofu' in _HERE: istofugit = True if istofugit: @@ -26,9 +27,29 @@ else: import tofu as tf from tofu.imas2tofu import MultiIDSLoader + +# default parameters +pfe = os.path.join(os.path.expanduser('~'), '.tofu', '_scripts_def.py') +if os.path.isfile(pfe): + # Make sure we load the user-specific file + # sys.path method + # sys.path.insert(1, os.path.join(os.path.expanduser('~'), '.tofu')) + # import _scripts_def as _defscripts + # _ = sys.path.pop(1) + # importlib method + import importlib.util + spec = importlib.util.spec_from_file_location("_defscripts", pfe) + _defscripts = importlib.util.module_from_spec(spec) + spec.loader.exec_module(_defscripts) +else: + try: + import tofu.scripts._def as _defscripts + except Exception as err: + from . import _def as _defscripts + tforigin = tf.__file__ tfversion = tf.__version__ - +print(tforigin, tfversion) if 'imas2tofu' not in dir(tf): msg = "imas does not seem to be available\n" @@ -43,17 +64,23 @@ # default values ################################################### -_RUN = 0 -_USER = 'imas_public' -_TOKAMAK = 'west' -_VERSION = '3' + +# User-customizable +_RUN = _defscripts._TFPLOT_RUN +_USER = _defscripts._TFPLOT_USER +_TOKAMAK = _defscripts._TFPLOT_TOKAMAK +_VERSION = _defscripts._TFPLOT_VERSION +_T0 = _defscripts._TFPLOT_T0 +_SHAREX = _defscripts._TFPLOT_SHAREX +_BCK = _defscripts._TFPLOT_BCK +_EXTRA = _defscripts._TFPLOT_EXTRA +_INDCH_AUTO = _defscripts._TFPLOT_INDCH_AUTO + +# Not user-customizable _LIDS_DIAG = MultiIDSLoader._lidsdiag _LIDS_PLASMA = tf.imas2tofu.MultiIDSLoader._lidsplasma _LIDS = _LIDS_DIAG + _LIDS_PLASMA + ['magfieldlines'] -_T0 = 'IGNITRON' -_SHAREX = False -_BCK = True -_EXTRA = True + ################################################### ################################################### @@ -75,7 +102,7 @@ def _get_exception(q, ids, qtype='quantity'): def call_tfloadimas(shot=None, run=_RUN, user=_USER, tokamak=_TOKAMAK, version=_VERSION, extra=_EXTRA, ids=None, quantity=None, X=None, t0=_T0, - sharex=_SHAREX, indch=None, indch_auto=None, + sharex=_SHAREX, indch=None, indch_auto=_INDCH_AUTO, background=_BCK, t=None, dR_sep=None, init=None): lidspla = [ids_ for ids_ in ids if ids_ in _LIDS_PLASMA] @@ -109,8 +136,8 @@ def _str2bool(v): raise argparse.ArgumentTypeError('Boolean value expected !') -if __name__ == '__main__': - +# if __name__ == '__main__': +def main(): # Parse input arguments msg = """Fast interactive visualization tool for diagnostics data in imas @@ -159,7 +186,7 @@ def _str2bool(v): nargs='+', default=None) parser.add_argument('-ichauto', '--indch_auto', type=_str2bool, required=False, help='automatically determine indices of' - + ' channels to be loaded', default=True) + + ' channels to be loaded', default=_INDCH_AUTO) parser.add_argument('-e', '--extra', type=_str2bool, required=False, help='If True loads separatrix and heating power', default=_EXTRA) @@ -174,3 +201,8 @@ def _str2bool(v): # Call wrapper function call_tfloadimas(**dict(args._get_kwargs())) + + +# Add this to make sure it remains executable even without install +if __name__ == '__main__': + main() diff --git a/tofu/utils.py b/tofu/utils.py index 82e053a2c..7d722b41d 100644 --- a/tofu/utils.py +++ b/tofu/utils.py @@ -453,8 +453,7 @@ def _filefind(name, path=None, lmodes=['.npz','.mat']): return name, mode, pfe - -def load(name, path=None, strip=None, verb=True): +def load(name, path=None, strip=None, verb=True, allow_pickle=None): """ Load a tofu object file Can load from .npz or .txt files @@ -483,7 +482,7 @@ def load(name, path=None, strip=None, verb=True): obj = _load_from_txt(name, pfe) else: if mode == 'npz': - dd = _load_npz(pfe) + dd = _load_npz(pfe, allow_pickle=allow_pickle) elif mode == 'mat': dd = _load_mat(pfe) @@ -580,13 +579,15 @@ def _get_load_npzmat_dict(out, pfe, mode='npz', exclude_keys=[]): return dout - -def _load_npz(pfe): +def _load_npz(pfe, allow_pickle=None): + if allow_pickle is None: + allow_pickle = True try: - out = np.load(pfe, mmap_mode=None) + out = np.load(pfe, mmap_mode=None, allow_pickle=allow_pickle) except UnicodeError: - out = np.load(pfe, mmap_mode=None, encoding='latin1') + out = np.load(pfe, mmap_mode=None, allow_pickle=allow_pickle, + encoding='latin1') except Exception as err: raise err @@ -1039,8 +1040,9 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, ids=None, Name=None, out=None, tlim=None, config=None, occ=None, indch=None, description_2d=None, equilibrium=None, dsig=None, data=None, X=None, t0=None, dextra=None, - Brightness=None, res=None, interp_t=None, extra=True, - plot=True, plot_compare=True, sharex=False, + Brightness=None, res=None, interp_t=None, extra=None, + plot=None, plot_compare=True, sharex=False, + input_file=None, output_file=None, bck=True, indch_auto=True, t=None, init=None): # ------------------- # import imas2tofu @@ -1049,19 +1051,30 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, import tofu.imas2tofu as imas2tofu except Exception as err: msg = str(err) - msg += "\n\n module imas2tofu does not seem available\n" - msg += " => imas may not be installed ?" + msg += ("\n\n module imas2tofu does not seem available\n" + + " => imas may not be installed?") raise Exception(msg) lok = ['Data'] c0 = out is None or out in lok if not c0: - msg = "Arg out must be in %s"%str(lok) + msg = "Arg out must be in {}".format(lok) raise Exception(msg) + if plot is None: + if output_file is not None: + plot = False + else: + plot = True + if extra is None: + if input_file is not None: + extra = False + else: + extra = True + # ------------------- # Prepare ids - assert ids is None or type(ids) in [list,str] + assert ids is None or type(ids) in [list, str] if type(ids) is str: ids = [ids] if type(ids) is list: @@ -1083,12 +1096,24 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, if nids > 1: assert not any([ids_ in ids for ids_ in lidscustom]) + # Check if input_file + if input_file is not None: + lids_input_file = ['bremsstrahlung_visible'] + if nids != 1 or ids[0] not in lids_input_file: + msg = ("input_file is only available for a single ids in:\n" + + "\t- " + "\n\t- ".join(lids_input_file)) + raise Exception(msg) # ------------------- # Prepare shot shot = np.r_[shot].astype(int) nshot = shot.size + # Check if input_file + if input_file is not None: + if nshot != 1: + msg = "input_file not available for multiple shots!" + raise Exception(msg) # ------------------- # Prepare out @@ -1107,8 +1132,8 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, if nids > 1: if not all([ids_ in imas2tofu.MultiIDSLoader._lidsdiag for ids_ in ids]): - msg = "tf.load_from_imas() only handles multipe ids\n" - msg += "if all are diagnostics ids !" + msg = ("tf.load_from_imas() only handles multipe ids " + + "if all are diagnostics ids!") raise Exception(msg) # ------------------- @@ -1215,29 +1240,84 @@ def calc_from_imas(shot=None, run=None, user=None, tokamak=None, version=None, # ------------------- # load - for ss in shot: - multi = imas2tofu.MultiIDSLoader(shot=ss, run=run, user=user, - tokamak=tokamak, version=version, - ids=lids, synthdiag=True) - - # export to instances - for ii in range(0,nids): - if out[ii] == "Data": - multi.calc_signal(ids=lids[ii], - tlim=tlim, dsig=dsig, - config=config, t=t, - res=res, indch=indch, - Brightness=Brightness, - interp_t=interp_t, - indch_auto=indch_auto, - t0=t0, dextra=dextra, - plot=True, - plot_compare=plot_compare) - - - + if input_file is None: + for ss in shot: + multi = imas2tofu.MultiIDSLoader(shot=ss, run=run, user=user, + tokamak=tokamak, version=version, + ids=lids, synthdiag=True) + + # export to instances + for ii in range(0, nids): + if out[ii] == "Data": + multi.calc_signal(ids=lids[ii], + tlim=tlim, dsig=dsig, + config=config, t=t, + res=res, indch=indch, + Brightness=Brightness, + interp_t=interp_t, + indch_auto=indch_auto, + t0=t0, dextra=dextra, + plot=True, + plot_compare=plot_compare) + else: + multi = imas2tofu.MultiIDSLoader(shot=shot[0], run=run, user=user, + tokamak=tokamak, version=version, + ids=lids, synthdiag=False, get=False) + if 'bremsstrahlung_visible' in lids: + multi.add_ids('equilibrium', get=True) + plasma = multi.to_Plasma2D() + lf = ['t', 'rhotn', 'brem'] + lamb = multi.get_data('bremsstrahlung_visible', sig='lamb')['lamb'] + dout = imas2tofu.get_data_from_matids(input_file, + return_fields=lf, + lamb=lamb[0]) + plasma.add_ref(key='core_profiles.t', data=dout['t'], group='time', + origin='input_file') + nrad = dout['rhotn'].shape[1] + plasma.add_ref(key='core_profiles.radius', data=np.arange(0, nrad), + group='radius', origin='input_file') + plasma.add_quantity(key='core_profiles.1drhotn', + data=dout['rhotn'], + depend=('core_profiles.t', + 'core_profiles.radius'), + origin='input_file', + quant='rhotn', dim='rho', units='adim.') + plasma.add_quantity(key='core_profiles.1dbrem', data=dout['brem'], + depend=('core_profiles.t', + 'core_profiles.radius'), + origin='input_file') + cam = multi.to_Cam(plot=False) + sig = cam.calc_signal_from_Plasma2D(plasma, + quant='core_profiles.1dbrem', + ref1d='core_profiles.1drhotn', + ref2d='equilibrium.2drhotn', + Brightness=True, plot=plot)[0] + if output_file is not None: + try: + # Format output dictionnary to be saved + dout = {'shot': shot[0], + 't': sig.t, + 'data': sig.data, + 'units_t': 's', + 'units_data': 'ph / (s.m2.sr.m)', + 'channels': sig.dchans('names'), + 'tofu_version': __version__} + + # Save to specified path + filename + extension + if output_file[-4:] != '.mat': + assert len(output_file.split('.')) == 1 + output_file += '.mat' + scpio.savemat(output_file, dout) + msg = ("Successfully saved in:\n" + + "\t{}".format(output_file)) + print(msg) + except Exception as err: + msg = str(err) + msg += "\nCould not save computed synthetic signal to:\n" + msg += "scpio.savemat({0}, dout)".format(output_file) + warnings.warn(msg) ############################################# @@ -1491,7 +1571,11 @@ def _getcharray(ar, col=None, sep=' ', line='-', just='l', msg=True): # Get just len nn = np.char.str_len(ar).max(axis=0) if col is not None: - assert len(col) in ar.shape + if len(col) not in ar.shape: + msg = ("len(col) should be in np.array(ar, dtype='U').shape:\n" + + "\t- len(col) = {}\n".format(len(col)) + + "\t- ar.shape = {}".format(ar.shape)) + raise Exception(msg) if len(col) != ar.shape[1]: ar = ar.T nn = np.char.str_len(ar).max(axis=0) diff --git a/tofu/version.py b/tofu/version.py index fbadd4a44..443c713ec 100644 --- a/tofu/version.py +++ b/tofu/version.py @@ -1,2 +1,2 @@ # Do not edit, pipeline versioning governed by git tags! -__version__ = '1.4.2' +__version__ = '1.4.2b4-92-gf832ab7e'