From ae63cb2bea41b76cc591cfa47c69ad5f1a3a9d3b Mon Sep 17 00:00:00 2001 From: juacrumar Date: Thu, 8 Sep 2022 16:45:45 +0200 Subject: [PATCH 1/9] remove recommonmark --- conda-recipe/meta.yaml | 2 +- doc/sphinx/source/buildmaster.md | 2 +- doc/sphinx/source/ci/index.md | 2 +- doc/sphinx/source/conf.py | 14 ++------------ doc/sphinx/source/contributing/git.md | 2 +- doc/sphinx/source/contributing/python-tools.md | 6 +++--- doc/sphinx/source/contributing/rules.md | 6 +++--- doc/sphinx/source/data/plotting_format.md | 2 +- doc/sphinx/source/external-code/apfelcomb.md | 4 ++-- doc/sphinx/source/external-code/grids.md | 2 +- doc/sphinx/source/external-code/pdf-codes.md | 2 +- doc/sphinx/source/get-started/git.md | 2 +- doc/sphinx/source/serverconf/index.md | 6 +++--- doc/sphinx/source/theory/theoryparamsinfo.md | 2 +- doc/sphinx/source/tutorials/APPLgrids_comp.md | 4 ++-- doc/sphinx/source/tutorials/apfelcomb.md | 10 +++++----- doc/sphinx/source/tutorials/closuretest.md | 4 ++-- doc/sphinx/source/tutorials/compare-fits.md | 4 ++-- doc/sphinx/source/tutorials/datthcomp.md | 2 +- doc/sphinx/source/tutorials/list-resources.md | 2 +- doc/sphinx/source/tutorials/report.md | 2 +- doc/sphinx/source/tutorials/run-fit.md | 8 ++++---- doc/sphinx/source/vp/api.md | 2 +- doc/sphinx/source/vp/datthcomp.md | 2 +- doc/sphinx/source/vp/download.md | 8 ++++---- doc/sphinx/source/vp/filters.md | 16 ++++++++-------- doc/sphinx/source/vp/nnprofile.md | 4 ++-- doc/sphinx/source/vp/upload.md | 12 ++++++------ 28 files changed, 62 insertions(+), 72 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index adc3c6328e..13e81d37ca 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -47,7 +47,7 @@ requirements: - prompt_toolkit - validobj - sphinx >=4.0.2 # documentation. Needs pinning becasue https://github.com/sphinx-doc/sphinx/issues/9216 - - recommonmark + - myst-parser - sphinx_rtd_theme >0.5 - sphinxcontrib-bibtex - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 diff --git a/doc/sphinx/source/buildmaster.md b/doc/sphinx/source/buildmaster.md index 44d3c6fcb6..dbb9f9d6bf 100644 --- a/doc/sphinx/source/buildmaster.md +++ b/doc/sphinx/source/buildmaster.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _buildmaster: ``` diff --git a/doc/sphinx/source/ci/index.md b/doc/sphinx/source/ci/index.md index e1b18baca6..8c5faa293f 100644 --- a/doc/sphinx/source/ci/index.md +++ b/doc/sphinx/source/ci/index.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _CI: ``` # Continuous integration and deployment diff --git a/doc/sphinx/source/conf.py b/doc/sphinx/source/conf.py index a155d04c18..f3b78b5e8f 100644 --- a/doc/sphinx/source/conf.py +++ b/doc/sphinx/source/conf.py @@ -15,8 +15,6 @@ # import sys # # -from recommonmark.transform import AutoStructify - # -- Project information ----------------------------------------------------- project = 'NNPDF' @@ -53,7 +51,7 @@ # particularly in markdown. See # https://recommonmark.readthedocs.io/en/latest/#linking-to-headings-in-other-files 'sphinx.ext.autosectionlabel', - 'recommonmark', + 'myst_parser' ] @@ -219,12 +217,4 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True -# Adapted this from -# https://github.com/readthedocs/recommonmark/blob/ddd56e7717e9745f11300059e4268e204138a6b1/docs/conf.py -# app setup hook -def setup(app): - app.add_config_value('recommonmark_config', { - #'url_resolver': lambda url: github_doc_root + url, - 'enable_eval_rst': True, - }, True) - app.add_transform(AutoStructify) +myst_heading_anchors = 3 diff --git a/doc/sphinx/source/contributing/git.md b/doc/sphinx/source/contributing/git.md index ee12d65223..2dc32f599e 100644 --- a/doc/sphinx/source/contributing/git.md +++ b/doc/sphinx/source/contributing/git.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _git: ``` # Git, GitHub and GitLab diff --git a/doc/sphinx/source/contributing/python-tools.md b/doc/sphinx/source/contributing/python-tools.md index b230f29f3a..d2842edd0d 100644 --- a/doc/sphinx/source/contributing/python-tools.md +++ b/doc/sphinx/source/contributing/python-tools.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _pytools: ``` @@ -29,7 +29,7 @@ Python code can be evaluated interactively, which can speed up the development. - [Jupyter notebook](https://jupyter.org/): Interactive development environment running on the browser. Useful for bigger experiments. -```eval_rst +```{eval-rst} .. note:: When developing :ref:`validphys ` related code interactively, be sure to read about the :ref:`API functionality `. @@ -51,7 +51,7 @@ Python code can be evaluated interactively, which can speed up the development. -```eval_rst +```{eval-rst} .. _pytoolsqa: ``` ## Code quality and reviewing diff --git a/doc/sphinx/source/contributing/rules.md b/doc/sphinx/source/contributing/rules.md index 356dd6460e..ab29db0728 100644 --- a/doc/sphinx/source/contributing/rules.md +++ b/doc/sphinx/source/contributing/rules.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _rules: ``` # Code development @@ -19,7 +19,7 @@ This PR should adhere to the following rules: address? If the reason for the PR has already been detailed in an issue, then this issue should be linked in the PR. -* The PR should contain **[documentation](../sphinx-documentation.md) describing +* The PR should contain **[documentation](./sphinx-documentation.md) describing the new features**, if applicable. * If the PR is fixing a bug, information should be given such that a reviewer can reproduce the bug. @@ -46,7 +46,7 @@ It demonstrates how to add a new feature, together with relevant tests and documentation, and refine it based on the discussion. -```eval_rst +```{eval-rst} .. _reviews: ``` ## Reviewing pull requests diff --git a/doc/sphinx/source/data/plotting_format.md b/doc/sphinx/source/data/plotting_format.md index 1a92d123b5..cc143e9b19 100644 --- a/doc/sphinx/source/data/plotting_format.md +++ b/doc/sphinx/source/data/plotting_format.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _plotting-format: ``` Plotting format diff --git a/doc/sphinx/source/external-code/apfelcomb.md b/doc/sphinx/source/external-code/apfelcomb.md index 5d99cc5bd0..3b28ab995b 100644 --- a/doc/sphinx/source/external-code/apfelcomb.md +++ b/doc/sphinx/source/external-code/apfelcomb.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _apfelcomb: ``` # Using APFELcomb @@ -14,7 +14,7 @@ APFELcomb is available from ``` https://github.com/NNPDF/apfelcomb ```` -The various data formats used in APFELcomb are described in [Experimental data files](../data/exp-data-files.html#exp-data-files). +The various data formats used in APFELcomb are described in [Experimental data files](../data/exp-data-files.rst#exp-data-files). APFELcomb depends on the following libraries diff --git a/doc/sphinx/source/external-code/grids.md b/doc/sphinx/source/external-code/grids.md index d7d2e95924..cee137028a 100644 --- a/doc/sphinx/source/external-code/grids.md +++ b/doc/sphinx/source/external-code/grids.md @@ -24,7 +24,7 @@ learn how to generate APPLgrids, please see the tutorial [here](../tutorials/APP APFELcomb generates FK tables for NNPDF fits. Information on how to use it can be found [here](./apfelcomb.md). You can read about the mechanism behind APFELcomb [here](https://arxiv.org/abs/1605.02070) and find more information about the theory behind FK tables -in the [Theory section](../Theory/FastInterface.rst). +in the [Theory section](../theory/FastInterface.rst). ## Other codes diff --git a/doc/sphinx/source/external-code/pdf-codes.md b/doc/sphinx/source/external-code/pdf-codes.md index e5e5f97fe7..d8f86be643 100644 --- a/doc/sphinx/source/external-code/pdf-codes.md +++ b/doc/sphinx/source/external-code/pdf-codes.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _lhapdf: ``` # PDF set storage and interpolation diff --git a/doc/sphinx/source/get-started/git.md b/doc/sphinx/source/get-started/git.md index aec6e24ce4..a440ef7a8c 100644 --- a/doc/sphinx/source/get-started/git.md +++ b/doc/sphinx/source/get-started/git.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _git: ``` # Downloading the code diff --git a/doc/sphinx/source/serverconf/index.md b/doc/sphinx/source/serverconf/index.md index e0822099a3..11be0bf9cb 100644 --- a/doc/sphinx/source/serverconf/index.md +++ b/doc/sphinx/source/serverconf/index.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _server: ``` Servers @@ -29,7 +29,7 @@ We perform every Sunday a `rsync` from the `/home/nnpdf` folder to the `nnpdf@lxplus` account at CERN. -```eval_rst +```{eval-rst} .. _server-access: ``` Access @@ -82,7 +82,7 @@ The relevant passwords can be found [here](https://www.wiki.ed.ac.uk/pages/viewpage.action?pageId=292165461). -```eval_rst +```{eval-rst} .. _web-scripts: ``` Web scripts diff --git a/doc/sphinx/source/theory/theoryparamsinfo.md b/doc/sphinx/source/theory/theoryparamsinfo.md index bd2d063826..7cadaf0f08 100644 --- a/doc/sphinx/source/theory/theoryparamsinfo.md +++ b/doc/sphinx/source/theory/theoryparamsinfo.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _th_parameter_info: ``` # Looking up the parameters of a theory diff --git a/doc/sphinx/source/tutorials/APPLgrids_comp.md b/doc/sphinx/source/tutorials/APPLgrids_comp.md index 9ececcf8e7..7cce10c173 100644 --- a/doc/sphinx/source/tutorials/APPLgrids_comp.md +++ b/doc/sphinx/source/tutorials/APPLgrids_comp.md @@ -1,7 +1,7 @@ # How to benchmark and store APPLgrid/FastNLO tables APPLgrid and fastNLO tables produced according to -[How to generate APPLgrid and FastNLO tables](.../tutorials/APPLgrids.md) +[How to generate APPLgrid and FastNLO tables](../tutorials/APPLgrids.md) must be benchmarked and properly stored before they can be used to produce FK tables, as outlined in [How to generate and implement FK tables](../tutorials/apfelcomb.md). @@ -68,7 +68,7 @@ fnlo-tk-merge see the [APPLgrid](https://applgrid.hepforge.org/) and [FastNLO](https://fastnlo.hepforge.org/) documentation for further details. -```eval_rst +```{eval-rst} .. _storage: ``` diff --git a/doc/sphinx/source/tutorials/apfelcomb.md b/doc/sphinx/source/tutorials/apfelcomb.md index c196607146..c3976c9c6d 100644 --- a/doc/sphinx/source/tutorials/apfelcomb.md +++ b/doc/sphinx/source/tutorials/apfelcomb.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _tutorialfktables: ``` @@ -11,7 +11,7 @@ achieved by taking DGLAP evolution kernels from ``APFEL`` and combining them with interpolated parton-level observable kernels in the APPLgrid or FastNLO format (see [How to generate APPLgrid and fastNLO tables](../tutorials/APPLgrids.md)). -The various data formats used in APFELcomb are described in [Experimental data files](../data/exp-data-files.html#exp-data-files). +The various data formats used in APFELcomb are described in [Experimental data files](../data/exp-data-files.rst#exp-data-files). The user is strongly encouraged to go through that note with care, in order to familiarise himself with the features and the structure of the APFELcomb @@ -236,10 +236,10 @@ points. The N operator is used to perform unit conversions or the like. ### Compound files and C-factors If the new dataset is a compound observable (that is, theory predictions are a function of more than one FK-product), then one should write a corresponding -`COMPOUND` file as described in [Theory data files](../data/th-data-files.html#compound-file-format). This compound file should be stored +`COMPOUND` file as described in [Theory data files](../data/th-data-files.rst#compound-file-format). This compound file should be stored in the APFELcomb repository under the `compound` directory. -C-factors should be in the format specified in [Theory data files](../data/th-data-files.html#cfactor-file-format) and stored in the nnpdfcpp +C-factors should be in the format specified in [Theory data files](../data/th-data-files.rst#cfactor-file-format) and stored in the nnpdfcpp repository under ``` nnpdf/nnpdfcpp/data/N*LOCFAC/ @@ -297,4 +297,4 @@ a cluster) cluster is then: ./merge_allgrids.py # Merge subgrids into FK tables # If merging is successful ./finalise.sh -# Results in a final theory at ./results/theory_.tgz \ No newline at end of file +# Results in a final theory at ./results/theory_.tgz diff --git a/doc/sphinx/source/tutorials/closuretest.md b/doc/sphinx/source/tutorials/closuretest.md index 1624a2ed46..595e38714e 100644 --- a/doc/sphinx/source/tutorials/closuretest.md +++ b/doc/sphinx/source/tutorials/closuretest.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _tut_closure: ``` @@ -182,4 +182,4 @@ $ n3fit fitname.yml ``` You will still need to evolve the fit and run ``postfit`` as with a standard -[``n3fit``](../n3fit/usage.md). +[``n3fit``](../tutorials/run-fit.md) diff --git a/doc/sphinx/source/tutorials/compare-fits.md b/doc/sphinx/source/tutorials/compare-fits.md index 374a847b9a..85ce2e0a6e 100644 --- a/doc/sphinx/source/tutorials/compare-fits.md +++ b/doc/sphinx/source/tutorials/compare-fits.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _compare-fits: ``` @@ -30,7 +30,7 @@ The resulting report produces a summary of the two fits and can be uploaded to the server by using `vp-upload `, where the folder is called `output` by default. -```eval_rst +```{eval-rst} The `vp-comparefits` application is implemented as a small wrapper on top of a specific `validphys` report. The wrapper code is defined in the :py:mod:`validphys.scripts.vp_comparefits` module, and the specific templates diff --git a/doc/sphinx/source/tutorials/datthcomp.md b/doc/sphinx/source/tutorials/datthcomp.md index 563a7b4b78..3b40010fff 100644 --- a/doc/sphinx/source/tutorials/datthcomp.md +++ b/doc/sphinx/source/tutorials/datthcomp.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _datthcomp: ``` # How to do a data theory comparison diff --git a/doc/sphinx/source/tutorials/list-resources.md b/doc/sphinx/source/tutorials/list-resources.md index 1ebb83ba44..7845e8aa0a 100644 --- a/doc/sphinx/source/tutorials/list-resources.md +++ b/doc/sphinx/source/tutorials/list-resources.md @@ -1,6 +1,6 @@ # How to list the available resources -```eval_rst +```{eval-rst} .. _vp-list: ``` diff --git a/doc/sphinx/source/tutorials/report.md b/doc/sphinx/source/tutorials/report.md index 4b7f432bd3..9b4594f128 100644 --- a/doc/sphinx/source/tutorials/report.md +++ b/doc/sphinx/source/tutorials/report.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _tut_report: ``` diff --git a/doc/sphinx/source/tutorials/run-fit.md b/doc/sphinx/source/tutorials/run-fit.md index dc26ec8d94..673e04e07f 100644 --- a/doc/sphinx/source/tutorials/run-fit.md +++ b/doc/sphinx/source/tutorials/run-fit.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _n3fit-usage: ``` @@ -105,7 +105,7 @@ hyperscan_config: ``` -```eval_rst +```{eval-rst} .. _run-n3fit-fit: ``` @@ -122,7 +122,7 @@ following the points presented above you can proceed with a fit. downloaded automatically. Alternatively they can be obtained with the ``vp-get`` tool. - ```eval_rst + ```{eval-rst} .. note:: This step is not strictly necessary when producing a standard fit with ``n3fit`` but it is required by :ref:`validphys ` @@ -180,7 +180,7 @@ folder, which contains a number of files: -```eval_rst +```{eval-rst} .. _upload-fit: ``` diff --git a/doc/sphinx/source/vp/api.md b/doc/sphinx/source/vp/api.md index eb1e59dda4..dba1c1bf22 100644 --- a/doc/sphinx/source/vp/api.md +++ b/doc/sphinx/source/vp/api.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _vpapi: ``` # Using the validphys API diff --git a/doc/sphinx/source/vp/datthcomp.md b/doc/sphinx/source/vp/datthcomp.md index 4ba5324bc5..d983ce5d10 100644 --- a/doc/sphinx/source/vp/datthcomp.md +++ b/doc/sphinx/source/vp/datthcomp.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _data-theory-comp: ``` diff --git a/doc/sphinx/source/vp/download.md b/doc/sphinx/source/vp/download.md index 1751da0cdc..9743c0f3cc 100644 --- a/doc/sphinx/source/vp/download.md +++ b/doc/sphinx/source/vp/download.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _download: ``` @@ -56,7 +56,7 @@ What can be downloaded The following resources are found automatically: -```eval_rst +```{eval-rst} Fits Fits (specified by the ``fit`` key) can be downloaded if they have previously been uploaded with :ref:`vp-upload `. The corresponding PDF @@ -81,7 +81,7 @@ Theories example for using an existing file instead of computing it. ``` -```eval_rst +```{eval-rst} .. _vp-get: ``` @@ -123,7 +123,7 @@ FitSpec(name='NNPDF31_nlo_as_0118_1000', path=PosixPath('/home/zah/anaconda3/env Downloading resources in code (``validphys.loader``) ---------------------------------------------------- -```eval_rst +```{eval-rst} The automatic download logic is implemented in the :py:mod:`validphys.loader`, specifically by the :py:class:`validphys.loader.RemoteLoader` and :py:class:`validphys.loader.FallbackLoader` classes. diff --git a/doc/sphinx/source/vp/filters.md b/doc/sphinx/source/vp/filters.md index 9b481fb781..db0b2dd875 100644 --- a/doc/sphinx/source/vp/filters.md +++ b/doc/sphinx/source/vp/filters.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _filters: ``` @@ -59,7 +59,7 @@ By default, these filters can have several entries: 5. `local_variables`: (optional) Any additional, non-standard local variables the user wishes to add for this rule only. -```eval_rst +```{eval-rst} .. note:: At least one of :code:`dataset` or :code:`process_type` is required. Additionally, a :code:`rule` entry is always required. @@ -80,7 +80,7 @@ discussed further [here](#PTO). One can see a full list of possible theory param `vp-checktheory ` -```eval_rst +```{eval-rst} .. important:: The :code:`rule` entry should be interpreted as a :code:`str` type within :code:`Python`. As such @@ -113,7 +113,7 @@ I mean by `w2`: w2: Q2 * (1 - x) / x ``` -```eval_rst +```{eval-rst} .. danger:: Defining :code:`local_variables` is non-commutative. The order of definition is important. If a local variable depends on other local variables, then the user must ensure all other @@ -131,7 +131,7 @@ The following would not w2: Q2 * (1 - x) / x w: sqrt(w2) ``` -```eval_rst +```{eval-rst} .. note:: :code:`local_variables` have a local scope. They apply to only the rule within which they are defined. @@ -158,7 +158,7 @@ and note that the relevant `key` is `'FNS'`. Thus to only evaluate my rule if th ``` Similarly, one can add any such theory description `key` into their rule. -```eval_rst +```{eval-rst} .. tip:: Sometimes, we may want to evaluate a rule provided the perturbative order is within a certain range. For example, we may want a rule to be evaluated if the perturbative @@ -196,7 +196,7 @@ filter_rules: rule: x > 0.2 ``` -```eval_rst +```{eval-rst} .. warning:: Adding a :code:`filter_rules` section to the runcard overwrites the default behaviour and does **not** append to the default behaviour. This is done intentionally since a rule cannot be @@ -212,7 +212,7 @@ filter_defaults: ``` As in the case of the rules, this overwrites the original defaults and does not append to them. -```eval_rst +```{eval-rst} .. attention:: To ensure backwards compatibility with old style runcards, if :code:`q2min` and :code:`w2min` are defined under the :code:`datacuts` namespace within the runcard, these values are read in and override the default diff --git a/doc/sphinx/source/vp/nnprofile.md b/doc/sphinx/source/vp/nnprofile.md index 6a4806feb1..dce1332d11 100644 --- a/doc/sphinx/source/vp/nnprofile.md +++ b/doc/sphinx/source/vp/nnprofile.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _nnprofile: ``` @@ -35,7 +35,7 @@ Options The following settings in the profile file are interpreted by different parts of the code. These should be specified in YAML format. -```eval_rst +```{eval-rst} ``data_path`` The path in the user's system where input data such as CommonData files and diff --git a/doc/sphinx/source/vp/upload.md b/doc/sphinx/source/vp/upload.md index d2f156dd3e..0260b612ce 100644 --- a/doc/sphinx/source/vp/upload.md +++ b/doc/sphinx/source/vp/upload.md @@ -1,4 +1,4 @@ -```eval_rst +```{eval-rst} .. _upload: ``` Uploading results to the `validphys` repository @@ -121,7 +121,7 @@ to share. Otherwise use [`vp-upload`](#the-vp-upload-script) after checking the `validphys` will check the SSH connection before doing any work, and it will fail early if it cannot be established. -```eval_rst +```{eval-rst} .. _vpupload: ``` The `vp-upload` script @@ -136,7 +136,7 @@ The output folder is expected to contain the [metadata](#metadata) (e.g. in the form of a `meta.yaml` file). If it doesn't exist or you want to upload and index arbitrary files, use the [`wiki-upload` command](#the-wiki-upload-script). -```eval_rst +```{eval-rst} The script automatically detects (:py:func:`validphys.uploadutils.check_input`) the type of the input. A `fit` is defined to be any folder structure that contains a `filter.yml` file at its root, a `PDF` is any folder containing a `.info` file at the root and a replica 0, and a report is any such structure containing an @@ -144,14 +144,14 @@ folder containing a `.info` file at the root and a replica 0, and a report is an server accordingly. ``` -```eval_rst +```{eval-rst} .. note:: If there is already a fit or PDF on the server with the same name as the fit or PDF you wish to upload, then this command will *not* overwrite the resource that already exists. To overwite such a resource on the server, use the :code:`--force` option. ``` -```eval_rst +```{eval-rst} The code is documented at :py:mod:`validphys.scripts.vp_upload`. ``` @@ -174,7 +174,7 @@ This will cause the user to be prompted for the various metadata fields and the file or folder to be uploaded to the server, together with a generated `meta.yaml` file used for indexing. -```eval_rst +```{eval-rst} The code is documented at :py:mod:`validphys.scripts.wiki_upload`. ``` From dfa2ff7ad1677268d0f3013604e62b82eb9e2f08 Mon Sep 17 00:00:00 2001 From: Zahari Kassabov Date: Thu, 8 Sep 2022 16:25:28 +0100 Subject: [PATCH 2/9] Move legacy md files to rst Writing docs in markdown has long been discouraged and supporting it has some costs in terms of tooling, as it requires less tested and developed extensions to sphnx that constitute extra dependencies. Convert exiting files to rst so we can remove markdown processing altogether. --- doc/sphinx/source/buildmaster.md | 41 - doc/sphinx/source/buildmaster.rst | 53 ++ doc/sphinx/source/ci/index.md | 107 --- doc/sphinx/source/ci/index.rst | 142 ++++ doc/sphinx/source/contributing/git.md | 49 -- doc/sphinx/source/contributing/git.rst | 61 ++ .../source/contributing/python-tools.md | 186 ----- .../source/contributing/python-tools.rst | 206 +++++ doc/sphinx/source/contributing/rules.md | 123 --- doc/sphinx/source/contributing/rules.rst | 143 ++++ doc/sphinx/source/data/plotting_format.md | 304 -------- doc/sphinx/source/data/plotting_format.rst | 311 ++++++++ doc/sphinx/source/external-code/apfelcomb.md | 59 -- doc/sphinx/source/external-code/apfelcomb.rst | 71 ++ doc/sphinx/source/external-code/cross-secs.md | 35 - .../source/external-code/cross-secs.rst | 45 ++ doc/sphinx/source/external-code/grids.md | 32 - doc/sphinx/source/external-code/grids.rst | 43 ++ doc/sphinx/source/external-code/pdf-codes.md | 45 -- doc/sphinx/source/external-code/pdf-codes.rst | 63 ++ doc/sphinx/source/get-started/git.md | 75 -- doc/sphinx/source/get-started/git.rst | 96 +++ doc/sphinx/source/serverconf/index.md | 269 ------- doc/sphinx/source/serverconf/index.rst | 288 +++++++ doc/sphinx/source/theory/theoryparamsinfo.md | 144 ---- doc/sphinx/source/theory/theoryparamsinfo.rst | 148 ++++ doc/sphinx/source/tutorials/APPLgrids.md | 539 ------------- doc/sphinx/source/tutorials/APPLgrids.rst | 708 ++++++++++++++++++ doc/sphinx/source/tutorials/APPLgrids_comp.md | 94 --- .../source/tutorials/APPLgrids_comp.rst | 120 +++ doc/sphinx/source/tutorials/apfelcomb.md | 300 -------- doc/sphinx/source/tutorials/apfelcomb.rst | 364 +++++++++ doc/sphinx/source/tutorials/closuretest.md | 185 ----- doc/sphinx/source/tutorials/closuretest.rst | 211 ++++++ doc/sphinx/source/tutorials/compare-fits.md | 41 - doc/sphinx/source/tutorials/compare-fits.rst | 44 ++ doc/sphinx/source/tutorials/conda.md | 11 - doc/sphinx/source/tutorials/conda.rst | 10 + doc/sphinx/source/tutorials/datthcomp.md | 54 -- doc/sphinx/source/tutorials/datthcomp.rst | 66 ++ doc/sphinx/source/tutorials/list-resources.md | 43 -- .../source/tutorials/list-resources.rst | 45 ++ doc/sphinx/source/tutorials/report.md | 83 -- doc/sphinx/source/tutorials/report.rst | 84 +++ doc/sphinx/source/tutorials/run-fit.md | 258 ------- doc/sphinx/source/tutorials/run-fit.rst | 275 +++++++ doc/sphinx/source/vp/api.md | 155 ---- doc/sphinx/source/vp/api.rst | 168 +++++ doc/sphinx/source/vp/cuts.md | 184 ----- doc/sphinx/source/vp/cuts.rst | 183 +++++ doc/sphinx/source/vp/datthcomp.md | 34 - doc/sphinx/source/vp/datthcomp.rst | 38 + doc/sphinx/source/vp/download.md | 192 ----- doc/sphinx/source/vp/download.rst | 202 +++++ doc/sphinx/source/vp/filters.md | 291 ------- doc/sphinx/source/vp/filters.rst | 344 +++++++++ doc/sphinx/source/vp/nnprofile.md | 89 --- doc/sphinx/source/vp/nnprofile.rst | 91 +++ doc/sphinx/source/vp/upload.md | 191 ----- doc/sphinx/source/vp/upload.rst | 212 ++++++ 60 files changed, 4835 insertions(+), 4213 deletions(-) delete mode 100644 doc/sphinx/source/buildmaster.md create mode 100644 doc/sphinx/source/buildmaster.rst delete mode 100644 doc/sphinx/source/ci/index.md create mode 100644 doc/sphinx/source/ci/index.rst delete mode 100644 doc/sphinx/source/contributing/git.md create mode 100644 doc/sphinx/source/contributing/git.rst delete mode 100644 doc/sphinx/source/contributing/python-tools.md create mode 100644 doc/sphinx/source/contributing/python-tools.rst delete mode 100644 doc/sphinx/source/contributing/rules.md create mode 100644 doc/sphinx/source/contributing/rules.rst delete mode 100644 doc/sphinx/source/data/plotting_format.md create mode 100644 doc/sphinx/source/data/plotting_format.rst delete mode 100644 doc/sphinx/source/external-code/apfelcomb.md create mode 100644 doc/sphinx/source/external-code/apfelcomb.rst delete mode 100644 doc/sphinx/source/external-code/cross-secs.md create mode 100644 doc/sphinx/source/external-code/cross-secs.rst delete mode 100644 doc/sphinx/source/external-code/grids.md create mode 100644 doc/sphinx/source/external-code/grids.rst delete mode 100644 doc/sphinx/source/external-code/pdf-codes.md create mode 100644 doc/sphinx/source/external-code/pdf-codes.rst delete mode 100644 doc/sphinx/source/get-started/git.md create mode 100644 doc/sphinx/source/get-started/git.rst delete mode 100644 doc/sphinx/source/serverconf/index.md create mode 100644 doc/sphinx/source/serverconf/index.rst delete mode 100644 doc/sphinx/source/theory/theoryparamsinfo.md create mode 100644 doc/sphinx/source/theory/theoryparamsinfo.rst delete mode 100644 doc/sphinx/source/tutorials/APPLgrids.md create mode 100644 doc/sphinx/source/tutorials/APPLgrids.rst delete mode 100644 doc/sphinx/source/tutorials/APPLgrids_comp.md create mode 100644 doc/sphinx/source/tutorials/APPLgrids_comp.rst delete mode 100644 doc/sphinx/source/tutorials/apfelcomb.md create mode 100644 doc/sphinx/source/tutorials/apfelcomb.rst delete mode 100644 doc/sphinx/source/tutorials/closuretest.md create mode 100644 doc/sphinx/source/tutorials/closuretest.rst delete mode 100644 doc/sphinx/source/tutorials/compare-fits.md create mode 100644 doc/sphinx/source/tutorials/compare-fits.rst delete mode 100644 doc/sphinx/source/tutorials/conda.md create mode 100644 doc/sphinx/source/tutorials/conda.rst delete mode 100644 doc/sphinx/source/tutorials/datthcomp.md create mode 100644 doc/sphinx/source/tutorials/datthcomp.rst delete mode 100644 doc/sphinx/source/tutorials/list-resources.md create mode 100644 doc/sphinx/source/tutorials/list-resources.rst delete mode 100644 doc/sphinx/source/tutorials/report.md create mode 100644 doc/sphinx/source/tutorials/report.rst delete mode 100644 doc/sphinx/source/tutorials/run-fit.md create mode 100644 doc/sphinx/source/tutorials/run-fit.rst delete mode 100644 doc/sphinx/source/vp/api.md create mode 100644 doc/sphinx/source/vp/api.rst delete mode 100644 doc/sphinx/source/vp/cuts.md create mode 100644 doc/sphinx/source/vp/cuts.rst delete mode 100644 doc/sphinx/source/vp/datthcomp.md create mode 100644 doc/sphinx/source/vp/datthcomp.rst delete mode 100644 doc/sphinx/source/vp/download.md create mode 100644 doc/sphinx/source/vp/download.rst delete mode 100644 doc/sphinx/source/vp/filters.md create mode 100644 doc/sphinx/source/vp/filters.rst delete mode 100644 doc/sphinx/source/vp/nnprofile.md create mode 100644 doc/sphinx/source/vp/nnprofile.rst delete mode 100644 doc/sphinx/source/vp/upload.md create mode 100644 doc/sphinx/source/vp/upload.rst diff --git a/doc/sphinx/source/buildmaster.md b/doc/sphinx/source/buildmaster.md deleted file mode 100644 index dbb9f9d6bf..0000000000 --- a/doc/sphinx/source/buildmaster.md +++ /dev/null @@ -1,41 +0,0 @@ -```{eval-rst} -.. _buildmaster: -``` - -## Handling experimental data: Buildmaster - -Buildmaster is the code that allows the user to generate the ``DATA`` and -``SYSTYPE`` files that contain, respectively, the experimental data and the -information pertaining to the treatment of systematic errors. It is available -as a folder within the nnpdf project -``` -https://github.com/NNPDF/nnpdf/buildmaster -``` -The structure of the files generated by the buildmaster project -is documented in [Experimental data files](exp_data_files). - -Once the remainder of the nnpdf project is compiled and installed, the buildmaster code can -be compiled and installed as follows -``` -mkdir build -cd build -cmake .. -make -j && make install -``` -This will generate the `buildmaster` executable which shall be run as -``` -./buildmaster -``` -The `DATA_` and `SYSTYPE_` files will be generated, respectively, in -``` -results/ -results/systypes/ -``` -They should then be copied into -``` -nnpdf/nnpdfcpp/data/commondata -nnpdf/nnpdfcpp/data/commondata/systypes/ -``` -Whenever a new data set is implemented, the buildmaster code should be -updated accordingly. Detailed instructions on how to do so are provided in -the tutorial [Implementing a new experiment in buildmaster](../tutorials/buildmaster.md). diff --git a/doc/sphinx/source/buildmaster.rst b/doc/sphinx/source/buildmaster.rst new file mode 100644 index 0000000000..b3c3597bab --- /dev/null +++ b/doc/sphinx/source/buildmaster.rst @@ -0,0 +1,53 @@ +.. code:: {eval-rst} + + .. _buildmaster: + +Handling experimental data: Buildmaster +--------------------------------------- + +Buildmaster is the code that allows the user to generate the ``DATA`` +and ``SYSTYPE`` files that contain, respectively, the experimental data +and the information pertaining to the treatment of systematic errors. It +is available as a folder within the nnpdf project + +:: + + https://github.com/NNPDF/nnpdf/buildmaster + +The structure of the files generated by the buildmaster project is +documented in `Experimental data files `__. + +Once the remainder of the nnpdf project is compiled and installed, the +buildmaster code can be compiled and installed as follows + +:: + + mkdir build + cd build + cmake .. + make -j && make install + +This will generate the ``buildmaster`` executable which shall be run as + +:: + + ./buildmaster + +The ``DATA_`` and ``SYSTYPE_`` files will be generated, respectively, in + +:: + + results/ + results/systypes/ + +They should then be copied into + +:: + + nnpdf/nnpdfcpp/data/commondata + nnpdf/nnpdfcpp/data/commondata/systypes/ + +Whenever a new data set is implemented, the buildmaster code should be +updated accordingly. Detailed instructions on how to do so are provided +in the tutorial `Implementing a new experiment in +buildmaster <../tutorials/buildmaster.md>`__. diff --git a/doc/sphinx/source/ci/index.md b/doc/sphinx/source/ci/index.md deleted file mode 100644 index 8c5faa293f..0000000000 --- a/doc/sphinx/source/ci/index.md +++ /dev/null @@ -1,107 +0,0 @@ -```{eval-rst} -.. _CI: -``` -# Continuous integration and deployment - -The NNPDF code base makes use of externally hosted services, to aid development -and testing. These are typically called *Continuous integration (CI)* or -*Continuous deployment* services. Their main task is to execute automated tests -on the code and produce [binary builds](conda) which allow it to be -automatically deployed. The services are configured so that they react to -[git](git) pushes to the GitHub server. - -Currently we are using actively [GitHub Actions](https://help.github.com/en/actions). -In the past, the [Travis CI](https://travis-ci.com/) service was used, but owing to timeout failures on Mac we have decided to move the CI to GitHub Actions. -The [Gitlab CI service hosted at -CERN](https://gitlab.cern.ch/) was also used in the past, but support was -discontinued due to the burden of requiring everyone to have a CERN account. - -Furthermore, we implement a self-hosted runner for GitHub Actions for long duration workflows, such as running a full fit pipeline. - -## Operation of CI tools - -Our CI service works roughly as follows: - - 1. Every time a commit is made to a given branch, a request to process the - code in that branch is made automatically. - 2. The code for the branch is downloaded to the CI server, and some action is - taken based on configuration found both in the git repository itself and in - the settings on the CI service. These actions include: - * Compiling the code. - * Running the tests. - * Possibly, uploading the compiled binaries and documentation to the - [NNPDF server](server). - We use [Conda-build](https://docs.conda.io/projects/conda-build/en/latest/) - to - do much of the heavy lifting for these actions. - 3. The CI service reports whether it has *succeeded* or *failed* to the GitHub - server, which displays that information next to the relevant pull request or - commit. Some logs are generated, which can aid in determining the cause of - errors. - 4. Generate docker images for tag releases with ready-to-run NNPDF installation. - -The progress reports of the various jobs at GitHub Actions, as well as the -corresponding logs are available at , upon logging in -with an authorized GitHub account. - - -## Configuration of GitHub Actions - -GitHub Actions uses both files found in the NNPDF repository and settings stored in the tab `Settings -> Secrets` of the repository itself. - -### Secrets stored in the GitHub Actions configuration - -To build and upload the packages GitHub Actions needs to be able to access some -secrets, which should not be stored in the git repository. These are represented -as environment variables, under the [secrets for the NNPDF -repository](https://github.com/NNPDF/nnpdf/settings/secrets). The secrets are encoded -using `base64` for simplicity. To use -them, do something like `echo "$" | base64 --decode`. - -The secrets are. - - - `NETRC_FILE` a base64 encoded string containing a `~/.netrc` file with - the [credentials](server-access) to the private conda repository - - - `NNPDF_SSH_KEY` a base64 string containing a private SSH key which is - authorized to access the [upload account of the NNPDF server](server-access). - -### Repository configuration - -The entry point for GitHub Actions are yaml rules files that can be found in the -[`.github/workflows/`](https://github.com/NNPDF/nnpdf/blob/master/.github/workflows/) folder. -They specify which operating systems and versions are tested, which -versions of Python, some environment variables, and command instructions for linux and macos. The commands basically call `conda build` and upload the relevant packages if required. - -By default only packages corresponding to commits to the master branch get -uploaded. For other branches, the build and testing happens, but the results are -discarded in the end. This behavior can be changed by (temporarily) commenting the lines starting with `if: github.ref == 'refs/heads/master'` in the `.github/workflows/rules.yml` file. This can be -useful to test modifications to the uploading. - -When a new tag is created the github action stored in -`.github/workflows/docker.yml` is executed. This action generates a new docker -image containing the tagged code. This docker image is then uploaded to the -[NNPDF GitHub Package -registry](https://github.com/NNPDF/nnpdf/pkgs/container/nnpdf). Finally, the -action stores the conda environment file created during the installation process -and opens a pull request placing the file inside `n3fit/runcards`. This feature -allows recovering the code and results obtained with specific tag of the code. - -## Operation of automatic fit bot - -Our GitHub Action service implements: - - 1. Every time the label `run-fit-bot` is added to a pull request, a request to process the code in that branch is made automatically. - 2. The code for the branch is downloaded to the CI self-hosted runner, and some action is - taken based on configuration found both in the git repository itself in `.github/workflow/rules.yml`. These actions include: - * Compiling and installing the code. - * Running a complete fit using the `n3fit/runcards/development.yml` runcard. - * Produces a report and upload results to the [NNPDF server](server). - 3. The CI service reports whether it has *succeeded* or *failed* to the GitHub - server, which displays that information next to the relevant pull request or - commit. Some logs are generated, which can aid in determining the cause of - errors. - 4. If the workflow succeeds, a comment to the initial pull request will appear with link references to the generated report and fit. - -The progress reports of the various jobs at [GitHub Actions](https://github.com/NNPDF/actions), upon logging in -with an authorized GitHub account. diff --git a/doc/sphinx/source/ci/index.rst b/doc/sphinx/source/ci/index.rst new file mode 100644 index 0000000000..6c0f566799 --- /dev/null +++ b/doc/sphinx/source/ci/index.rst @@ -0,0 +1,142 @@ +.. code:: {eval-rst} + + .. _CI: + +Continuous integration and deployment +===================================== + +The NNPDF code base makes use of externally hosted services, to aid +development and testing. These are typically called *Continuous +integration (CI)* or *Continuous deployment* services. Their main task +is to execute automated tests on the code and produce `binary +builds `__ which allow it to be automatically deployed. The +services are configured so that they react to `git `__ pushes to +the GitHub server. + +Currently we are using actively `GitHub +Actions `__. In the past, the +`Travis CI `__ service was used, but owing to +timeout failures on Mac we have decided to move the CI to GitHub +Actions. The `Gitlab CI service hosted at +CERN `__ was also used in the past, but support +was discontinued due to the burden of requiring everyone to have a CERN +account. + +Furthermore, we implement a self-hosted runner for GitHub Actions for +long duration workflows, such as running a full fit pipeline. + +Operation of CI tools +--------------------- + +Our CI service works roughly as follows: + +1. Every time a commit is made to a given branch, a request to process + the code in that branch is made automatically. +2. The code for the branch is downloaded to the CI server, and some + action is taken based on configuration found both in the git + repository itself and in the settings on the CI service. These + actions include: + + - Compiling the code. + - Running the tests. + - Possibly, uploading the compiled binaries and documentation to the + `NNPDF server `__. We use + `Conda-build `__ + to do much of the heavy lifting for these actions. + +3. The CI service reports whether it has *succeeded* or *failed* to the + GitHub server, which displays that information next to the relevant + pull request or commit. Some logs are generated, which can aid in + determining the cause of errors. +4. Generate docker images for tag releases with ready-to-run NNPDF + installation. + +The progress reports of the various jobs at GitHub Actions, as well as +the corresponding logs are available at +https://github.com/NNPDF/nnpdf/actions, upon logging in with an +authorized GitHub account. + +Configuration of GitHub Actions +------------------------------- + +GitHub Actions uses both files found in the NNPDF repository and +settings stored in the tab ``Settings -> Secrets`` of the repository +itself. + +Secrets stored in the GitHub Actions configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To build and upload the packages GitHub Actions needs to be able to +access some secrets, which should not be stored in the git repository. +These are represented as environment variables, under the `secrets for +the NNPDF +repository `__. The +secrets are encoded using ``base64`` for simplicity. To use them, do +something like ``echo "$" | base64 --decode``. + +The secrets are. + +- ``NETRC_FILE`` a base64 encoded string containing a ``~/.netrc`` file + with the `credentials `__ to the private conda + repository https://packages.nnpdf.science/conda-private/ +- ``NNPDF_SSH_KEY`` a base64 string containing a private SSH key which + is authorized to access the `upload account of the NNPDF + server `__. + +Repository configuration +~~~~~~~~~~~~~~~~~~~~~~~~ + +The entry point for GitHub Actions are yaml rules files that can be +found in the +```.github/workflows/`` `__ +folder. They specify which operating systems and versions are tested, +which versions of Python, some environment variables, and command +instructions for linux and macos. The commands basically call +``conda build`` and upload the relevant packages if required. + +By default only packages corresponding to commits to the master branch +get uploaded. For other branches, the build and testing happens, but the +results are discarded in the end. This behavior can be changed by +(temporarily) commenting the lines starting with +``if: github.ref == 'refs/heads/master'`` in the +``.github/workflows/rules.yml`` file. This can be useful to test +modifications to the uploading. + +When a new tag is created the github action stored in +``.github/workflows/docker.yml`` is executed. This action generates a +new docker image containing the tagged code. This docker image is then +uploaded to the `NNPDF GitHub Package +registry `__. +Finally, the action stores the conda environment file created during the +installation process and opens a pull request placing the file inside +``n3fit/runcards``. This feature allows recovering the code and results +obtained with specific tag of the code. + +Operation of automatic fit bot +------------------------------ + +Our GitHub Action service implements: + +1. Every time the label ``run-fit-bot`` is added to a pull request, a + request to process the code in that branch is made automatically. +2. The code for the branch is downloaded to the CI self-hosted runner, + and some action is taken based on configuration found both in the git + repository itself in ``.github/workflow/rules.yml``. These actions + include: + + - Compiling and installing the code. + - Running a complete fit using the + ``n3fit/runcards/development.yml`` runcard. + - Produces a report and upload results to the `NNPDF + server `__. + +3. The CI service reports whether it has *succeeded* or *failed* to the + GitHub server, which displays that information next to the relevant + pull request or commit. Some logs are generated, which can aid in + determining the cause of errors. +4. If the workflow succeeds, a comment to the initial pull request will + appear with link references to the generated report and fit. + +The progress reports of the various jobs at `GitHub +Actions `__, upon logging in with an +authorized GitHub account. diff --git a/doc/sphinx/source/contributing/git.md b/doc/sphinx/source/contributing/git.md deleted file mode 100644 index 2dc32f599e..0000000000 --- a/doc/sphinx/source/contributing/git.md +++ /dev/null @@ -1,49 +0,0 @@ -```{eval-rst} -.. _git: -``` -# Git, GitHub and GitLab - - -## Git - -[Git](https://git-scm.com/) is the version control system adopted within the NNPDF Collaboration. -Among other things, Git allows multiple people to edit code simultaneously; it allows users to -track changes to the code, i.e. a user can see who edited what and when; and, it allows a user to -view any past version of the code. The latter two points are particularly useful for running tests -should a bug be introduced to the code, for example. - -### Learning Git - -Using Git can be slightly confusing at first, but by mastering a few basic commands you will be able -to do most of the tasks that you need to do day-to-day. Git also has the advantage that at the -moment it is probably the most popular version control system out there, so any time you in invest -in learning Git will most likely be useful for projects outside of NNPDF. Many online tutorials and -guides exist for learning Git, but here are two that I have used before: a [quick -guide](http://rogerdudler.github.io/git-guide/) and a more in depth -[tutorial](https://www.codecademy.com/learn/learn-git). The [official -documentation](https://git-scm.com/docs) might be useful as well. - - -### GitHub development workflow - -GitHub provides the following workflow: - -* Users can create Projects and Milestones for each project. - -* For each project users can open issues, which can be used to request bug fixes, new features, new -documentation, or simply to facilitate a general discussion. The user can then assign one or more -people to help deal with the issue. Issues should be opened in the relevant repository. For -example, for something that is related to validphys, one should open an issue in -[nnpdf](https://github.com/NNPDF/nnpdf), while for something that is related to data -implementation, one should open an issue in [buildmaster](https://github.com/NNPDF/buildmaster). -Example issues can be found [here](https://github.com/NNPDF/nnpdf/issues). - -* When it is clear how an issue should be dealt with, a -[branch](https://github.com/NNPDF/nnpdf/branches) can be opened where a user can implement the -requested feature. - -* Once a feature is ready to be considered for merging into the master version of the code, a [pull -request](https://github.com/NNPDF/nnpdf/pulls) (PR) can be opened. At least two code reviewers -must then be assigned, after which the code will be reviewed and discussed. The modification will -then be accepted or rejected. Further general information on PRs can found -[here](https://help.github.com/en/articles/about-pull-requests). diff --git a/doc/sphinx/source/contributing/git.rst b/doc/sphinx/source/contributing/git.rst new file mode 100644 index 0000000000..d88249d2c0 --- /dev/null +++ b/doc/sphinx/source/contributing/git.rst @@ -0,0 +1,61 @@ +.. code:: {eval-rst} + + .. _git: + +Git, GitHub and GitLab +====================== + +Git +--- + +`Git `__ is the version control system adopted +within the NNPDF Collaboration. Among other things, Git allows multiple +people to edit code simultaneously; it allows users to track changes to +the code, i.e. a user can see who edited what and when; and, it allows a +user to view any past version of the code. The latter two points are +particularly useful for running tests should a bug be introduced to the +code, for example. + +Learning Git +~~~~~~~~~~~~ + +Using Git can be slightly confusing at first, but by mastering a few +basic commands you will be able to do most of the tasks that you need to +do day-to-day. Git also has the advantage that at the moment it is +probably the most popular version control system out there, so any time +you in invest in learning Git will most likely be useful for projects +outside of NNPDF. Many online tutorials and guides exist for learning +Git, but here are two that I have used before: a `quick +guide `__ and a more in depth +`tutorial `__. The `official +documentation `__ might be useful as well. + +GitHub development workflow +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +GitHub provides the following workflow: + +- Users can create Projects and Milestones for each project. + +- For each project users can open issues, which can be used to request + bug fixes, new features, new documentation, or simply to facilitate a + general discussion. The user can then assign one or more people to + help deal with the issue. Issues should be opened in the relevant + repository. For example, for something that is related to validphys, + one should open an issue in + `nnpdf `__, while for something that + is related to data implementation, one should open an issue in + `buildmaster `__. Example + issues can be found `here `__. + +- When it is clear how an issue should be dealt with, a + `branch `__ can be opened + where a user can implement the requested feature. + +- Once a feature is ready to be considered for merging into the master + version of the code, a `pull + request `__ (PR) can be opened. + At least two code reviewers must then be assigned, after which the + code will be reviewed and discussed. The modification will then be + accepted or rejected. Further general information on PRs can found + `here `__. diff --git a/doc/sphinx/source/contributing/python-tools.md b/doc/sphinx/source/contributing/python-tools.md deleted file mode 100644 index d2842edd0d..0000000000 --- a/doc/sphinx/source/contributing/python-tools.md +++ /dev/null @@ -1,186 +0,0 @@ -```{eval-rst} -.. _pytools: -``` - -# Tools for developing with the Python programming language - -This page summarizes auxiliary Python tools that we commonly use to develop the -project. Note that this page is meant to be a quick index. Consult the -documentation on each specific tool for details. - - -## Python editors - - - The [Spyder editor](https://www.spyder-ide.org/) is good for getting started - with scientific Python coding, because of various inspection and interactive - features. - - [`vscode`](https://code.visualstudio.com/) is a more full featured editor. - - In the long run, the most efficient approach is to learn a terminal based - editor such as [`vim`](https://www.vim.org/). Note that `vim` editing modes can be - added as extensions to graphical editors such as `vscode`. - - -## Interactive development - -Python code can be evaluated interactively, which can speed up the development. - - - [IPython shell](https://ipython.org/): It is notably nicer to use than the - standard interactive interpreter. - - [Jupyter notebook](https://jupyter.org/): Interactive development - environment running on the browser. Useful for bigger experiments. - -```{eval-rst} -.. note:: - When developing :ref:`validphys ` related code interactively, be - sure to read about the :ref:`API functionality `. -``` - -## Testing - - - [`pytest`](https://docs.pytest.org/en/latest/): It is a framework for - writing and running tests. It finds tests in the codebase (basically - modules and functions that start with `test`), enhances the `assert` - statement to provide rich error reporting and allows to structure - dependencies between the tests (in a way similar to `reportengine`). - Tests are stored in the codebase and executed by pytest either manually or - as a part of the continuous integration process. - - [`coverage.py`](https://coverage.readthedocs.io/en/coverage-5.2.1/) is a - program that traces which lines of code have been executed when a given - Python program (notably pytest) is running. The main use case is to verify - that tests probe our code paths. - - - -```{eval-rst} -.. _pytoolsqa: -``` -## Code quality and reviewing - -See also [*Reviewing pull requests*](reviews). Note that these can typically be -integrated with your editor of choice. - - - The [`pylint`](https://www.pylint.org/) tool allows for the catching of - common problems in Python code. The top level - [`.pylintrc` file](https://github.com/NNPDF/nnpdf/blob/master/.pylintrc) - comes with a useful and not overly noisy configuration. - - The [`black` code formatter](https://github.com/psf/black) runs almost - without configuration and produces typically good results. It is good to run - it by default, to avoid spending time on formatting (or arguing about it). - -## Debugging - -Usually the most efficient way to debug a piece of Python code, such as a -`validphys` action is to insert `print` statements to check the state at various -places in the code. A few alternatives exists when that is not enough: - - - [IPython embed](https://ipython.readthedocs.io/en/stable/api/generated/IPython.terminal.embed.html): - The [IPython](https://ipython.org/) shell can be easily dropped at any - arbitrary point in the code. Write - ```python - import IPython - IPython.embed() - ``` - at the location of the code you want to debug. You will then be able to - query (and manipulate) the state of the code using a rich shell. - - - PDB: The standard [Python debugger](https://docs.python.org/3/library/pdb.html) - can be used as an alternative. Compared to `IPython` it has the advantage that - it allows to automatically step in the execution of the code, but the disadvantage - that the interface is somewhat more complex and often surprising (hint: always - [prefix interpreter commands with `!`](https://docs.python.org/3/library/pdb.html#pdbcommand-!)). - -## Performance profiling - -Sometimes a piece of code runs slower than expected. The reasons can often be -surprising. It is a good idea to measure where the problems actually are. - - - [`py-spy`](https://github.com/benfred/py-spy): A performance measuring - program (*profiler*) that provides good information and little overhead. - Prefer it to the standard `cProfile`. The output is typically presented in - the form of "Flamegraphs" that show the relative time spent on each piece of - code. - -## Documentation - - - We use the [Sphinx tool](https://www.sphinx-doc.org/) to document code - projects. It can render and organize special purpose documentation files as - well as read Python source files to automatically document interfaces. It - supports extensive customization and plugins. In particular because the - default formatting for docstrings is somewhat unwieldy, it is recommended - to enable the `napoleon` extension which allows for a more lenient - [`numpydoc`](https://numpydoc.readthedocs.io/en/latest/format.html) style. - Similarly the default RST markup language can be overwhelming for simple - documents. We enable the - [recommonmark](https://recommonmark.readthedocs.io/en/latest/) extension to - be able to compose files also in markdown format. - -## Python static checks and code style - -We use [Pylint](https://www.pylint.org/) to provide static checking e.g. -finding basic errors that a compiler would catch in compiled languages. An example -is using an unknown variable name. Pylint also provides basic guidelines on the -structure of the code (e.g. avoid functions that are to complicated). Because -Pylint is way too pedantic by default, we limit the checks to only those -considered useful. The `.pylintrc` file at the top level configures Pylint to -only mind those checks. Most Python IDEs and editors have some kind of support -for Pylint. It is strongly recommended to configure the editor to show the -problematic pieces of code proactively. - -New code should use the [Black](https://black.readthedocs.io/en/stable/>) tool to -format the code. This tool should not be used to aggressively reformat existing -files. - - -## Matplotlib Image Comparison Tests - -It is possible to create tests which perform an image comparison between a -generated plot and a pre-existing baseline plot. Clearly this allows one to check -consistency in figure generation. - -Before beginning you will need to ensure that you have the tests dependencies, -which can be checked in `nnpdf/conda-recipe/meta.yml`. - -The next step is to write the test function. It is highly recommended to use the -[validphys API](../vp/api.md) for this, both to simplify the code and to make it agnostic to the -structure of backend providers - provided that they produce the same results. See -for example a function which tests the `plot_pdfs` provider: - -```python -@pytest.mark.mpl_image_compare -def test_plotpdfs(): - pdfs = ['NNPDF31_nnlo_as_0118'] - Q = 10 - flavours = ['g'] - #plot_pdfs returns a generator with (figure, name_hint) - return next(API.plot_pdfs(pdfs=pdfs, Q=Q, flavours=flavours))[0] -``` - -We see that the function needs to return a valid matplotlib figure, and should -be decorated with `@pytest.mark.mpl_image_compare`. - -Now the baseline figure needs to be generated, this can be achieved by running - -``` -pytest -k --mpl-generate-path=baseline -``` - -which will generate a PNG of the figure in the `src/validphys/tests/baseline` -directory. It is recommended to put all baseline plots in this directory so that -they are automatically installed, and so will be in the correct location when -the [CI](../ci/index.md) runs the test suite. - -Now that the baseline figure exists you can check that your test works: - -``` -pytest -k --mpl -``` - -Also you can check that the test has been added to the full test suite: - -``` -pytest --pyargs --mpl validphys -``` - -Just note that if you do not put the `--mpl` flag then the test will just check -that the function runs without error, and won't check that the output matches to -baseline. diff --git a/doc/sphinx/source/contributing/python-tools.rst b/doc/sphinx/source/contributing/python-tools.rst new file mode 100644 index 0000000000..1ae7a93546 --- /dev/null +++ b/doc/sphinx/source/contributing/python-tools.rst @@ -0,0 +1,206 @@ +.. code:: {eval-rst} + + .. _pytools: + +Tools for developing with the Python programming language +========================================================= + +This page summarizes auxiliary Python tools that we commonly use to +develop the project. Note that this page is meant to be a quick index. +Consult the documentation on each specific tool for details. + +Python editors +-------------- + +- The `Spyder editor `__ is good for + getting started with scientific Python coding, because of various + inspection and interactive features. +- ```vscode`` `__ is a more full + featured editor. +- In the long run, the most efficient approach is to learn a terminal + based editor such as ```vim`` `__. Note that + ``vim`` editing modes can be added as extensions to graphical editors + such as ``vscode``. + +Interactive development +----------------------- + +Python code can be evaluated interactively, which can speed up the +development. + +- `IPython shell `__: It is notably nicer to use + than the standard interactive interpreter. +- `Jupyter notebook `__: Interactive development + environment running on the browser. Useful for bigger experiments. + +.. code:: {eval-rst} + + .. note:: + When developing :ref:`validphys ` related code interactively, be + sure to read about the :ref:`API functionality `. + +Testing +------- + +- ```pytest`` `__: It is a + framework for writing and running tests. It finds tests in the + codebase (basically modules and functions that start with ``test``), + enhances the ``assert`` statement to provide rich error reporting and + allows to structure dependencies between the tests (in a way similar + to ``reportengine``). Tests are stored in the codebase and executed + by pytest either manually or as a part of the continuous integration + process. +- ```coverage.py`` `__ + is a program that traces which lines of code have been executed when + a given Python program (notably pytest) is running. The main use case + is to verify that tests probe our code paths. + +.. code:: {eval-rst} + + .. _pytoolsqa: + +Code quality and reviewing +-------------------------- + +See also `Reviewing pull requests `__. Note that these can +typically be integrated with your editor of choice. + +- The ```pylint`` `__ tool allows for the + catching of common problems in Python code. The top level + ```.pylintrc`` + file `__ comes + with a useful and not overly noisy configuration. +- The ```black`` code formatter `__ runs + almost without configuration and produces typically good results. It + is good to run it by default, to avoid spending time on formatting + (or arguing about it). + +Debugging +--------- + +Usually the most efficient way to debug a piece of Python code, such as +a ``validphys`` action is to insert ``print`` statements to check the +state at various places in the code. A few alternatives exists when that +is not enough: + +- `IPython + embed `__: + The `IPython `__ shell can be easily dropped at + any arbitrary point in the code. Write + ``python import IPython IPython.embed()`` at the location of + the code you want to debug. You will then be able to query (and + manipulate) the state of the code using a rich shell. + +- PDB: The standard `Python + debugger `__ can be used + as an alternative. Compared to ``IPython`` it has the advantage that + it allows to automatically step in the execution of the code, but the + disadvantage that the interface is somewhat more complex and often + surprising (hint: always `prefix interpreter commands with + ``!`` `__). + +Performance profiling +--------------------- + +Sometimes a piece of code runs slower than expected. The reasons can +often be surprising. It is a good idea to measure where the problems +actually are. + +- ```py-spy`` `__: A performance + measuring program (*profiler*) that provides good information and + little overhead. Prefer it to the standard ``cProfile``. The output + is typically presented in the form of “Flamegraphs” that show the + relative time spent on each piece of code. + +Documentation +------------- + +- We use the `Sphinx tool `__ to document + code projects. It can render and organize special purpose + documentation files as well as read Python source files to + automatically document interfaces. It supports extensive + customization and plugins. In particular because the default + formatting for docstrings is somewhat unwieldy, it is recommended to + enable the ``napoleon`` extension which allows for a more lenient + ```numpydoc`` `__ + style. Similarly the default RST markup language can be overwhelming + for simple documents. We enable the + `recommonmark `__ + extension to be able to compose files also in markdown format. + +Python static checks and code style +----------------------------------- + +We use `Pylint `__ to provide static checking +e.g. finding basic errors that a compiler would catch in compiled +languages. An example is using an unknown variable name. Pylint also +provides basic guidelines on the structure of the code (e.g. avoid +functions that are to complicated). Because Pylint is way too pedantic +by default, we limit the checks to only those considered useful. The +``.pylintrc`` file at the top level configures Pylint to only mind those +checks. Most Python IDEs and editors have some kind of support for +Pylint. It is strongly recommended to configure the editor to show the +problematic pieces of code proactively. + +New code should use the +`Black `__ tool to format +the code. This tool should not be used to aggressively reformat existing +files. + +Matplotlib Image Comparison Tests +--------------------------------- + +It is possible to create tests which perform an image comparison between +a generated plot and a pre-existing baseline plot. Clearly this allows +one to check consistency in figure generation. + +Before beginning you will need to ensure that you have the tests +dependencies, which can be checked in ``nnpdf/conda-recipe/meta.yml``. + +The next step is to write the test function. It is highly recommended to +use the `validphys API <../vp/api.md>`__ for this, both to simplify the +code and to make it agnostic to the structure of backend providers - +provided that they produce the same results. See for example a function +which tests the ``plot_pdfs`` provider: + +.. code:: python + + @pytest.mark.mpl_image_compare + def test_plotpdfs(): + pdfs = ['NNPDF31_nnlo_as_0118'] + Q = 10 + flavours = ['g'] + #plot_pdfs returns a generator with (figure, name_hint) + return next(API.plot_pdfs(pdfs=pdfs, Q=Q, flavours=flavours))[0] + +We see that the function needs to return a valid matplotlib figure, and +should be decorated with ``@pytest.mark.mpl_image_compare``. + +Now the baseline figure needs to be generated, this can be achieved by +running + +:: + + pytest -k --mpl-generate-path=baseline + +which will generate a PNG of the figure in the +``src/validphys/tests/baseline`` directory. It is recommended to put all +baseline plots in this directory so that they are automatically +installed, and so will be in the correct location when the +`CI <../ci/index.md>`__ runs the test suite. + +Now that the baseline figure exists you can check that your test works: + +:: + + pytest -k --mpl + +Also you can check that the test has been added to the full test suite: + +:: + + pytest --pyargs --mpl validphys + +Just note that if you do not put the ``--mpl`` flag then the test will +just check that the function runs without error, and won’t check that +the output matches to baseline. diff --git a/doc/sphinx/source/contributing/rules.md b/doc/sphinx/source/contributing/rules.md deleted file mode 100644 index ab29db0728..0000000000 --- a/doc/sphinx/source/contributing/rules.md +++ /dev/null @@ -1,123 +0,0 @@ -```{eval-rst} -.. _rules: -``` -# Code development - -Code development is carried out using Github. -For more information on the Git workflow that NNPDF adopts, see the [Git and GitHub](./git.md) section. - -## Code contributions - -Code contributions should be presented in the form of [Pull -Requests](https://github.com/NNPDF/nnpdf/pulls)(PRs) to the repository. -Avoid committing modifications directly to the master version of the code. Instead, -create a new branch and make modifications on it. - -This PR should adhere to the following rules: - -* **A clear explanation of the aims of the PR** should be given, i.e. what issue(s) are you trying to -address? If the reason for the PR has already been detailed in an issue, then this issue should be -linked in the PR. - -* The PR should contain **[documentation](./sphinx-documentation.md) describing - the new features**, if applicable. - -* If the PR is fixing a bug, information should be given such that a reviewer can reproduce the bug. - -* The PR should have **at least one developer assigned to it**, whose task it is to [review](reviews) the -code. The PR cannot be merged into master before the reviewer has approved it. - -* Before a PR can be merged into master, the **Travis build for it must pass** (see [here](../ci/index.md)). -Practically, this means that you should find a green tick next to your PR on the relevant [PR -page](https://github.com/NNPDF/nnpdf/pulls). If you instead find a red cross next to your PR, the -reason for the failure must be investigated and dealt with appropriately. - -* When writing examples, please use the recommended resources detailed -[here](vpexamples). - -## Example pull request - -You may find it instructive to go though this pull request that -implements new convolution methods: - - - -It demonstrates how to add a new feature, together with relevant tests and -documentation, and refine it based on the discussion. - - -```{eval-rst} -.. _reviews: -``` -## Reviewing pull requests - -All changes to the code [should](rules) be reviewed by at least one person (and ideally -at least two). The expected benefits of the policy are: - - - It should improve the overall quality of the code. - - - It should provide the author of the change with a reasonably quick feedback - loop to discuss the technical details involved in the changes. - - - It should make at least two people (the author and the reviewer) familiar - with the changes. It should also ensure that the changes are easy to read - and maintain in the future, and conform to the structure of the rest of the - project. - -### Guidelines for reviewing - -The following approach has been found helpful for reviewers, when reviewing pull -requests: - - - Make sure you actually understand what the changes are about. Unclear - details should not pass code review. Ask for clarifications, documentation, - and changes in the code that make it more clear. If you are not in the - position of taking the time, consider asking somebody else to help reviewing - the changes. If the changes are big and difficult to comprehend at once, - consider requesting that the author breaks them down in easier to - understand, self contained, pull requests. Note that it is for the authors - to proactively discuss the proposed changes before they become too difficult - for anyone else to follow, and, failing that, it is fair to ask them to go - through the work of making them intelligible. - - - Look at the big picture first. Think about whether the overall idea and - implementation is sound or instead could benefit from going in a different - direction. Ideally before a lot of work has gone into fine tuning details. - - - - Review the code in detail. Try to identify areas where the changes - could be clearly improved in terms of clarity, speed or style. Consider - implementing minor changes yourself, although note that there are - trade-offs: People are more likely to assimilate good patterns if they - implement them a few times, which may be a win long term, even if it takes - longer to ship this particular code change. - - - Ideally changes should come with automatic tests supporting their - correctness. - - - Use [automated tools](pytoolsqa) which could catch a few extra - problems. In particular - * Do look at the automated tests that run with the PR. - New code should not break them. - * Use [`pylint`](https://www.pylint.org/) with [our default - configuration](https://github.com/NNPDF/nnpdf/blob/master/.pylintrc) to - catch common problems with Python code. - * New Python code should come formatted with - [`black` tool](https://github.com/psf/black). - * Changes in compiled code should be tested in debug mode, with - the address sanitizer enabled. This is done with the - `-DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON` options in `cmake`. - - - Regardless of automated tests, always run code with the new changes - manually. This gives great insight into possible pitfalls and areas of - improvement. - - - Make sure the changes are appropriately documented: Interface functions - should come with rich docstrings, ideally with examples, larger pieces of - functionality should come with some prose explaining what they are for. - - - Consider the effects on the larger system: Did this change make some example - or piece of documentation obsolete and therefore mean needs to be updated? - Did it break compatibility with something that we rely on? Should an email - be sent around announcing the change? Does the change solve or unblock some - outstanding issues? diff --git a/doc/sphinx/source/contributing/rules.rst b/doc/sphinx/source/contributing/rules.rst new file mode 100644 index 0000000000..91b415c81e --- /dev/null +++ b/doc/sphinx/source/contributing/rules.rst @@ -0,0 +1,143 @@ +.. code:: {eval-rst} + + .. _rules: + +Code development +================ + +Code development is carried out using Github. For more information on +the Git workflow that NNPDF adopts, see the `Git and +GitHub <./git.md>`__ section. + +Code contributions +------------------ + +Code contributions should be presented in the form of `Pull +Requests `__\ (PRs) to the +repository. Avoid committing modifications directly to the master +version of the code. Instead, create a new branch and make modifications +on it. + +This PR should adhere to the following rules: + +- **A clear explanation of the aims of the PR** should be given, + i.e. what issue(s) are you trying to address? If the reason for the + PR has already been detailed in an issue, then this issue should be + linked in the PR. + +- The PR should contain + `documentation <./sphinx-documentation.md>`__\ **describing the new + features**, if applicable. + +- If the PR is fixing a bug, information should be given such that a + reviewer can reproduce the bug. + +- The PR should have **at least one developer assigned to it**, whose + task it is to `review `__ the code. The PR cannot be merged + into master before the reviewer has approved it. + +- Before a PR can be merged into master, the **Travis build for it must + pass** (see `here <../ci/index.md>`__). Practically, this means that + you should find a green tick next to your PR on the relevant `PR + page `__. If you instead find a + red cross next to your PR, the reason for the failure must be + investigated and dealt with appropriately. + +- When writing examples, please use the recommended resources detailed + `here `__. + +Example pull request +-------------------- + +You may find it instructive to go though this pull request that +implements new convolution methods: + +https://github.com/NNPDF/nnpdf/pull/708/ + +It demonstrates how to add a new feature, together with relevant tests +and documentation, and refine it based on the discussion. + +.. code:: {eval-rst} + + .. _reviews: + +Reviewing pull requests +----------------------- + +All changes to the code `should `__ be reviewed by at least one +person (and ideally at least two). The expected benefits of the policy +are: + +- It should improve the overall quality of the code. + +- It should provide the author of the change with a reasonably quick + feedback loop to discuss the technical details involved in the + changes. + +- It should make at least two people (the author and the reviewer) + familiar with the changes. It should also ensure that the changes are + easy to read and maintain in the future, and conform to the structure + of the rest of the project. + +Guidelines for reviewing +~~~~~~~~~~~~~~~~~~~~~~~~ + +The following approach has been found helpful for reviewers, when +reviewing pull requests: + +- Make sure you actually understand what the changes are about. Unclear + details should not pass code review. Ask for clarifications, + documentation, and changes in the code that make it more clear. If + you are not in the position of taking the time, consider asking + somebody else to help reviewing the changes. If the changes are big + and difficult to comprehend at once, consider requesting that the + author breaks them down in easier to understand, self contained, pull + requests. Note that it is for the authors to proactively discuss the + proposed changes before they become too difficult for anyone else to + follow, and, failing that, it is fair to ask them to go through the + work of making them intelligible. + +- Look at the big picture first. Think about whether the overall idea + and implementation is sound or instead could benefit from going in a + different direction. Ideally before a lot of work has gone into fine + tuning details. + +- Review the code in detail. Try to identify areas where the changes + could be clearly improved in terms of clarity, speed or style. + Consider implementing minor changes yourself, although note that + there are trade-offs: People are more likely to assimilate good + patterns if they implement them a few times, which may be a win long + term, even if it takes longer to ship this particular code change. + +- Ideally changes should come with automatic tests supporting their + correctness. + +- Use `automated tools `__ which could catch a few extra + problems. In particular + + - Do look at the automated tests that run with the PR. New code + should not break them. + - Use ```pylint`` `__ with `our default + configuration `__ + to catch common problems with Python code. + - New Python code should come formatted with ```black`` + tool `__. + - Changes in compiled code should be tested in debug mode, with the + address sanitizer enabled. This is done with the + ``-DCMAKE_BUILD_TYPE=Debug -DENABLE_ASAN=ON`` options in + ``cmake``. + +- Regardless of automated tests, always run code with the new changes + manually. This gives great insight into possible pitfalls and areas + of improvement. + +- Make sure the changes are appropriately documented: Interface + functions should come with rich docstrings, ideally with examples, + larger pieces of functionality should come with some prose explaining + what they are for. + +- Consider the effects on the larger system: Did this change make some + example or piece of documentation obsolete and therefore mean needs + to be updated? Did it break compatibility with something that we rely + on? Should an email be sent around announcing the change? Does the + change solve or unblock some outstanding issues? diff --git a/doc/sphinx/source/data/plotting_format.md b/doc/sphinx/source/data/plotting_format.md deleted file mode 100644 index cc143e9b19..0000000000 --- a/doc/sphinx/source/data/plotting_format.md +++ /dev/null @@ -1,304 +0,0 @@ -```{eval-rst} -.. _plotting-format: -``` -Plotting format -=============== - -A *plotting file* defines a set of options that are used for analysis -and representation purposes, particularly to determine how datasets -should be represented in plots and how they should be grouped -together according to various criteria. The plotting files should be -considered part of the implementation of the dataset, and should be -read by various tools that want to sensibly represent the data. - -## Naming convention - -Plotting files are located in the `commondata` -folder (`nnpdfcpp/data/commondata`). -For a dataset labeled ``, the corresponding file name is -`PLOTTING_.yaml` or `PLOTTING_.yml` - -For example, given the dataset "HERA1CCEP", the corresponding -plotting file name is: - -```` -PLOTTING_HERA1CCEP.yaml -```` - -Additionally, the configuration is loaded from a per-process-type file -called: - -``` -PLOTTINGTYPE_.yaml -``` - -See [kinematic labels](#kinematic-labels) below for a list of defined types. When a key -is present both in the dataset-specific and the per-process-type file, the -dataset-specific one always takes precedence. - - -## Format - -The plotting file specifies the variable in which the data -is to be plotted (in the *x* axis) as well as the variables -in which the data will be split in different lines in the -same figure or in different figures. The possible variables -('*kinematic labels*') are described below. - -The format also allows the control of several plotting properties, such -as whether to use log scale, or the axes labels. - -### Data label - -A key called `dataset_label` can be used to specify a nice plotting -and display label for each dataset. LaTeX math is allowed between -dollar signs. See the [example](#example) plotting file for usage. - -### Kinematic labels - -The default kinematic variables are inferred from the *process type* -declared in the commondata files (more specifically from -a substring). Currently they are: - -``` -'DIS': ('$x$', '$Q^2 (GeV^2)$', '$y$'), -'DYP': ('$y$', '$M^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWJ_JPT': ('$p_T (GeV)$', '$M^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWJ_JRAP': ('$\\eta/y$', '$M^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWJ_MLL': ('$M_{ll} (GeV)$', '$M_{ll}^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWJ_PT': ('$p_T (GeV)$', '$M^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWJ_PTRAP': ('$\\eta/y$', '$p_T^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWJ_RAP': ('$\\eta/y$', '$M^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWK_MLL': ('$M_{ll} (GeV)$', '$M_{ll}^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWK_PT': ('$p_T$ (GeV)', '$M^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWK_PTRAP': ('$\\eta/y$', '$p_T^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'EWK_RAP': ('$\\eta/y$', '$M^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'HIG_RAP': ('$y$', '$M_H^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'HQP_MQQ': ('$M^{QQ} (GeV)$', '$\\mu^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'HQP_PTQ': ('$p_T^Q (GeV)$', '$\\mu^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'HQP_PTQQ': ('$p_T^{QQ} (GeV)$', '$\\mu^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'HQP_YQ': ('$y^Q$', '$\\mu^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'HQP_YQQ': ('$y^{QQ} (GeV)$', '$\\mu^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'INC': ('$0$', '$\\mu^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'JET': ('$\\eta$', '$p_T^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'PHT': ('$\\eta_\\gamma$', '$E_{T,\\gamma}^2 (GeV^2)$', '$\\sqrt{s} (GeV)$'), -'SIA': ('$z$', '$Q^2 (GeV^2)$', '$y$') -``` - -This mapping is declared as `CommonData.kinLabel_latex` in the C++ -code (and accessible as `validphys.plotoptions.core.kinlabels_latex` -in the Python code). - -The three kinematic variables are referred to as `k1`, `k2` and `k3` -in the plotting files. For example, for DIS processes, `k1` refers to `x`, -`k2` to `Q`, and `k3` to `y`. - -These kinematic values can be overridden by some transformation of -them. For that purpose, it is possible to define -a `kinematics_override` key. The value must be a class defined -in: `validphys2/src/validphys/plotoptions/kintransforms.py` - -The class must have a `__call__` method that takes three parameters: -`(k1, k2 k3)` as defined in the dataset implementation, and returns -three new values `('k1', 'k2', k3')` which are the "transformed" -kinematical variables, which will be used for plotting purposes every -time the kinematic variables `k1`, `k2` and `k3` are referred to. -Additionally, the class must implement a `new_labels` method, that -takes the old labels and returns the new ones, and an `xq2map` -function that takes the kinematic variables and returns a tuple of (x, -Q²) with some approximate values. An example of such transform is: - -````python -class dis_sqrt_scale: - def __call__(self, k1, k2, k3): - ecm = sqrt(k2/(k1*k3)) - return k1, sqrt(k2), ceil(ecm) - - def new_labels(self, *old_labels): - return ('$x$', '$Q$ (GeV)', r'$\sqrt{s} (GeV)$') - - def xq2map(self, k1, k2, k3, **extra_labels): - return k1, k2*k2 -```` - - -Additional labels can be specified by declaring an **extra_labels** -key in the plotting file, and specifying for each new label a value -for each point in the dataset. - -For example: - -```` -extra_labels: - idat2bin: [0, 0, 0, 0, 0, 0, 0, 0, 100, 100, 100, 100, 100, 200, 200, 200, 300, 300, 300, 400, 400, 400, 500, 500, 600, 600, 700, 700, 800, 800, 900, 1000, 1000, 1100] -```` - -defines one label where the values for each of the datapoints are -given in the list. Note that the name of the extra_label (in this case -`idat2bin` is completely arbitrary, and will be used for plotting -purposes (LaTeX math syntax is allowed as well). However, adding labels -manually for each point can be tedious. This should only be reserved -for information that cannot be recovered from the kinematics as -defined in the CommonData file. Instead, new labels can be generated -programmatically: every function defined in `validphys2/src/validphys/plotoptions/labelers.py` -is a valid label. These functions take as keyword arguments the -(possibly transformed) kinematical variables, as well as any extra -label declared in the plotting file. For example, one might declare: - -```` -def high_xq(k1, k2, k3, **kwargs): - return k1 > 1e-2 and k2 > 1000 - -```` - -Note that it is convenient to always declare the `**kwargs` -parameter so that the code doesn't crash when the function is called -with extra arguments. Similarly to the kinematics transforms, it is -possible to decorate them with a `@label` describing a nicer latex -label than the function name. For example: - -```` -@label(r"$I(x>10^{-2})\times I(Q > 1000 GeV)$") -def high_xq(k1, k2, k3, **kwargs): - return (k1 > 1e-2) & (k2 > 1000) - -```` - -### Plotting and grouping - -The variable in which the data is plotted is simply -declared as - -```` -x: