Make the sqlite theorydb into a folder of yaml files

conda-recipe/meta.yaml

@@ -26,7 +26,6 @@ requirements:
         - pymongo <4
         - seaborn
         - lhapdf
-        - sqlite
         - numpy
         - pkg-config
         - reportengine

doc/sphinx/Makefile

@@ -30,4 +30,4 @@ view: html
 	$(BROWSER) $(BUILDDIR)/html/index.html
 
 MakeTheoryCSV:
-	sqlite3 -header -csv ../../validphys2/src/validphys/datafiles/theory.db "select * from TheoryIndex;" > source/theories.csv
+	python make_theory_csv.py source/theories.csv

doc/sphinx/make_theory_csv.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+    Generate the theory csv using validphys functions
+"""
+from argparse import ArgumentParser
+from pathlib import Path
+
+from validphys.datafiles import theory_cards
+from validphys.theorydbutils import fetch_all
+
+if __name__ == "__main__":
+    parser = ArgumentParser()
+
+    parser.add_argument("csvpath", type=Path, help="Path to write the csv to")
+
+    args = parser.parse_args()
+
+    theory_df = fetch_all(theory_cards)
+    theory_df.to_csv(args.csvpath)

doc/sphinx/source/data/data-config.rst

@@ -30,14 +30,27 @@ all data files to be installed must have a ``.yaml`` extension.
 Theory lookup table
 ===================
 
-In order to organise the various different theoretical treatments available, a
-lookup table is provided in ``sqlite3`` format. This lookup table can be found
-in the ``nnpdf`` repository data directory at:
+In order to organise the various different theoretical treatments available,
+the theory definitions are saved in theory cards located in
 
-	``validphys/src/validphys2/datafiles/theory.db``
+	``validphys/src/validphys2/datafiles/theory_cards``
 
-This file should only be edited in order to add new theory options. It may be
-edited with any appropriate ``sqlite3``-supported software. A script is provided to
+in the form of ``yaml`` files. A new theory can be added by simply adding a new
+``yaml`` file with the desired theory ID. The definition of the accepted and required parameters
+can be found at:
+
+	``validphys/src/validphys2/theorydbutils.py``
+
+The following lines will check whether a newly added theory can be read by validphys
+(change 700 by the id of your newly added theory).
+
+..  code-block:: python
+
+    from validphys.datafiles import theory_cards
+    from validphys.theorydbutils import fetch_theory
+    theory = fetch_theory(theory_cards, 700)
+
+A script is provided to
 give a brief overview of the various theory options available. It can be found
 at
 

doc/sphinx/source/theory/theoryindex.rst

@@ -4,13 +4,12 @@ Theory indexes
 ==============
 
 In the table below you can see explicit content of the ``TheoryIndex`` table
-from the ``theory.db`` file. Note that not every theory listed in the database
+from the cards in the ``theory_cards`` folder. Note that not every theory listed in the database
 is available to be downloaded from the NNPDF :ref:`server`. In particular,
 theories that are outdated are not stored on the server, but their settings will
 remain in the database. To find a list of the theories that are available on the
-server, one can use the vp-list script (see :ref:`vp-list`) as so: :code:`vp-
-list -r theories`.
+server, one can use the vp-list script (see :ref:`vp-list`) as so: :code:`vp-list -r theories`.
 
-.. csv-table:: theory.db
+.. csv-table:: theory_card
    :file: ../theories.csv
    :header-rows: 1

doc/sphinx/source/theory/theoryparamsdefinitions.rst

@@ -4,8 +4,8 @@ Theory parameter definitions
 ============================
 
 Table describing the variables used to specify theory choices in :code:`nnpdf`. Here they are shown
-directly as the fields of the :code:`theory.db` lookup table. The `Type` column refers directly to
-the :code:`sqlite3` data type of the field.
+directly as the fields of the runcards in :code:`theory_cards`.
+The `Type` column refers directly to data type of the field.
 
 .. csv-table::
    :file: ../theory_params.csv

doc/sphinx/source/theory/theoryparamsinfo.rst

@@ -0,0 +1,95 @@
+.. _th_parameter_info:
+
+=====================================
+Looking up the parameters of a theory
+=====================================
+
+The parameters for all of the theories can be found in the ``theory_cards`` folder.
+The ``theory_cards`` folder is installed together, with the data, as part of the validphys package.
+This is an sqlite database file. The information contained within this file can also be
+viewed [within the docs](theory-indexes).
+
+The tools required to extract the parameters for a given theory are already in
+the validphys framework:
+
+..  code-block:: python
+
+  from validphys.loader import Loader
+
+  l = Loader()
+  # replace 53 with the relevant theoryID
+  tinfo = l.check_theoryID(53)
+  print(tinfo.get_description())
+  {'ID': 53, 'PTO': 2, 'FNS': 'FONLL-C', 'DAMP': 0, 'IC': 1, 'ModEv': 'TRN',
+  'XIR': 1.0, 'XIF': 1.0, 'NfFF': 5, 'MaxNfAs': 5, 'MaxNfPdf': 5, 'Q0': 1.65,
+  'alphas': 0.118, 'Qref': 91.2, 'QED': 0, 'alphaqed': 0.007496252,
+  'Qedref': 1.777, 'SxRes': 0, 'SxOrd': 'LL', 'HQ': 'POLE', 'mc': 1.51,
+  'Qmc': 1.51, 'kcThr': 1.0, 'mb': 4.92, 'Qmb': 4.92, 'kbThr': 1.0, 'mt': 172.5,
+  'Qmt': 172.5, 'ktThr': 1.0, 'CKM': '0.97428 0.22530 0.003470 0.22520 0.97345 0.041000 0.00862 0.04030 0.999152',
+  'MZ': 91.1876, 'MW': 80.398, 'GF': 1.1663787e-05, 'SIN2TW': 0.23126, 'TMC': 1,
+  'MP': 0.938, 'Comments': 'NNPDF3.1 NNLO central', 'global_nx': 0, 'EScaleVar': 1}
+
+
+Given that printing a dictionary in a python terminal is a bit cumbersome, the above
+method for checking theory parameters has been added to a command line
+script ``vp-checktheory`` which essentially does the same thing but prints the
+table in a nicer format.
+
+The usage is simple:
+
+..  code-block:: bash
+
+  $ vp-checktheory 53
+                                            Info for theory 53
+  ID                                                        53
+  PTO                                                        2
+  FNS                                                  FONLL-C
+  DAMP                                                       0
+  IC                                                         1
+  ModEv                                                    TRN
+  XIR                                                        1
+  XIF                                                        1
+  NfFF                                                       5
+  MaxNfAs                                                    5
+  MaxNfPdf                                                   5
+  Q0                                                      1.65
+  alphas                                                 0.118
+  Qref                                                    91.2
+  QED                                                        0
+  alphaqed                                          0.00749625
+  Qedref                                                 1.777
+  SxRes                                                      0
+  SxOrd                                                     LL
+  HQ                                                      POLE
+  mc                                                      1.51
+  Qmc                                                     1.51
+  kcThr                                                      1
+  mb                                                      4.92
+  Qmb                                                     4.92
+  kbThr                                                      1
+  mt                                                     172.5
+  Qmt                                                    172.5
+  ktThr                                                      1
+  CKM        0.97428 0.22530 0.003470 0.22520 0.97345 0.041...
+  MZ                                                   91.1876
+  MW                                                    80.398
+  GF                                               1.16638e-05
+  SIN2TW                                               0.23126
+  TMC                                                        1
+  MP                                                     0.938
+  Comments                               NNPDF3.1 NNLO central
+  global_nx                                                  0
+  EScaleVar                                                  1
+
+Some of the entries of the table have been truncated to fit in the terminal, for
+example the CKM matrix elements. If you want to see the
+full output or wish to keep a permanent copy of the table then you can use the
+command line option ``-d``, which stands for ``--dumptable`` which will save the
+table in your current working directory as ``theory_<id>_info.csv``::
+
+  $vp-checktheory --fit FIT
+
+where ``FIT`` is a valid fit name. If the fit cannot be found locally, the script
+will attempt to download it.
+
+The parameters in the above are defined [here](./theoryparamsdefinitions).

doc/sphinx/source/theory_params.csv

@@ -37,4 +37,4 @@ TMC, Integer, Target mass corrections, Boolean
 MP, Real, :math:`M_P` Proton mass, Units: GeV
 global_nx, Integer, Global x-grid precision, Default (0) uses set-by-set precision
 EScaleVar, Real, Switch for DGLAP scale variation, Boolean
-Comments, Text, General comments,
+Comments, Text, General comments,

pyproject.toml

@@ -31,13 +31,18 @@ include = [
     "validphys2/src/validphys/nnprofile_default.yaml",
     # Same for commondata and theory.db
     "validphys2/src/validphys/datafiles/commondata/*",
-    "validphys2/src/validphys/datafiles/theory.db",
+    "validphys2/src/validphys/datafiles/theory_cards/*.yaml",
     # From the new commondata we are only interested on top-level yaml files
     "validphys2/src/validphys/datafiles/new_commondata/*/*.yaml",
     "validphys2/src/validphys/datafiles/new_commondata/dataset_names.yml",
     # The version file is ignored by git so it needs to be explicitly included
     "validphys2/src/validphys/_version.py"
 ]
+# Exclude intermediate files data files
+exclude = [
+    "validphys2/src/validphys/datafiles/new_commondata/*/rawdata",
+    "validphys2/src/validphys/datafiles/new_commondata/*/*.py"
+]
 
 [tool.poetry.scripts]
 n3fit = "n3fit.scripts.n3fit_exec:main"

validphys2/src/validphys/config.py

@@ -934,8 +934,8 @@ def produce_dataspecs_with_matched_cuts(self, dataspecs):
         return res
 
     def produce_theory_database(self):
-        """Produces path to the theory.db file"""
-        return self.loader.theorydb_file
+        """Produces path to the folder of the theory runcards"""
+        return self.loader.theorydb_folder
 
     def produce_combined_shift_and_theory_dataspecs(self, dataspecs):
         matched_datasets = self.produce_matched_datasets_from_dataspecs(dataspecs)

validphys2/src/validphys/core.py

@@ -782,13 +782,13 @@ def __str__(self):
     def __hash__(self):
         return hash(self.path.as_posix())
 
-    @property
-    def yamldb_path(self):
-        return self.path / "yamldb"
-
     def is_pineappl(self):
-        """Check whether this theory is a pineappl-based theory"""
-        return self.yamldb_path.exists()
+        """Check whether this theory is a pineappl-based theory
+        Assume yes unless a compound directory is found
+        """
+        if (self.path / "compound").is_dir():
+            return False
+        return True
 
 
 class ThCovMatSpec: