Pseudodata in diagonal basis and saving rotation

.github/workflows/all_tests_nnpdf.yml

@@ -103,7 +103,11 @@ jobs:
         sudo rm -rf /opt/hostedtoolcache/Python/{3.9*}
         sudo rm -rf /opt/hostedtoolcache/CodeQL
         sudo rm -rf /usr/local/lib/android/sdk
-        pip install .[nolha,torch]
+        # torch 2.12.0 pulls triton 3.7.0, whose libtriton C extension segfaults
+        # at import on CPU-only Linux + CPython 3.13. Pinning torch<2.12 also
+        # pins triton to 3.6.x. Remove this pin once the upstream issue is
+        # resolved.
+        pip install ".[nolha,torch]" "torch<2.12"
         # Since there is no LHAPDF in the system, initialize the folder and download pdfsets.index
         lhapdf-management update --init
     - name: Test we can run one runcard

doc/sphinx/source/n3fit/runcard_detailed.rst

@@ -429,6 +429,12 @@ according to their experiment. Additionally, the union of these two is saved in
 ``<fit_directory>/replica_<number>/datacuts_theory_fitting_pseudodata_table.csv``
 if one is not interested in the exact nature of the splitting.
 
+When ``diagonal_basis: true`` is used (by default), the saved pseudodata indices are labeled as
+``eigenmode <i>`` corresponding to the diagonal basis used in the fit. ``vp-setupfit`` writes one
+file with the diagonal-basis elements under:
+``<fit_directory>/tables/diagonal_basis_rotation.csv``.
+This file stores both the eigenvalues and eigenvectors used for the rotation.
+
 
 Imposing sum rules
 ^^^^^^^^^^^^^^^^^^

n3fit/src/n3fit/scripts/vp_setupfit.py

@@ -25,6 +25,7 @@
 # top.
 
 
+import copy
 import hashlib
 import logging
 import pathlib
@@ -54,9 +55,10 @@
     'validphys.results',
     'validphys.theorycovariance.construction',
     'validphys.photon.compute',
+    'validphys.n3fit_data',
 ]
 
-SETUPFIT_DEFAULTS = dict(use_cuts='internal')
+SETUPFIT_DEFAULTS = dict(use_cuts='internal', use_t0=True)
 
 
 log = logging.getLogger(__name__)
@@ -141,6 +143,9 @@ class SetupFitConfig(Config):
 
     @classmethod
     def from_yaml(cls, o, *args, **kwargs):
+        # Create a fresh copy of the fixed config to avoid in-place modifications
+        fixed_config = copy.deepcopy(SETUPFIT_FIXED_CONFIG)
+
         try:
             file_content = yaml_safe.load(o)
         except error.YAMLError as e:
@@ -156,10 +161,10 @@ def from_yaml(cls, o, *args, **kwargs):
             # Use faketheoryid to create the L0 data to be stored into the filter folder
             # (L1 data is stored if fakedata is True)
             if 'faketheoryid' in closuredict:
-                # make sure theory key exists in SETUPFIT_FIXED_CONFIG
-                SETUPFIT_FIXED_CONFIG.setdefault('theory', {})
+                # make sure theory key exists in fixed_config
+                fixed_config.setdefault('theory', {})
                 # overwrite theoryid with the faketheoryid
-                SETUPFIT_FIXED_CONFIG['theory']['theoryid'] = closuredict['faketheoryid']
+                fixed_config['theory']['theoryid'] = closuredict['faketheoryid']
                 # download theoryid since it will be used in the fit
                 try:
                     loader.check_theoryID(file_content['theory']['theoryid'])
@@ -171,8 +176,14 @@ def from_yaml(cls, o, *args, **kwargs):
             filter_action = 'datacuts::theory::fitting filter'
             check_n3fit_action = 'datacuts::theory::fitting n3fit_checks_action'
 
+        # Add rotation action for the total covariance matrix
+        if file_content.get('theorycovmatconfig') is not None:
+            rotation_action = 'datacuts::theory::theorycovmatconfig fitting_covmat_table'
+        else:
+            rotation_action = 'datacuts::theory fitting_covmat_table'
+
         # The settings for these actions depend on the presence of closuretest
-        SETUPFIT_FIXED_CONFIG['actions_'] += [check_n3fit_action, filter_action]
+        fixed_config['actions_'] += [check_n3fit_action, filter_action, rotation_action]
 
         # Check theory covariance matrix configuration
         thconfig = file_content.get('theorycovmatconfig', {})
@@ -184,7 +195,7 @@ def from_yaml(cls, o, *args, **kwargs):
                 "`point_prescriptions: ['9 point', '3 point']`"
             )
         if thconfig:
-            SETUPFIT_FIXED_CONFIG['actions_'].append(
+            fixed_config['actions_'].append(
                 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat'
             )
 
@@ -214,14 +225,14 @@ def from_yaml(cls, o, *args, **kwargs):
             if compute_in_setupfit:
                 log.info("Forcing photon computation with FiatLux during setupfit.")
                 # Since the photon will be computed, check that the luxset and additional_errors exist
-                SETUPFIT_FIXED_CONFIG['actions_'].append('fiatlux check_luxset_exists')
+                fixed_config['actions_'].append('fiatlux check_luxset_exists')
                 if fiatlux.get("additional_errors"):
-                    SETUPFIT_FIXED_CONFIG['actions_'].append('fiatlux check_additional_errors')
-                SETUPFIT_FIXED_CONFIG['actions_'].append('fiatlux::theory compute_photon_to_disk')
+                    fixed_config['actions_'].append('fiatlux check_additional_errors')
+                fixed_config['actions_'].append('fiatlux::theory compute_photon_to_disk')
 
         # Check positivity bound
         if file_content.get('positivity_bound') is not None:
-            SETUPFIT_FIXED_CONFIG['actions_'].append('positivity_bound check_unpolarized_bc')
+            fixed_config['actions_'].append('positivity_bound check_unpolarized_bc')
 
         # Check hyperscan
         trials_config = file_content.get('trial_specs', {})
@@ -233,7 +244,7 @@ def from_yaml(cls, o, *args, **kwargs):
             file_content.setdefault(k, v)
 
         # Update file content with fixed configuration
-        file_content.update(SETUPFIT_FIXED_CONFIG)
+        file_content.update(fixed_config)
 
         return cls(file_content, *args, **kwargs)
 

n3fit/src/n3fit/tests/test_fit.py

@@ -283,8 +283,11 @@ def test_parallel_against_sequential(tmp_path, rep_from=6, rep_to=8):
         "ATLAS_TTBAR_8TEV_TOT_X-SEC",
         "CMS_SINGLETOP_13TEV_TCHANNEL-XSEC",
     ]
-    dataset_inputs = [{"dataset": d, "frac": 0.6, "variant": "legacy"} for d in datasets]
+    dataset_inputs = [{"dataset": d, "variant": "legacy"} for d in datasets]
     n3fit_input["dataset_inputs"] = dataset_inputs
+    # Using diaogonal basis
+    n3fit_input["diagonal_basis"] = True
+    n3fit_input["diagonal_frac"] = 0.5
     # Exit inmediately
     n3fit_input["parameters"]["epochs"] = 1
     # Save pseudodata
@@ -311,8 +314,9 @@ def test_parallel_against_sequential(tmp_path, rep_from=6, rep_to=8):
     for csvfile_seq in folder_seq.glob("*/*.csv"):
         csvfile_par = folder_par / csvfile_seq.relative_to(folder_seq)
 
-        result_seq = pd.read_csv(csvfile_seq, sep="\t", index_col=[0, 1, 2], header=0)
-        result_par = pd.read_csv(csvfile_par, sep="\t", index_col=[0, 1, 2], header=0)
+        # Diagonal basis writes single-index csv files
+        result_seq = pd.read_csv(csvfile_seq, sep="\t", index_col=[0], header=0)
+        result_par = pd.read_csv(csvfile_par, sep="\t", index_col=[0], header=0)
         pd.testing.assert_frame_equal(result_seq, result_par)
 
     # Check the rest of the fit, while numerical differences are expected between sequential

validphys2/src/validphys/config.py

@@ -812,10 +812,11 @@ def produce_dataset_inputs_fitting_covmat(self, use_thcovmat_in_fitting=False):
         separate the multiplcative errors and whether to compute the
         experimental covmat using the t0 prescription.
         """
+
         from validphys import covmats
 
         if use_thcovmat_in_fitting:
-            return covmats.dataset_inputs_t0_total_covmat
+            return covmats.dataset_load_inputs_t0_total_covmat
         return covmats.dataset_inputs_t0_exp_covmat
 
     def produce_sep_mult(self, separate_multiplicative=False):
@@ -851,9 +852,9 @@ def produce_dataset_inputs_sampling_covmat(
         if use_t0_sampling:
             if use_thcovmat_in_sampling:
                 if sep_mult:
-                    return covmats.dataset_inputs_t0_total_covmat_separate
+                    return covmats.dataset_load_inputs_t0_total_covmat_separate
                 else:
-                    return covmats.dataset_inputs_t0_total_covmat
+                    return covmats.dataset_load_inputs_t0_total_covmat
             else:
                 if sep_mult:
                     return covmats.dataset_inputs_t0_exp_covmat_separate
@@ -863,9 +864,9 @@ def produce_dataset_inputs_sampling_covmat(
         else:
             if use_thcovmat_in_sampling:
                 if sep_mult:
-                    return covmats.dataset_inputs_total_covmat_separate
+                    return covmats.dataset_load_inputs_total_covmat_separate
                 else:
-                    return covmats.dataset_inputs_total_covmat
+                    return covmats.dataset_load_inputs_total_covmat
             else:
                 if sep_mult:
                     return covmats.dataset_inputs_exp_covmat_separate
@@ -885,6 +886,7 @@ def produce_loaded_theory_covmat(
         Loads the theory covmat from the correct file according to how it
         was generated by vp-setupfit.
         """
+
         if not use_thcovmat_in_sampling and not use_thcovmat_in_fitting:
             return 0.0
         # Load correct file according to how the thcovmat was generated by vp-setupfit
@@ -1328,6 +1330,7 @@ def produce_nnfit_theory_covmat(
         This function is only used in vp-setupfit to store the necessary covmats as .csv files in
         the tables directory.
         """
+
         if point_prescriptions is not None:
             if user_covmat_path is not None:
                 # Both scalevar and user uncertainties

validphys2/src/validphys/covmats.py

@@ -225,6 +225,7 @@ def dataset_inputs_covmat_from_systematics(
         covmat = (covmat / sqrt_weights).T / sqrt_weights
     if norm_threshold is not None:
         covmat = regularize_covmat(covmat, norm_threshold=norm_threshold)
+
     return covmat
 
 
@@ -405,7 +406,7 @@ def dataset_inputs_t0_covmat_from_systematics(
     )
 
 
-def dataset_inputs_t0_total_covmat_separate(
+def dataset_load_inputs_t0_total_covmat_separate(
     dataset_inputs_t0_exp_covmat_separate, loaded_theory_covmat
 ):
     """
@@ -416,6 +417,17 @@ def dataset_inputs_t0_total_covmat_separate(
     return dataset_inputs_t0_exp_covmat_separate + loaded_theory_covmat
 
 
+def dataset_inputs_t0_total_covmat_separate(
+    dataset_inputs_t0_exp_covmat_separate, nnfit_theory_covmat
+):
+    """
+    Function to compute the covmat to be used for the sampling by make_replica.
+    In this case the t0 prescription is used for the experimental covmat and the multiplicative
+    errors are separated. Moreover, the theory covmat is added to experimental covmat.
+    """
+    return dataset_inputs_t0_exp_covmat_separate + nnfit_theory_covmat
+
+
 def dataset_inputs_t0_exp_covmat_separate(
     dataset_inputs_loaded_cd_with_cuts,
     *,
@@ -440,7 +452,9 @@ def dataset_inputs_t0_exp_covmat_separate(
     return covmat
 
 
-def dataset_inputs_total_covmat_separate(dataset_inputs_exp_covmat_separate, loaded_theory_covmat):
+def dataset_load_inputs_total_covmat_separate(
+    dataset_inputs_exp_covmat_separate, loaded_theory_covmat
+):
     """
     Function to compute the covmat to be used for the sampling by make_replica.
     In this case the t0 prescription is not used for the experimental covmat and the multiplicative
@@ -449,6 +463,15 @@ def dataset_inputs_total_covmat_separate(dataset_inputs_exp_covmat_separate, loa
     return dataset_inputs_exp_covmat_separate + loaded_theory_covmat
 
 
+def dataset_inputs_total_covmat_separate(dataset_inputs_exp_covmat_separate, nnfit_theory_covmat):
+    """
+    Function to compute the covmat to be used for the sampling by make_replica.
+    In this case the t0 prescription is not used for the experimental covmat and the multiplicative
+    errors are separated. Moreover, the theory covmat is added to experimental covmat.
+    """
+    return dataset_inputs_exp_covmat_separate + nnfit_theory_covmat
+
+
 def dataset_inputs_exp_covmat_separate(
     dataset_inputs_loaded_cd_with_cuts,
     *,
@@ -472,7 +495,17 @@ def dataset_inputs_exp_covmat_separate(
     return covmat
 
 
-def dataset_inputs_t0_total_covmat(dataset_inputs_t0_exp_covmat, loaded_theory_covmat):
+def dataset_inputs_t0_total_covmat(dataset_inputs_t0_exp_covmat, nnfit_theory_covmat):
+    """
+    Function to compute the covmat to be used for the sampling by make_replica and for the chi2
+    by fitting_data_dict. In this case the t0 prescription is used for the experimental covmat
+    and the multiplicative errors are included in it. Moreover, the theory covmat is added to experimental covmat.
+    """
+
+    return dataset_inputs_t0_exp_covmat + nnfit_theory_covmat
+
+
+def dataset_load_inputs_t0_total_covmat(dataset_inputs_t0_exp_covmat, loaded_theory_covmat):
     """
     Function to compute the covmat to be used for the sampling by make_replica and for the chi2
     by fitting_data_dict. In this case the t0 prescription is used for the experimental covmat
@@ -502,10 +535,11 @@ def dataset_inputs_t0_exp_covmat(
         dataset_inputs_t0_predictions,
         False,
     )
+
     return covmat
 
 
-def dataset_inputs_total_covmat(dataset_inputs_exp_covmat, loaded_theory_covmat):
+def dataset_load_inputs_total_covmat(dataset_inputs_exp_covmat, loaded_theory_covmat):
     """
     Function to compute the covmat to be used for the sampling by make_replica and for the chi2
     by fitting_data_dict. In this case the t0 prescription is not used for the experimental covmat
@@ -514,6 +548,15 @@ def dataset_inputs_total_covmat(dataset_inputs_exp_covmat, loaded_theory_covmat)
     return dataset_inputs_exp_covmat + loaded_theory_covmat
 
 
+def dataset_inputs_total_covmat(dataset_inputs_exp_covmat, nnfit_theory_covmat):
+    """
+    Function to compute the covmat to be used for the sampling by make_replica and for the chi2
+    by fitting_data_dict. In this case the t0 prescription is not used for the experimental covmat
+    and the multiplicative errors are included in it. Moreover, the theory covmat is added to experimental covmat.
+    """
+    return dataset_inputs_exp_covmat + nnfit_theory_covmat
+
+
 def dataset_inputs_exp_covmat(
     dataset_inputs_loaded_cd_with_cuts,
     *,