From 1a3f5d6230aba8c285613dac1d0c5531ef593033 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 9 Nov 2022 11:59:48 +0100 Subject: [PATCH 001/204] Initialize module for the computation of ph in validphys --- validphys2/src/validphys/compute_photon.py | 130 +++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 validphys2/src/validphys/compute_photon.py diff --git a/validphys2/src/validphys/compute_photon.py b/validphys2/src/validphys/compute_photon.py new file mode 100644 index 0000000000..b01119870f --- /dev/null +++ b/validphys2/src/validphys/compute_photon.py @@ -0,0 +1,130 @@ +"""Script that calls fiatlux to add the photon PDF.""" + +from validphys import lhapdfset +from validphys.api import API +import fiatlux +import numpy as np +from eko.couplings import Couplings +from eko.compatibility import update_theory +from eko.runner import Runner + +import yaml + +def photon_1GeV(xgrid, theoryID, pdf_name, replica): + r""" + Compute the photon PDF for every point in the grid xgrid. + + Parameters + ---------- + xgrid: list + grid of the x points + pdf_name: string + name of the QCD set + replica: int + number of replica + + Returns + ------- + photon_1GeV: numpay.array + photon PDF at the scale 1 GeV + """ + set = lhapdfset.LHAPDFSet(pdf_name, "replicas") + pdfs = np.zeros((len(set.flavors()), len(xgrid))) + for j, pid in enumerate(set.flavors()): + if not set.hasFlavor(pid): + continue + pdfs[j] = np.array( + [ + set.xfxQ(x, 100., replica, pid) / x + for x in xgrid + ] + ) + out_grid = {} + + def f2(x, q): + # call yadism to give f2 + return 0. + + def fl(x, q): + # call yadism to give fl + return 0. + + def f2lo(x, q): + # call yadism to give f20 + return 0. + + theory = API.theoryid(theoryid = theoryID).get_description() + # fiatlux_runcard = + fiatlux_runcard = { + 'apfel': True, + 'qed_running': True, + 'q2_max': '1e9', + 'eps_base': '1e-5', + 'eps_rel': '1e-1', + 'mproton': 0.938272046, + 'mum_proton': 2.792847356, + 'elastic_param': 'A1_world_pol_spline', + 'elastic_electric_rescale': 1, + 'elastic_magnetic_rescale': 1, + 'inelastic_param': 'LHAPDF_Hermes_ALLM_CLAS', + 'rescale_r_twist4': 0, + 'rescale_r': 1, 'allm_limits': 0, + 'rescale_non_resonance': 1, + 'rescale_resonance': 1, + 'use_mu2_as_upper_limit': False, + 'q2min_inel_override': 0.0, + 'q2max_inel_override': '1E300', + 'lhapdf_transition_q2': 9, + 'verbose': True + } + # TODO : move this dict into the runcard and load it + lux = fiatlux.FiatLux(yaml.dump(fiatlux_runcard)) + qref_qed = theory["Qref"] + + couplings = Couplings.from_dict(update_theory(theory)) + def alpha_em(q): + return couplings.a(q**2)[1] * 4 * np.pi + + lux.PlugAlphaQED(alpha_em, qref_qed) + lux.PlugStructureFunctions(f2, fl, f2lo) + lux.InsertInelasticSplitQ([4.18, 1e100]) + eko_operator = dict( + sorted( + dict( + interpolation_xgrid=xgrid.tolist(), + interpolation_polynomial_degree=4, + interpolation_is_log=True, + ev_op_max_order=10, + ev_op_iterations=10, + backward_inversion="expanded", + n_integration_cores=0, + debug_skip_non_singlet=False, + debug_skip_singlet=False, + Q2grid=[1.**2], + inputgrid=None, + targetgrid=None, + inputpids=None, + targetpids=None, + ).items() + ) + ) + q2 = 100.**2 + photon_100GeV = np.array([]) + for x in xgrid: + pht = lux.EvaluatePhoton(x, q2) + np.append(photon_100GeV, pht.total) + pdfs[11] = photon_100GeV + + runner = Runner(theory, eko_operator) + eko = runner.get_output() + + for q2, elem in eko.items(): + pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) + error_final = np.einsum("ajbk,bk", elem.error, pdfs) + out_grid[q2] = { + "pdfs": dict(zip(eko.rotations.targetpids, pdf_final)), + "errors": dict(zip(eko.rotations.targetpids, error_final)), + } + photon_1GeV = pdf_final[11] + + return photon_1GeV \ No newline at end of file From 0d4c17de5fa33ec81d7ce707580d7cef467f832f Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 11 Nov 2022 10:16:58 +0100 Subject: [PATCH 002/204] Modify compute_photon.py --- n3fit/src/n3fit/model_trainer.py | 10 ++ n3fit/src/n3fit/scripts/n3fit_exec.py | 6 ++ validphys2/src/validphys/compute_photon.py | 102 ++++++++++----------- 3 files changed, 62 insertions(+), 56 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 14a9724b34..c4621bfc77 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -19,6 +19,8 @@ from n3fit.vpinterface import N3PDF import n3fit.hyper_optimization.penalties import n3fit.hyper_optimization.rewards +from validphys.compute_photon import photon_1GeV +from n3fit.scripts.n3fit_exec import N3FIT_FIXED_CONFIG log = logging.getLogger(__name__) @@ -355,6 +357,14 @@ def _model_generation(self, pdf_models, partition, partition_idx): # Apply feature scaling if given input_arr = self._scaler(input_arr) input_layer = op.numpy_to_input(input_arr) + + photon_list = [ + photon_1GeV( + xgrid=grid[0], + theoryid=N3FIT_FIXED_CONFIG['theoryid'], + fiatlux_runcard = N3FIT_FIXED_CONFIG['fiatlux'], + ) for grid in inputs_unique + ] # For multireplica fits: # The trainable part of the n3fit framework is a concatenation of all PDF models diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 264acb4eee..05f025a2a0 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -145,6 +145,12 @@ def from_yaml(cls, o, *args, **kwargs): validation_action = namespace + "validation_pseudodata" N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) + if (thconfig:=file_content.get('theory')) is not None: + N3FIT_FIXED_CONFIG['theoryid']=thconfig.get('theoryid', True) + if (thconfig:=file_content.get('fiatlux')) is not None: + N3FIT_FIXED_CONFIG['fiatlux']=thconfig + else : + N3FIT_FIXED_CONFIG['fiatlux']=None #Theorycovmat flags and defaults N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = False diff --git a/validphys2/src/validphys/compute_photon.py b/validphys2/src/validphys/compute_photon.py index b01119870f..a3fa3e2408 100644 --- a/validphys2/src/validphys/compute_photon.py +++ b/validphys2/src/validphys/compute_photon.py @@ -10,7 +10,7 @@ import yaml -def photon_1GeV(xgrid, theoryID, pdf_name, replica): +def photon_1GeV(xgrid, theoryid, fiatlux_runcard): r""" Compute the photon PDF for every point in the grid xgrid. @@ -28,18 +28,10 @@ def photon_1GeV(xgrid, theoryID, pdf_name, replica): photon_1GeV: numpay.array photon PDF at the scale 1 GeV """ - set = lhapdfset.LHAPDFSet(pdf_name, "replicas") - pdfs = np.zeros((len(set.flavors()), len(xgrid))) - for j, pid in enumerate(set.flavors()): - if not set.hasFlavor(pid): - continue - pdfs[j] = np.array( - [ - set.xfxQ(x, 100., replica, pid) / x - for x in xgrid - ] - ) - out_grid = {} + if fiatlux_runcard is None : + return np.zeros(len(xgrid)) + set = lhapdfset.LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") + qcd_pdfs = set.members[0] #use for the moment central replica TODO: change it def f2(x, q): # call yadism to give f2 @@ -53,32 +45,22 @@ def f2lo(x, q): # call yadism to give f20 return 0. - theory = API.theoryid(theoryid = theoryID).get_description() - # fiatlux_runcard = - fiatlux_runcard = { - 'apfel': True, - 'qed_running': True, - 'q2_max': '1e9', - 'eps_base': '1e-5', - 'eps_rel': '1e-1', - 'mproton': 0.938272046, - 'mum_proton': 2.792847356, - 'elastic_param': 'A1_world_pol_spline', - 'elastic_electric_rescale': 1, - 'elastic_magnetic_rescale': 1, - 'inelastic_param': 'LHAPDF_Hermes_ALLM_CLAS', - 'rescale_r_twist4': 0, - 'rescale_r': 1, 'allm_limits': 0, - 'rescale_non_resonance': 1, - 'rescale_resonance': 1, - 'use_mu2_as_upper_limit': False, - 'q2min_inel_override': 0.0, - 'q2max_inel_override': '1E300', - 'lhapdf_transition_q2': 9, - 'verbose': True - } - # TODO : move this dict into the runcard and load it - lux = fiatlux.FiatLux(yaml.dump(fiatlux_runcard)) + theory = API.theoryid(theoryid = theoryid).get_description().copy() + # import ipdb; ipdb.set_trace() + theory["nfref"] = None + theory["nf0"] = None + theory["fact_to_ren_scale_ratio"] = 1. + theory["ModSV"] = None + theory["IC"]=0 + theory["IB"]=0 + theory["FNS"] = "VFNS" + q_in = 100 + q_in2 = q_in ** 2 + q_fin = theory["Q0"] + theory["Q0"]= q_in + + lux = fiatlux.FiatLux(fiatlux_runcard) + # TODO : we are passing a dict but fiatlux wants a yaml file qref_qed = theory["Qref"] couplings = Couplings.from_dict(update_theory(theory)) @@ -88,7 +70,7 @@ def alpha_em(q): lux.PlugAlphaQED(alpha_em, qref_qed) lux.PlugStructureFunctions(f2, fl, f2lo) lux.InsertInelasticSplitQ([4.18, 1e100]) - eko_operator = dict( + operator_card = dict( sorted( dict( interpolation_xgrid=xgrid.tolist(), @@ -100,7 +82,7 @@ def alpha_em(q): n_integration_cores=0, debug_skip_non_singlet=False, debug_skip_singlet=False, - Q2grid=[1.**2], + Q2grid=[q_fin**2], inputgrid=None, targetgrid=None, inputpids=None, @@ -108,23 +90,31 @@ def alpha_em(q): ).items() ) ) - q2 = 100.**2 - photon_100GeV = np.array([]) - for x in xgrid: - pht = lux.EvaluatePhoton(x, q2) - np.append(photon_100GeV, pht.total) - pdfs[11] = photon_100GeV - runner = Runner(theory, eko_operator) - eko = runner.get_output() - - for q2, elem in eko.items(): + photon_100GeV = np.array( + [lux.EvaluatePhoton(x, q2).total / x for x in xgrid] + ) + # TODO: fiatlux returns gamma(x) or x*gamma(x) ? + runner = Runner(theory, operator_card) + output = runner.get_output() + pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) + for j, pid in enumerate(output.rotations.inputpids): + if pid == 22 : + pdfs[j] = photon_100GeV + if not qcd_pdfs.hasFlavor(pid): + continue + pdfs[j] = np.array( + [ + qcd_pdfs.xfxQ2(pid, x, q_in2) / x + for x in xgrid + ] + ) + + pdfs[11] = photon_100GeV + for q2, elem in output.items(): pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) - error_final = np.einsum("ajbk,bk", elem.error, pdfs) - out_grid[q2] = { - "pdfs": dict(zip(eko.rotations.targetpids, pdf_final)), - "errors": dict(zip(eko.rotations.targetpids, error_final)), - } + # error_final = np.einsum("ajbk,bk", elem.error, pdfs) + photon_1GeV = pdf_final[11] return photon_1GeV \ No newline at end of file From d5462fd1db672e6649505f7e0d34e9b399cfc6fc Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 14 Nov 2022 17:14:12 +0100 Subject: [PATCH 003/204] Add replica id to compute_photon --- n3fit/src/n3fit/model_trainer.py | 8 +++-- n3fit/src/n3fit/performfit.py | 3 +- validphys2/src/validphys/compute_photon.py | 36 +++++++++++++--------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index c4621bfc77..ce3bfb293a 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -98,6 +98,7 @@ def __init__( model_file=None, sum_rules=None, parallel_models=1, + replica_id=None, ): """ Parameters @@ -150,6 +151,7 @@ def __init__( self.all_datasets = [] self._scaler = None self._parallel_models = parallel_models + self.replica_id = replica_id # Initialise internal variables which define behaviour if debug: @@ -359,12 +361,14 @@ def _model_generation(self, pdf_models, partition, partition_idx): input_layer = op.numpy_to_input(input_arr) photon_list = [ - photon_1GeV( + np.array([photon_1GeV( xgrid=grid[0], theoryid=N3FIT_FIXED_CONFIG['theoryid'], fiatlux_runcard = N3FIT_FIXED_CONFIG['fiatlux'], - ) for grid in inputs_unique + replica=self.replica_id, + )]) for grid in inputs_unique ] + input_photon = np.concatenate(photon_list, axis=1).T # For multireplica fits: # The trainable part of the n3fit framework is a concatenation of all PDF models diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index a98040dd96..9d98f73bdf 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -193,7 +193,8 @@ def performfit( max_cores=maxcores, model_file=load, sum_rules=sum_rules, - parallel_models=n_models + parallel_models=n_models, + replica_id=replica_idxs[0] ) # This is just to give a descriptive name to the fit function diff --git a/validphys2/src/validphys/compute_photon.py b/validphys2/src/validphys/compute_photon.py index a3fa3e2408..69120b22d5 100644 --- a/validphys2/src/validphys/compute_photon.py +++ b/validphys2/src/validphys/compute_photon.py @@ -10,13 +10,13 @@ import yaml -def photon_1GeV(xgrid, theoryid, fiatlux_runcard): +def photon_1GeV(xgrid, theoryid, fiatlux_runcard, replica): r""" Compute the photon PDF for every point in the grid xgrid. Parameters ---------- - xgrid: list + xgrid: numpy.array grid of the x points pdf_name: string name of the QCD set @@ -25,13 +25,13 @@ def photon_1GeV(xgrid, theoryid, fiatlux_runcard): Returns ------- - photon_1GeV: numpay.array + photon_1GeV: numpy.array photon PDF at the scale 1 GeV """ if fiatlux_runcard is None : return np.zeros(len(xgrid)) set = lhapdfset.LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") - qcd_pdfs = set.members[0] #use for the moment central replica TODO: change it + qcd_pdfs = set.members[replica] def f2(x, q): # call yadism to give f2 @@ -46,7 +46,6 @@ def f2lo(x, q): return 0. theory = API.theoryid(theoryid = theoryid).get_description().copy() - # import ipdb; ipdb.set_trace() theory["nfref"] = None theory["nf0"] = None theory["fact_to_ren_scale_ratio"] = 1. @@ -59,17 +58,22 @@ def f2lo(x, q): q_fin = theory["Q0"] theory["Q0"]= q_in - lux = fiatlux.FiatLux(fiatlux_runcard) - # TODO : we are passing a dict but fiatlux wants a yaml file - qref_qed = theory["Qref"] + # lux = fiatlux.FiatLux(fiatlux_runcard) + # we have a dict but fiatlux wants a yaml file + # TODO : remove this trick + ff = open('fiatlux_runcard.yml', 'w+') + yaml.dump(fiatlux_runcard, ff) + + lux = fiatlux.FiatLux('fiatlux_runcard.yml') couplings = Couplings.from_dict(update_theory(theory)) def alpha_em(q): return couplings.a(q**2)[1] * 4 * np.pi - lux.PlugAlphaQED(alpha_em, qref_qed) + lux.PlugAlphaQED(alpha_em, theory["Qref"]) lux.PlugStructureFunctions(f2, fl, f2lo) lux.InsertInelasticSplitQ([4.18, 1e100]) + operator_card = dict( sorted( dict( @@ -95,12 +99,12 @@ def alpha_em(q): [lux.EvaluatePhoton(x, q2).total / x for x in xgrid] ) # TODO: fiatlux returns gamma(x) or x*gamma(x) ? - runner = Runner(theory, operator_card) - output = runner.get_output() + pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) for j, pid in enumerate(output.rotations.inputpids): if pid == 22 : pdfs[j] = photon_100GeV + ph_id = j if not qcd_pdfs.hasFlavor(pid): continue pdfs[j] = np.array( @@ -109,12 +113,14 @@ def alpha_em(q): for x in xgrid ] ) - - pdfs[11] = photon_100GeV + + runner = Runner(theory, operator_card) + output = runner.get_output() for q2, elem in output.items(): pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) # error_final = np.einsum("ajbk,bk", elem.error, pdfs) - photon_1GeV = pdf_final[11] + photon_1GeV = pdf_final[ph_id] - return photon_1GeV \ No newline at end of file + # we want x * gamma(x) + return xgrid * photon_1GeV \ No newline at end of file From 10fe59baa0c0636e18449bb4a498c9872085ea6b Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 16 Nov 2022 10:14:09 +0100 Subject: [PATCH 004/204] Add layer AddPhoton --- n3fit/src/n3fit/layers/__init__.py | 2 +- n3fit/src/n3fit/layers/rotations.py | 35 +++++++++++++++++++++++++++++ n3fit/src/n3fit/model_gen.py | 3 ++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/layers/__init__.py b/n3fit/src/n3fit/layers/__init__.py index e5839046cc..d7d288bf9b 100644 --- a/n3fit/src/n3fit/layers/__init__.py +++ b/n3fit/src/n3fit/layers/__init__.py @@ -1,5 +1,5 @@ from .preprocessing import Preprocessing -from .rotations import FkRotation, FlavourToEvolution, ObsRotation +from .rotations import FkRotation, FlavourToEvolution, ObsRotation, AddPhoton from .x_operations import xIntegrator, xDivide from .msr_normalization import MSR_Normalization from .DIS import DIS diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 88fa8d722b..aa940a22fa 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -91,6 +91,41 @@ def call(self, pdf_raw): # Concatenating destroys the batch index so we have to regenerate it return op.batchit(ret) +class AddPhoton(MetaLayer): + """ + Changes the value of the photon to non-zero. + + Both input and output have shape (1, None, 14). However, + the input has a photon which is set to zero, while for the + output it is set to its real value. + """ + + def __init__(self, output_dim=14, **kwargs): + self.output_dim = output_dim + super().__init__(**kwargs) + + def call(self, pdfs, pdf_ph): + photon_tensor = op.numpy_to_tensor(np.expand_dims(pdf_ph, axis=1)) + x = op.transpose(pdfs) + pdfs_list=[ + photon_tensor, + x[1], + x[2], + x[3], + x[4], + x[5], + x[6], + x[7], + x[8], + x[9], + x[10], + x[11], + x[12], + x[13] + ] + ret = op.concatenate(pdfs_list) + return op.batchit(ret) + class ObsRotation(MetaLayer): """ diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index a89ce19c19..3062cee64d 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -13,7 +13,7 @@ import numpy as np from n3fit.msr import msr_impose from n3fit.layers import DIS, DY, ObsRotation, losses -from n3fit.layers import Preprocessing, FkRotation, FlavourToEvolution +from n3fit.layers import Preprocessing, FkRotation, FlavourToEvolution, AddPhoton from n3fit.layers.observable import is_unique from n3fit.backends import MetaModel, Input @@ -559,6 +559,7 @@ def pdfNN_layer_generator( # Evolution layer layer_evln = FkRotation(input_shape=(last_layer_nodes,), output_dim=out) + layer_photon = AddPhoton(input_shape=(last_layer_nodes,), output_dim=out) # Basis rotation basis_rotation = FlavourToEvolution(flav_info=flav_info, fitbasis=fitbasis) From 747f589318f8b8657b062c031668c36395e9b459 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Sat, 19 Nov 2022 18:15:19 +0100 Subject: [PATCH 005/204] Implement module for the computation of DIS structure functions --- .../{ => photon_pdf}/compute_photon.py | 39 +++++++++++-------- .../photon_pdf/structure_functions.py | 33 ++++++++++++++++ 2 files changed, 55 insertions(+), 17 deletions(-) rename validphys2/src/validphys/{ => photon_pdf}/compute_photon.py (78%) create mode 100644 validphys2/src/validphys/photon_pdf/structure_functions.py diff --git a/validphys2/src/validphys/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py similarity index 78% rename from validphys2/src/validphys/compute_photon.py rename to validphys2/src/validphys/photon_pdf/compute_photon.py index 69120b22d5..3b0078c30e 100644 --- a/validphys2/src/validphys/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -1,12 +1,14 @@ """Script that calls fiatlux to add the photon PDF.""" -from validphys import lhapdfset +# from validphys import lhapdfset from validphys.api import API +import lhapdf import fiatlux import numpy as np from eko.couplings import Couplings from eko.compatibility import update_theory from eko.runner import Runner +from .structure_functions import StructureFunction import yaml @@ -30,20 +32,8 @@ def photon_1GeV(xgrid, theoryid, fiatlux_runcard, replica): """ if fiatlux_runcard is None : return np.zeros(len(xgrid)) - set = lhapdfset.LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") - qcd_pdfs = set.members[replica] - - def f2(x, q): - # call yadism to give f2 - return 0. - - def fl(x, q): - # call yadism to give fl - return 0. - - def f2lo(x, q): - # call yadism to give f20 - return 0. + pdf_name = fiatlux_runcard["pdf_name"] + qcd_pdfs = lhapdf.mkPDF(pdf_name, replica) theory = API.theoryid(theoryid = theoryid).get_description().copy() theory["nfref"] = None @@ -57,6 +47,7 @@ def f2lo(x, q): q_in2 = q_in ** 2 q_fin = theory["Q0"] theory["Q0"]= q_in + qref = theory["Qref"] # lux = fiatlux.FiatLux(fiatlux_runcard) # we have a dict but fiatlux wants a yaml file @@ -70,8 +61,22 @@ def f2lo(x, q): def alpha_em(q): return couplings.a(q**2)[1] * 4 * np.pi - lux.PlugAlphaQED(alpha_em, theory["Qref"]) - lux.PlugStructureFunctions(f2, fl, f2lo) + lux.PlugAlphaQED(alpha_em, qref) + + path_to_grid = fiatlux_runcard["path_to_grid"] + path_to_F2 = fiatlux_runcard["path_to_F2"] + path_to_FL = fiatlux_runcard["path_to_FL"] + path_to_F2LO = fiatlux_runcard["path_to_F2LO"] + order = theory["order"] + xir = theory["XIR"] + xif = theory["XIF"] + + f2 = StructureFunction(path_to_grid, path_to_F2, pdf_name, order, xir, xif) + fl = StructureFunction(path_to_grid, path_to_FL, pdf_name, order, xir, xif) + f2lo = StructureFunction(path_to_grid, path_to_F2LO, pdf_name, order, xir, xif) + + lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, f2lo.FxQ) + lux.InsertInelasticSplitQ([4.18, 1e100]) operator_card = dict( diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon_pdf/structure_functions.py new file mode 100644 index 0000000000..4b5cee216b --- /dev/null +++ b/validphys2/src/validphys/photon_pdf/structure_functions.py @@ -0,0 +1,33 @@ +from pathlib import Path +import pineappl +import numpy as np +from scipy.interpolate import interp2d + +class StructureFunction : + def __init__(self, path_to_grid, path_to_fktable, pdfs, order, xir, xif): + # self.path_to_grid = Path(path_to_grid) + self.path_to_fktable = Path(path_to_fktable) + # self.grid = pineappl.grid.Grid.read(path_to_grid) + self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) + self.pdfs = pdfs + self.pdgid = int(pdfs.set().get_entry("Particle")) + # self.order = order + # self.xir = xir + # self.xif = xif + self.produce_interpolator(self) + + + def produce_interpolator(self): + x = np.unique(self.fktable.bin_left(1)) + q2 = np.unique(self.fktable.bin_left(0)) + predictions = self.fktable.convolute_with_one(self.pdgid, self.pdfs.xfxQ2) + # here we require that the (x,Q2) couples that we passed + # to pinefarm is a rectangular matrix + grid2D = predictions.reshape(len(x),len(q2)) + # TODO: are len(x) and len(q2) in the correct order? + self.interpolator = interp2d(x, q2, grid2D) + + def FxQ(self, x, Q): + return self.interpolator(x, Q**2) + + From 86fadace171f481051f79bb00130c1de7e2d90f8 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 21 Nov 2022 14:46:21 +0100 Subject: [PATCH 006/204] Remove path_to_grid from StructureFunction --- .../src/validphys/photon_pdf/compute_photon.py | 12 +++++------- .../src/validphys/photon_pdf/structure_functions.py | 5 +---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index 3b0078c30e..e0624697ab 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -63,17 +63,15 @@ def alpha_em(q): lux.PlugAlphaQED(alpha_em, qref) - path_to_grid = fiatlux_runcard["path_to_grid"] path_to_F2 = fiatlux_runcard["path_to_F2"] path_to_FL = fiatlux_runcard["path_to_FL"] path_to_F2LO = fiatlux_runcard["path_to_F2LO"] - order = theory["order"] - xir = theory["XIR"] - xif = theory["XIF"] + # xir = theory["XIR"] + # xif = theory["XIF"] - f2 = StructureFunction(path_to_grid, path_to_F2, pdf_name, order, xir, xif) - fl = StructureFunction(path_to_grid, path_to_FL, pdf_name, order, xir, xif) - f2lo = StructureFunction(path_to_grid, path_to_F2LO, pdf_name, order, xir, xif) + f2 = StructureFunction(path_to_F2, pdf_name) + fl = StructureFunction(path_to_FL, pdf_name) + f2lo = StructureFunction(path_to_F2LO, pdf_name) lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, f2lo.FxQ) diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon_pdf/structure_functions.py index 4b5cee216b..37dbbaaadd 100644 --- a/validphys2/src/validphys/photon_pdf/structure_functions.py +++ b/validphys2/src/validphys/photon_pdf/structure_functions.py @@ -4,16 +4,13 @@ from scipy.interpolate import interp2d class StructureFunction : - def __init__(self, path_to_grid, path_to_fktable, pdfs, order, xir, xif): + def __init__(self, path_to_fktable, pdfs): # self.path_to_grid = Path(path_to_grid) self.path_to_fktable = Path(path_to_fktable) # self.grid = pineappl.grid.Grid.read(path_to_grid) self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs self.pdgid = int(pdfs.set().get_entry("Particle")) - # self.order = order - # self.xir = xir - # self.xif = xif self.produce_interpolator(self) From d24d1d0e12cc43b1db1e18bfac20886c3eac85c5 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 21 Nov 2022 22:54:34 +0100 Subject: [PATCH 007/204] Fix computation of photon pdf --- validphys2/src/validphys/photon_pdf/compute_photon.py | 10 ++++++---- .../src/validphys/photon_pdf/structure_functions.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index e0624697ab..695bb3223d 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -69,9 +69,9 @@ def alpha_em(q): # xir = theory["XIR"] # xif = theory["XIF"] - f2 = StructureFunction(path_to_F2, pdf_name) - fl = StructureFunction(path_to_FL, pdf_name) - f2lo = StructureFunction(path_to_F2LO, pdf_name) + f2 = StructureFunction(path_to_F2, qcd_pdfs) + fl = StructureFunction(path_to_FL, qcd_pdfs) + f2lo = StructureFunction(path_to_F2LO, qcd_pdfs) lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, f2lo.FxQ) @@ -97,9 +97,11 @@ def alpha_em(q): ).items() ) ) + for x in xgrid: + print(lux.EvaluatePhoton(x, q_in2).total / x) photon_100GeV = np.array( - [lux.EvaluatePhoton(x, q2).total / x for x in xgrid] + [lux.EvaluatePhoton(x, q_in2).total / x for x in xgrid] ) # TODO: fiatlux returns gamma(x) or x*gamma(x) ? diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon_pdf/structure_functions.py index 37dbbaaadd..c23c17c021 100644 --- a/validphys2/src/validphys/photon_pdf/structure_functions.py +++ b/validphys2/src/validphys/photon_pdf/structure_functions.py @@ -11,7 +11,7 @@ def __init__(self, path_to_fktable, pdfs): self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs self.pdgid = int(pdfs.set().get_entry("Particle")) - self.produce_interpolator(self) + self.produce_interpolator() def produce_interpolator(self): @@ -20,8 +20,8 @@ def produce_interpolator(self): predictions = self.fktable.convolute_with_one(self.pdgid, self.pdfs.xfxQ2) # here we require that the (x,Q2) couples that we passed # to pinefarm is a rectangular matrix - grid2D = predictions.reshape(len(x),len(q2)) - # TODO: are len(x) and len(q2) in the correct order? + grid2D = predictions.reshape(len(x),len(q2)).T + # TODO: are len(x) and len(q2) in the correct order? self.interpolator = interp2d(x, q2, grid2D) def FxQ(self, x, Q): From 990745c082ea6a33a7727f159d39094f731403fc Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 1 Dec 2022 11:47:04 +0100 Subject: [PATCH 008/204] Modify AddPhoton --- n3fit/src/n3fit/layers/rotations.py | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index aa940a22fa..7f2e7d054b 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -105,25 +105,12 @@ def __init__(self, output_dim=14, **kwargs): super().__init__(**kwargs) def call(self, pdfs, pdf_ph): - photon_tensor = op.numpy_to_tensor(np.expand_dims(pdf_ph, axis=1)) x = op.transpose(pdfs) - pdfs_list=[ - photon_tensor, - x[1], - x[2], - x[3], - x[4], - x[5], - x[6], - x[7], - x[8], - x[9], - x[10], - x[11], - x[12], - x[13] - ] - ret = op.concatenate(pdfs_list) + pdf_ph_t = op.transpose(pdf_ph) + pdf_raw_list = [x[i] for i in range(14)] + pdf_raw_list[0] = pdf_ph_t[0] + ret = op.concatenate(x) + # Concatenating destroys the batch index so we have to regenerate it return op.batchit(ret) From 22ca1d190eb70db91dd81a1bbb71a20eaac79230 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 1 Dec 2022 16:54:25 +0100 Subject: [PATCH 009/204] Rename photon_1GeV -> photon_fitting_scale --- .../validphys/photon_pdf/compute_photon.py | 47 +++++++++++++------ .../photon_pdf/structure_functions.py | 2 + 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index 695bb3223d..04d047e861 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -12,23 +12,23 @@ import yaml -def photon_1GeV(xgrid, theoryid, fiatlux_runcard, replica): +def photon_fitting_scale(xgrid, theoryid, fiatlux_runcard, replica): r""" Compute the photon PDF for every point in the grid xgrid. Parameters ---------- - xgrid: numpy.array - grid of the x points - pdf_name: string - name of the QCD set - replica: int - number of replica + xgrid: numpy.array + grid of the x points + pdf_name: string + name of the QCD set + replica: int + number of replica Returns ------- - photon_1GeV: numpy.array - photon PDF at the scale 1 GeV + photon_fitting_scale: numpy.array + photon PDF at the scale 1 GeV """ if fiatlux_runcard is None : return np.zeros(len(xgrid)) @@ -65,15 +65,34 @@ def alpha_em(q): path_to_F2 = fiatlux_runcard["path_to_F2"] path_to_FL = fiatlux_runcard["path_to_FL"] - path_to_F2LO = fiatlux_runcard["path_to_F2LO"] # xir = theory["XIR"] # xif = theory["XIF"] + + def F2LO(x, Q): + mcharm = theory["mc"] + mbottom = theory["mb"] + mtop = theory["mt"] + # at LO we use ZM-VFS + if Q < mcharm : + hq = 3 + elif Q < mbottom : + hq = 4 + elif Q < mtop : + hq = 5 + else : + hq = 6 + e2u = 4/9 + e2d = 1/9 + e2q = [e2d, e2u, e2d, e2u, e2d, e2u] + res = 0 + for i in range(1, hq+1): + res += e2q[i-1] * (qcd_pdfs.xfxQ(x, Q)[i] + qcd_pdfs.xfxQ(x, Q)[-i]) + return res f2 = StructureFunction(path_to_F2, qcd_pdfs) fl = StructureFunction(path_to_FL, qcd_pdfs) - f2lo = StructureFunction(path_to_F2LO, qcd_pdfs) - lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, f2lo.FxQ) + lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, F2LO) lux.InsertInelasticSplitQ([4.18, 1e100]) @@ -125,7 +144,7 @@ def alpha_em(q): pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) # error_final = np.einsum("ajbk,bk", elem.error, pdfs) - photon_1GeV = pdf_final[ph_id] + photon_fitting_scale = pdf_final[ph_id] # we want x * gamma(x) - return xgrid * photon_1GeV \ No newline at end of file + return xgrid * photon_fitting_scale \ No newline at end of file diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon_pdf/structure_functions.py index c23c17c021..2193250c52 100644 --- a/validphys2/src/validphys/photon_pdf/structure_functions.py +++ b/validphys2/src/validphys/photon_pdf/structure_functions.py @@ -2,6 +2,7 @@ import pineappl import numpy as np from scipy.interpolate import interp2d +# from scipy.interpolate import RectBivariateSpline class StructureFunction : def __init__(self, path_to_fktable, pdfs): @@ -23,6 +24,7 @@ def produce_interpolator(self): grid2D = predictions.reshape(len(x),len(q2)).T # TODO: are len(x) and len(q2) in the correct order? self.interpolator = interp2d(x, q2, grid2D) + # consider using scipy.RectBivariateSpline. def FxQ(self, x, Q): return self.interpolator(x, Q**2) From 418c02710ac5c8a8f9a71c165f2c89bb6025d5d3 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 2 Dec 2022 15:22:32 +0100 Subject: [PATCH 010/204] Rename photon_1GeV -> photon_fitting_scale --- n3fit/src/n3fit/model_gen.py | 3 ++- n3fit/src/n3fit/model_trainer.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 3062cee64d..f8ca51cd39 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -535,6 +535,7 @@ def pdfNN_layer_generator( # First prepare the input for the PDF model and any scaling if needed placeholder_input = Input(shape=(None, 1), batch_size=1) + # placeholder_ph = ??? subtract_one = False process_input = Lambda(lambda x: x) @@ -559,7 +560,7 @@ def pdfNN_layer_generator( # Evolution layer layer_evln = FkRotation(input_shape=(last_layer_nodes,), output_dim=out) - layer_photon = AddPhoton(input_shape=(last_layer_nodes,), output_dim=out) + layer_photon = AddPhoton(input_shape=(out,), output_dim=out) # Basis rotation basis_rotation = FlavourToEvolution(flav_info=flav_info, fitbasis=fitbasis) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index ce3bfb293a..53db0b6426 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -19,7 +19,7 @@ from n3fit.vpinterface import N3PDF import n3fit.hyper_optimization.penalties import n3fit.hyper_optimization.rewards -from validphys.compute_photon import photon_1GeV +from validphys.photon_pdf.compute_photon import photon_fitting_scale from n3fit.scripts.n3fit_exec import N3FIT_FIXED_CONFIG log = logging.getLogger(__name__) @@ -361,7 +361,7 @@ def _model_generation(self, pdf_models, partition, partition_idx): input_layer = op.numpy_to_input(input_arr) photon_list = [ - np.array([photon_1GeV( + np.array([photon_fitting_scale( xgrid=grid[0], theoryid=N3FIT_FIXED_CONFIG['theoryid'], fiatlux_runcard = N3FIT_FIXED_CONFIG['fiatlux'], From e678d00d4da925cc86adc1811623fb610ae1dfb5 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 2 Dec 2022 15:23:45 +0100 Subject: [PATCH 011/204] Comment unnecessary lines --- validphys2/src/validphys/photon_pdf/compute_photon.py | 10 +++++----- .../src/validphys/photon_pdf/structure_functions.py | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index 04d047e861..8beb6e5896 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -36,12 +36,12 @@ def photon_fitting_scale(xgrid, theoryid, fiatlux_runcard, replica): qcd_pdfs = lhapdf.mkPDF(pdf_name, replica) theory = API.theoryid(theoryid = theoryid).get_description().copy() - theory["nfref"] = None - theory["nf0"] = None - theory["fact_to_ren_scale_ratio"] = 1. + # theory["nfref"] = None + # theory["nf0"] = None + # theory["fact_to_ren_scale_ratio"] = 1. theory["ModSV"] = None - theory["IC"]=0 - theory["IB"]=0 + # theory["IC"]=0 + # theory["IB"]=0 theory["FNS"] = "VFNS" q_in = 100 q_in2 = q_in ** 2 diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon_pdf/structure_functions.py index 2193250c52..2e380b9fbc 100644 --- a/validphys2/src/validphys/photon_pdf/structure_functions.py +++ b/validphys2/src/validphys/photon_pdf/structure_functions.py @@ -28,5 +28,3 @@ def produce_interpolator(self): def FxQ(self, x, Q): return self.interpolator(x, Q**2) - - From 03b247ef2be206e1c56d908df7cb445cdb8527f1 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 6 Dec 2022 10:41:36 +0100 Subject: [PATCH 012/204] Restructure compute_photon --- .../validphys/photon_pdf/compute_photon.py | 229 +++++++++--------- .../photon_pdf/structure_functions.py | 4 +- 2 files changed, 116 insertions(+), 117 deletions(-) diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index 8beb6e5896..6ea5fc9a0c 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -12,66 +12,34 @@ import yaml -def photon_fitting_scale(xgrid, theoryid, fiatlux_runcard, replica): - r""" - Compute the photon PDF for every point in the grid xgrid. - - Parameters - ---------- - xgrid: numpy.array - grid of the x points - pdf_name: string - name of the QCD set - replica: int - number of replica - - Returns - ------- - photon_fitting_scale: numpy.array - photon PDF at the scale 1 GeV - """ - if fiatlux_runcard is None : - return np.zeros(len(xgrid)) - pdf_name = fiatlux_runcard["pdf_name"] - qcd_pdfs = lhapdf.mkPDF(pdf_name, replica) - - theory = API.theoryid(theoryid = theoryid).get_description().copy() - # theory["nfref"] = None - # theory["nf0"] = None - # theory["fact_to_ren_scale_ratio"] = 1. - theory["ModSV"] = None - # theory["IC"]=0 - # theory["IB"]=0 - theory["FNS"] = "VFNS" - q_in = 100 - q_in2 = q_in ** 2 - q_fin = theory["Q0"] - theory["Q0"]= q_in - qref = theory["Qref"] - - # lux = fiatlux.FiatLux(fiatlux_runcard) - # we have a dict but fiatlux wants a yaml file - # TODO : remove this trick - ff = open('fiatlux_runcard.yml', 'w+') - yaml.dump(fiatlux_runcard, ff) - - lux = fiatlux.FiatLux('fiatlux_runcard.yml') - - couplings = Couplings.from_dict(update_theory(theory)) - def alpha_em(q): - return couplings.a(q**2)[1] * 4 * np.pi - - lux.PlugAlphaQED(alpha_em, qref) - - path_to_F2 = fiatlux_runcard["path_to_F2"] - path_to_FL = fiatlux_runcard["path_to_FL"] - # xir = theory["XIR"] - # xif = theory["XIF"] +class Photon: + def __init__(self, theoryid, fiatlux_runcard, replica=0): + # TODO : for the moment we do the 0-th replica then we change it + self.theory = API.theoryid(theoryid = theoryid).get_description().copy() + self.fiatlux_runcard = fiatlux_runcard + # theory["nfref"] = None + # theory["nf0"] = None + # theory["fact_to_ren_scale_ratio"] = 1. + self.theory["ModSV"] = None + # theory["IC"]=0 + # theory["IB"]=0 + self.theory["FNS"] = "VFNS" + self.q_in = 100 + self.q_in2 = self.q_in ** 2 + self.q_fin = self.theory["Q0"] + self.theory["Q0"]= self.q_in + self.qref = self.theory["Qref"] + # xir = theory["XIR"] + # xif = theory["XIF"] + self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) + self.couplings = Couplings.from_dict(update_theory(self.theory)) + self.path_to_F2 = fiatlux_runcard["path_to_F2"] + self.path_to_FL = fiatlux_runcard["path_to_FL"] - def F2LO(x, Q): - mcharm = theory["mc"] - mbottom = theory["mb"] - mtop = theory["mt"] + def F2LO(self, x, Q): + mcharm = self.theory["mc"] + mbottom = self.theory["mb"] + mtop = self.theory["mt"] # at LO we use ZM-VFS if Q < mcharm : hq = 3 @@ -83,68 +51,101 @@ def F2LO(x, Q): hq = 6 e2u = 4/9 e2d = 1/9 - e2q = [e2d, e2u, e2d, e2u, e2d, e2u] + e2q = [e2d, e2u, e2d, e2u, e2d, e2u] # d u s c b t res = 0 for i in range(1, hq+1): - res += e2q[i-1] * (qcd_pdfs.xfxQ(x, Q)[i] + qcd_pdfs.xfxQ(x, Q)[-i]) + res += e2q[i-1] * (self.qcd_pdfs.xfxQ(x, Q)[i] + self.qcd_pdfs.xfxQ(x, Q)[-i]) return res - f2 = StructureFunction(path_to_F2, qcd_pdfs) - fl = StructureFunction(path_to_FL, qcd_pdfs) - - lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, F2LO) - - lux.InsertInelasticSplitQ([4.18, 1e100]) + def alpha_em(self, q): + return self.couplings.a(q**2)[1] * 4 * np.pi - operator_card = dict( - sorted( - dict( - interpolation_xgrid=xgrid.tolist(), - interpolation_polynomial_degree=4, - interpolation_is_log=True, - ev_op_max_order=10, - ev_op_iterations=10, - backward_inversion="expanded", - n_integration_cores=0, - debug_skip_non_singlet=False, - debug_skip_singlet=False, - Q2grid=[q_fin**2], - inputgrid=None, - targetgrid=None, - inputpids=None, - targetpids=None, - ).items() - ) - ) - for x in xgrid: - print(lux.EvaluatePhoton(x, q_in2).total / x) + def photon_fitting_scale(self, xgrid): + r""" + Compute the photon PDF for every point in the grid xgrid. - photon_100GeV = np.array( - [lux.EvaluatePhoton(x, q_in2).total / x for x in xgrid] - ) - # TODO: fiatlux returns gamma(x) or x*gamma(x) ? - - pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) - for j, pid in enumerate(output.rotations.inputpids): - if pid == 22 : - pdfs[j] = photon_100GeV - ph_id = j - if not qcd_pdfs.hasFlavor(pid): - continue - pdfs[j] = np.array( - [ - qcd_pdfs.xfxQ2(pid, x, q_in2) / x - for x in xgrid - ] + Parameters + ---------- + xgrid: numpy.array + grid of the x points + pdf_name: string + name of the QCD set + replica: int + number of replica + + Returns + ------- + photon_fitting_scale: numpy.array + photon PDF at the scale 1 GeV + """ + if self.fiatlux_runcard is None : + return np.zeros(len(xgrid)) + + # lux = fiatlux.FiatLux(fiatlux_runcard) + # we have a dict but fiatlux wants a yaml file + # TODO : remove this trick + ff = open('fiatlux_runcard.yml', 'w+') + yaml.dump(self.fiatlux_runcard, ff) + + lux = fiatlux.FiatLux('fiatlux_runcard.yml') + lux.PlugAlphaQED(self.alpha_em, self.qref) + + f2 = StructureFunction(self.path_to_F2, self.qcd_pdfs) + fl = StructureFunction(self.path_to_FL, self.qcd_pdfs) + + lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, self.F2LO) + + lux.InsertInelasticSplitQ([4.18, 1e100]) + + photon_100GeV = np.array( + [lux.EvaluatePhoton(x, self.q_in2).total / x for x in xgrid] + ) + # TODO: fiatlux returns gamma(x) or x*gamma(x) ? + + pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) + for j, pid in enumerate(output.rotations.inputpids): + if pid == 22 : + pdfs[j] = photon_100GeV + ph_id = j + if not self.qcd_pdfs.hasFlavor(pid): + continue + pdfs[j] = np.array( + [ + self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x + for x in xgrid + ] + ) + + operator_card = dict( + sorted( + dict( + interpolation_xgrid=xgrid.tolist(), + interpolation_polynomial_degree=4, + interpolation_is_log=True, + ev_op_max_order=10, + ev_op_iterations=10, + backward_inversion="expanded", + n_integration_cores=0, + debug_skip_non_singlet=False, + debug_skip_singlet=False, + Q2grid=[self.q_fin**2], + inputgrid=None, + targetgrid=None, + inputpids=None, + targetpids=None, + ).items() + ) ) - runner = Runner(theory, operator_card) - output = runner.get_output() - for q2, elem in output.items(): - pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) - # error_final = np.einsum("ajbk,bk", elem.error, pdfs) + # TODO : this EKO should be precomputed and stored since it never changes + runner = Runner(self.theory, operator_card) + output = runner.get_output() + for q2, elem in output.items(): + pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) + # error_final = np.einsum("ajbk,bk", elem.error, pdfs) - photon_fitting_scale = pdf_final[ph_id] + photon_fitting_scale = pdf_final[ph_id] - # we want x * gamma(x) - return xgrid * photon_fitting_scale \ No newline at end of file + # we want x * gamma(x) + return xgrid * photon_fitting_scale + \ No newline at end of file diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon_pdf/structure_functions.py index 2e380b9fbc..e2134efc8d 100644 --- a/validphys2/src/validphys/photon_pdf/structure_functions.py +++ b/validphys2/src/validphys/photon_pdf/structure_functions.py @@ -2,7 +2,6 @@ import pineappl import numpy as np from scipy.interpolate import interp2d -# from scipy.interpolate import RectBivariateSpline class StructureFunction : def __init__(self, path_to_fktable, pdfs): @@ -24,7 +23,6 @@ def produce_interpolator(self): grid2D = predictions.reshape(len(x),len(q2)).T # TODO: are len(x) and len(q2) in the correct order? self.interpolator = interp2d(x, q2, grid2D) - # consider using scipy.RectBivariateSpline. def FxQ(self, x, Q): - return self.interpolator(x, Q**2) + return self.interpolator(x, Q**2)[0] From 64d98d0d5d239f48f734a9658b440b2a132ccdc1 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 6 Dec 2022 10:44:28 +0100 Subject: [PATCH 013/204] Modify layer AddPhoton --- n3fit/src/n3fit/layers/rotations.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 5cab4f0614..d47a59ed72 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -70,6 +70,7 @@ def __init__(self, output_dim=14, name="evolution", **kwargs): def call(self, pdf_raw): # Transpose the PDF so that the flavour index is the first one + import ipdb; ipdb.set_trace() x = op.transpose(pdf_raw) pdf_raw_list = [ 0 * x[0], # photon @@ -100,18 +101,20 @@ class AddPhoton(MetaLayer): output it is set to its real value. """ - def __init__(self, output_dim=14, **kwargs): + def __init__(self, pdf_ph, output_dim=14, **kwargs): + # pdf_ph must be a tensor of shape (1, xgrid) + self.pdf_ph = pdf_ph self.output_dim = output_dim super().__init__(**kwargs) - def call(self, pdfs, pdf_ph): - x = op.transpose(pdfs) - pdf_ph_t = op.transpose(pdf_ph) - pdf_raw_list = [x[i] for i in range(14)] - pdf_raw_list[0] = pdf_ph_t[0] - ret = op.concatenate(x) - # Concatenating destroys the batch index so we have to regenerate it - return op.batchit(ret) + def call(self, pdfs): + pdf_list = [self.pdf_ph] + for i in range(1, self.output_dim): + pdf_list.append(pdfs[:,:,i]) + # all the elements of the list must have the same + # shape, i.e. (1, xgrid), so that op.stack returns + # a tensor of shape (1, xgrid, 14) + return op.stack(pdf_list, axis=2) class ObsRotation(MetaLayer): From 6ac76525ac25c303e850ee50be850a888c1c3da4 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 6 Dec 2022 12:05:09 +0100 Subject: [PATCH 014/204] Remove ipdb --- n3fit/src/n3fit/layers/rotations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index d47a59ed72..390cc7d96b 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -70,7 +70,6 @@ def __init__(self, output_dim=14, name="evolution", **kwargs): def call(self, pdf_raw): # Transpose the PDF so that the flavour index is the first one - import ipdb; ipdb.set_trace() x = op.transpose(pdf_raw) pdf_raw_list = [ 0 * x[0], # photon From 59d4639409f085290f45febadce2294d171fbe71 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 6 Dec 2022 12:09:21 +0100 Subject: [PATCH 015/204] Add theoryid and fiatlux_runcard to ModelTrainer --- n3fit/src/n3fit/model_trainer.py | 10 ++++++++-- n3fit/src/n3fit/performfit.py | 3 +++ n3fit/src/n3fit/scripts/n3fit_exec.py | 3 +-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 9d166e1c23..86abd5b823 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -20,8 +20,7 @@ from n3fit.vpinterface import N3PDF import n3fit.hyper_optimization.penalties import n3fit.hyper_optimization.rewards -from validphys.photon_pdf.compute_photon import photon_fitting_scale -from n3fit.scripts.n3fit_exec import N3FIT_FIXED_CONFIG +from validphys.photon_pdf.compute_photon import Photon log = logging.getLogger(__name__) @@ -101,6 +100,8 @@ def __init__( model_file=None, sum_rules=None, parallel_models=1, + theoryid=None, + fiatlux_runcard=None, replica_id=None, ): """ @@ -154,6 +155,8 @@ def __init__( self.all_datasets = [] self._scaler = None self._parallel_models = parallel_models + self.theoryid = theoryid + self.fiatlux_runcard = fiatlux_runcard self.replica_id = replica_id # Initialise internal variables which define behaviour @@ -858,6 +861,9 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() + + # compute photon here + # photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) ### Training loop for k, partition in enumerate(self.kpartitions): diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 9d98f73bdf..7f84adf443 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -23,6 +23,7 @@ def performfit( posdatasets_fitting_pos_dict, integdatasets_fitting_integ_dict, theoryid, + fiatlux, basis, fitbasis, sum_rules=True, @@ -194,6 +195,8 @@ def performfit( model_file=load, sum_rules=sum_rules, parallel_models=n_models, + theoryid=theoryid, + fiatlux_runcard=fiatlux, replica_id=replica_idxs[0] ) diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 06554332e4..032f2d748c 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -145,8 +145,7 @@ def from_yaml(cls, o, *args, **kwargs): validation_action = namespace + "validation_pseudodata" N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) - if (thconfig:=file_content.get('theory')) is not None: - N3FIT_FIXED_CONFIG['theoryid']=thconfig.get('theoryid', True) + if (thconfig:=file_content.get('fiatlux')) is not None: N3FIT_FIXED_CONFIG['fiatlux']=thconfig else : From 2118528450313d85ce11896ac4e0a1181406a52d Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 6 Dec 2022 14:57:16 +0100 Subject: [PATCH 016/204] Add photon in model_trainer --- n3fit/src/n3fit/model_gen.py | 6 +-- n3fit/src/n3fit/model_trainer.py | 17 ++++++-- .../validphys/photon_pdf/compute_photon.py | 40 ++++++++++--------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index acdfcc3a34..e0ddb83617 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -403,6 +403,7 @@ def pdfNN_layer_generator( impose_sumrule=None, scaler=None, parallel_models=1, + pdf_ph=None, ): # pylint: disable=too-many-locals """ Generates the PDF model which takes as input a point in x (from 0 to 1) @@ -535,7 +536,6 @@ def pdfNN_layer_generator( # First prepare the input for the PDF model and any scaling if needed placeholder_input = Input(shape=(None, 1), batch_size=1) - # placeholder_ph = ??? subtract_one = False process_input = Lambda(lambda x: x) @@ -560,7 +560,7 @@ def pdfNN_layer_generator( # Evolution layer layer_evln = FkRotation(input_shape=(last_layer_nodes,), output_dim=out) - layer_photon = AddPhoton(input_shape=(out,), output_dim=out) + layer_photon = AddPhoton(pdf_ph=pdf_ph, input_shape=(out,), output_dim=out) # Basis rotation basis_rotation = FlavourToEvolution(flav_info=flav_info, fitbasis=fitbasis) @@ -639,7 +639,7 @@ def layer_fitbasis(x): # Rotation layer, changes from the 8-basis to the 14-basis def layer_pdf(x): - return layer_evln(layer_fitbasis(x)) + return layer_photon(layer_evln(layer_fitbasis(x))) # Final PDF (apply normalization) final_pdf = sumrule_layer(layer_pdf) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 86abd5b823..2d6392fd8a 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -350,7 +350,7 @@ def _xgrid_generation(self): op.split, op_args=sp_ar, op_kwargs=sp_kw, name="pdf_split" ) - return InputInfo(input_layer, sp_layer, inputs_idx) + return InputInfo(input_layer, sp_layer, inputs_idx), inputs_unique def _model_generation(self, xinput, pdf_models, partition, partition_idx): """ @@ -640,6 +640,7 @@ def _generate_pdf( regularizer, regularizer_args, seed, + pdf_ph, ): """ Defines the internal variable layer_pdf @@ -691,6 +692,7 @@ def _generate_pdf( impose_sumrule=self.impose_sumrule, scaler=self._scaler, parallel_models=self._parallel_models, + pdf_ph=pdf_ph, ) return pdf_models @@ -860,10 +862,16 @@ def hyperparametrizable(self, params): exp_models = [] # Generate the grid in x, note this is the same for all partitions - xinput = self._xgrid_generation() + xinput, inputs_unique = self._xgrid_generation() - # compute photon here - # photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) + # compute photon: + photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) + photon_array = np.array([]) + for i in inputs_unique: + xgrid = i[0] + photon_array = np.append(photon_array, photon.photon_fitting_scale(xgrid)) + + ph_pdf = op.batchit(op.numpy_to_tensor(photon_array)) ### Training loop for k, partition in enumerate(self.kpartitions): @@ -883,6 +891,7 @@ def hyperparametrizable(self, params): params.get("regularizer", None), # regularizer optional params.get("regularizer_args", None), seeds, + ph_pdf, ) # Model generation joins all the different observable layers diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index 6ea5fc9a0c..802a10e27d 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -15,26 +15,27 @@ class Photon: def __init__(self, theoryid, fiatlux_runcard, replica=0): # TODO : for the moment we do the 0-th replica then we change it - self.theory = API.theoryid(theoryid = theoryid).get_description().copy() + self.theory = API.theoryid(theoryid = theoryid.id).get_description().copy() self.fiatlux_runcard = fiatlux_runcard - # theory["nfref"] = None - # theory["nf0"] = None - # theory["fact_to_ren_scale_ratio"] = 1. - self.theory["ModSV"] = None - # theory["IC"]=0 - # theory["IB"]=0 - self.theory["FNS"] = "VFNS" - self.q_in = 100 - self.q_in2 = self.q_in ** 2 - self.q_fin = self.theory["Q0"] - self.theory["Q0"]= self.q_in - self.qref = self.theory["Qref"] - # xir = theory["XIR"] - # xif = theory["XIF"] - self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) - self.couplings = Couplings.from_dict(update_theory(self.theory)) - self.path_to_F2 = fiatlux_runcard["path_to_F2"] - self.path_to_FL = fiatlux_runcard["path_to_FL"] + if fiatlux_runcard is not None: + # theory["nfref"] = None + # theory["nf0"] = None + # theory["fact_to_ren_scale_ratio"] = 1. + self.theory["ModSV"] = None + # theory["IC"]=0 + # theory["IB"]=0 + self.theory["FNS"] = "VFNS" + self.q_in = 100 + self.q_in2 = self.q_in ** 2 + self.q_fin = self.theory["Q0"] + self.theory["Q0"]= self.q_in + self.qref = self.theory["Qref"] + # xir = theory["XIR"] + # xif = theory["XIF"] + self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) + self.couplings = Couplings.from_dict(update_theory(self.theory)) + self.path_to_F2 = fiatlux_runcard["path_to_F2"] + self.path_to_FL = fiatlux_runcard["path_to_FL"] def F2LO(self, x, Q): mcharm = self.theory["mc"] @@ -140,6 +141,7 @@ def photon_fitting_scale(self, xgrid): # TODO : this EKO should be precomputed and stored since it never changes runner = Runner(self.theory, operator_card) output = runner.get_output() + for q2, elem in output.items(): pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) # error_final = np.einsum("ajbk,bk", elem.error, pdfs) From c3b4aa878ab7df177f9fe60be3574dcf0c2bacf0 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 9 Dec 2022 12:20:39 +0100 Subject: [PATCH 017/204] Change calling of theory in Photon --- .../validphys/photon_pdf/compute_photon.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index 802a10e27d..174a553e35 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -15,23 +15,23 @@ class Photon: def __init__(self, theoryid, fiatlux_runcard, replica=0): # TODO : for the moment we do the 0-th replica then we change it - self.theory = API.theoryid(theoryid = theoryid.id).get_description().copy() + self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: - # theory["nfref"] = None - # theory["nf0"] = None - # theory["fact_to_ren_scale_ratio"] = 1. + # the commented objects should be passed from theory runcard, + # however EKO complains that he doesn't find them + # self.theory["nfref"] = None + # self.theory["nf0"] = None + # self.theory["fact_to_ren_scale_ratio"] = 1. self.theory["ModSV"] = None - # theory["IC"]=0 - # theory["IB"]=0 + # self.theory["IC"] = 1 + # self.theory["IB"] = 0 self.theory["FNS"] = "VFNS" self.q_in = 100 self.q_in2 = self.q_in ** 2 self.q_fin = self.theory["Q0"] self.theory["Q0"]= self.q_in self.qref = self.theory["Qref"] - # xir = theory["XIR"] - # xif = theory["XIF"] self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) self.couplings = Couplings.from_dict(update_theory(self.theory)) self.path_to_F2 = fiatlux_runcard["path_to_F2"] @@ -98,9 +98,10 @@ def photon_fitting_scale(self, xgrid): lux.InsertInelasticSplitQ([4.18, 1e100]) - photon_100GeV = np.array( - [lux.EvaluatePhoton(x, self.q_in2).total / x for x in xgrid] - ) + photon_100GeV = np.zeros(len(xgrid)) + for i, x in enumerate(xgrid): + print("computing grid point", i+1, "/", len(xgrid)) + photon_100GeV[i] = lux.EvaluatePhoton(x, self.q_in2).total / x # TODO: fiatlux returns gamma(x) or x*gamma(x) ? pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) From 383f2bac4b077fa7f1e72de4e12f999c028fc6d9 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 9 Dec 2022 12:42:20 +0100 Subject: [PATCH 018/204] Modify call in AddPhoton --- n3fit/src/n3fit/layers/rotations.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 390cc7d96b..878d074c46 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -100,16 +100,14 @@ class AddPhoton(MetaLayer): output it is set to its real value. """ - def __init__(self, pdf_ph, output_dim=14, **kwargs): + def __init__(self, pdf_ph, output_dim, **kwargs): # pdf_ph must be a tensor of shape (1, xgrid) self.pdf_ph = pdf_ph self.output_dim = output_dim super().__init__(**kwargs) def call(self, pdfs): - pdf_list = [self.pdf_ph] - for i in range(1, self.output_dim): - pdf_list.append(pdfs[:,:,i]) + pdf_list = [self.pdf_ph] + [pdfs[:,:,i] for i in range(1,self.output_dim)] # all the elements of the list must have the same # shape, i.e. (1, xgrid), so that op.stack returns # a tensor of shape (1, xgrid, 14) From 0b565f971183d7557377ca18b21c84b6fd6e9da7 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 9 Dec 2022 12:48:51 +0100 Subject: [PATCH 019/204] Split rotation and photon layer --- n3fit/src/n3fit/model_gen.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index e0ddb83617..e4121702c8 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -639,10 +639,14 @@ def layer_fitbasis(x): # Rotation layer, changes from the 8-basis to the 14-basis def layer_pdf(x): - return layer_photon(layer_evln(layer_fitbasis(x))) + return layer_evln(layer_fitbasis(x)) + + # Photon layer, changes the photon from zero to non-zero + def apply_photon(x): + return layer_photon(layer_pdf(x)) # Final PDF (apply normalization) - final_pdf = sumrule_layer(layer_pdf) + final_pdf = sumrule_layer(apply_photon) # Create the model pdf_model = MetaModel( From cfed0c4770fc287bb86d4a409ea65813c905923f Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 9 Dec 2022 14:20:40 +0100 Subject: [PATCH 020/204] Clean calling of fiatlux --- n3fit/src/n3fit/model_trainer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 2d6392fd8a..6afb6fd8e1 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -866,11 +866,7 @@ def hyperparametrizable(self, params): # compute photon: photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) - photon_array = np.array([]) - for i in inputs_unique: - xgrid = i[0] - photon_array = np.append(photon_array, photon.photon_fitting_scale(xgrid)) - + photon_array = np.concatenate([photon.photon_fitting_scale(xgrid[0]) for xgrid in inputs_unique]) ph_pdf = op.batchit(op.numpy_to_tensor(photon_array)) ### Training loop From 26f021490db487cafae11651dd4fca9b3a92192a Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 9 Dec 2022 16:51:18 +0100 Subject: [PATCH 021/204] Use RectBivariateSpline instead of interp2d --- .../src/validphys/photon_pdf/structure_functions.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon_pdf/structure_functions.py index e2134efc8d..e5b392b0fc 100644 --- a/validphys2/src/validphys/photon_pdf/structure_functions.py +++ b/validphys2/src/validphys/photon_pdf/structure_functions.py @@ -1,13 +1,11 @@ from pathlib import Path import pineappl import numpy as np -from scipy.interpolate import interp2d +from scipy.interpolate import RectBivariateSpline class StructureFunction : def __init__(self, path_to_fktable, pdfs): - # self.path_to_grid = Path(path_to_grid) self.path_to_fktable = Path(path_to_fktable) - # self.grid = pineappl.grid.Grid.read(path_to_grid) self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs self.pdgid = int(pdfs.set().get_entry("Particle")) @@ -20,9 +18,10 @@ def produce_interpolator(self): predictions = self.fktable.convolute_with_one(self.pdgid, self.pdfs.xfxQ2) # here we require that the (x,Q2) couples that we passed # to pinefarm is a rectangular matrix - grid2D = predictions.reshape(len(x),len(q2)).T - # TODO: are len(x) and len(q2) in the correct order? - self.interpolator = interp2d(x, q2, grid2D) + + grid2D = predictions.reshape(len(x),len(q2)) + # RectBivariateSpline is faster than interp2d + self.interpolator = RectBivariateSpline(x, q2, grid2D) def FxQ(self, x, Q): - return self.interpolator(x, Q**2)[0] + return self.interpolator(x, Q**2)[0,0] From d90ea4dd0b73f755a8309b033391bcef2fe0c032 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 9 Dec 2022 23:07:51 +0100 Subject: [PATCH 022/204] Refactor passing of xgrid --- n3fit/src/n3fit/layers/rotations.py | 8 +- n3fit/src/n3fit/model_trainer.py | 8 +- .../validphys/photon_pdf/compute_photon.py | 150 +++++++++++------- 3 files changed, 100 insertions(+), 66 deletions(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 878d074c46..50e5ba8f9a 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -101,17 +101,13 @@ class AddPhoton(MetaLayer): """ def __init__(self, pdf_ph, output_dim, **kwargs): - # pdf_ph must be a tensor of shape (1, xgrid) + # pdf_ph must be a tensor of shape (1, xgrid, 1) self.pdf_ph = pdf_ph self.output_dim = output_dim super().__init__(**kwargs) def call(self, pdfs): - pdf_list = [self.pdf_ph] + [pdfs[:,:,i] for i in range(1,self.output_dim)] - # all the elements of the list must have the same - # shape, i.e. (1, xgrid), so that op.stack returns - # a tensor of shape (1, xgrid, 14) - return op.stack(pdf_list, axis=2) + return op.concatenate([self.pdf_ph, pdfs[:,:,1:]], axis=-1) class ObsRotation(MetaLayer): diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 6afb6fd8e1..7b8f6375ec 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -350,7 +350,7 @@ def _xgrid_generation(self): op.split, op_args=sp_ar, op_kwargs=sp_kw, name="pdf_split" ) - return InputInfo(input_layer, sp_layer, inputs_idx), inputs_unique + return InputInfo(input_layer, sp_layer, inputs_idx) def _model_generation(self, xinput, pdf_models, partition, partition_idx): """ @@ -862,12 +862,12 @@ def hyperparametrizable(self, params): exp_models = [] # Generate the grid in x, note this is the same for all partitions - xinput, inputs_unique = self._xgrid_generation() + xinput = self._xgrid_generation() # compute photon: photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) - photon_array = np.concatenate([photon.photon_fitting_scale(xgrid[0]) for xgrid in inputs_unique]) - ph_pdf = op.batchit(op.numpy_to_tensor(photon_array)) + photon_array = photon.photon_fitting_scale(xinput.input_l.tensor_content[0,:,0]) + ph_pdf = op.batchit(op.numpy_to_tensor(photon_array[:, np.newaxis])) ### Training loop for k, partition in enumerate(self.kpartitions): diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon_pdf/compute_photon.py index 174a553e35..7be1cecf6a 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon_pdf/compute_photon.py @@ -20,12 +20,12 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): if fiatlux_runcard is not None: # the commented objects should be passed from theory runcard, # however EKO complains that he doesn't find them - # self.theory["nfref"] = None - # self.theory["nf0"] = None - # self.theory["fact_to_ren_scale_ratio"] = 1. + self.theory["nfref"] = None + self.theory["nf0"] = None + self.theory["fact_to_ren_scale_ratio"] = 1. self.theory["ModSV"] = None - # self.theory["IC"] = 1 - # self.theory["IB"] = 0 + self.theory["IC"] = 1 + self.theory["IB"] = 0 self.theory["FNS"] = "VFNS" self.q_in = 100 self.q_in2 = self.q_in ** 2 @@ -37,6 +37,16 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.path_to_F2 = fiatlux_runcard["path_to_F2"] self.path_to_FL = fiatlux_runcard["path_to_FL"] + def exctract_grids(self, xgrids): + xgrid_list = [] + imin = 0 + for i in range(1, len(xgrids)): + if xgrids[i-1] > xgrids[i] : + xgrid_list.append(xgrids[imin:i]) + imin = i + xgrid_list.append(xgrids[imin:]) + return xgrid_list + def F2LO(self, x, Q): mcharm = self.theory["mc"] mbottom = self.theory["mb"] @@ -61,7 +71,7 @@ def F2LO(self, x, Q): def alpha_em(self, q): return self.couplings.a(q**2)[1] * 4 * np.pi - def photon_fitting_scale(self, xgrid): + def photon_fitting_scale(self, xgrids): r""" Compute the photon PDF for every point in the grid xgrid. @@ -80,7 +90,9 @@ def photon_fitting_scale(self, xgrid): photon PDF at the scale 1 GeV """ if self.fiatlux_runcard is None : - return np.zeros(len(xgrid)) + return np.zeros(len(xgrids)) + + xgrid_list = self.exctract_grids(xgrids) # lux = fiatlux.FiatLux(fiatlux_runcard) # we have a dict but fiatlux wants a yaml file @@ -97,58 +109,84 @@ def photon_fitting_scale(self, xgrid): lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, self.F2LO) lux.InsertInelasticSplitQ([4.18, 1e100]) - - photon_100GeV = np.zeros(len(xgrid)) - for i, x in enumerate(xgrid): - print("computing grid point", i+1, "/", len(xgrid)) - photon_100GeV[i] = lux.EvaluatePhoton(x, self.q_in2).total / x - # TODO: fiatlux returns gamma(x) or x*gamma(x) ? - - pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) - for j, pid in enumerate(output.rotations.inputpids): - if pid == 22 : - pdfs[j] = photon_100GeV - ph_id = j - if not self.qcd_pdfs.hasFlavor(pid): - continue - pdfs[j] = np.array( - [ - self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x - for x in xgrid - ] - ) - operator_card = dict( - sorted( - dict( - interpolation_xgrid=xgrid.tolist(), - interpolation_polynomial_degree=4, - interpolation_is_log=True, - ev_op_max_order=10, - ev_op_iterations=10, - backward_inversion="expanded", - n_integration_cores=0, - debug_skip_non_singlet=False, - debug_skip_singlet=False, - Q2grid=[self.q_fin**2], - inputgrid=None, - targetgrid=None, - inputpids=None, - targetpids=None, - ).items() + grid_count = 0 + photon_list = [] + for xgrid in xgrid_list : + photon_100GeV = np.zeros(len(xgrid)) + for i, x in enumerate(xgrid): + print("computing grid point", grid_count + i+1, "/", len(xgrids)) + photon_100GeV[i] = lux.EvaluatePhoton(x, self.q_in2).total / x + # TODO: fiatlux returns gamma(x) or x*gamma(x) ? + grid_count += len(xgrid) + + operator_card = dict( + sorted( + dict( + interpolation_xgrid=xgrid.tolist(), + interpolation_polynomial_degree=4, + interpolation_is_log=True, + ev_op_max_order=10, + ev_op_iterations=10, + backward_inversion="expanded", + n_integration_cores=0, + debug_skip_non_singlet=False, + debug_skip_singlet=False, + Q2grid=[self.q_fin**2], + inputgrid=None, + targetgrid=None, + inputpids=None, + targetpids=None, + ).items() + ) ) - ) - # TODO : this EKO should be precomputed and stored since it never changes - runner = Runner(self.theory, operator_card) - output = runner.get_output() - - for q2, elem in output.items(): - pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) - # error_final = np.einsum("ajbk,bk", elem.error, pdfs) + # TODO : this EKO should be precomputed and stored since it never changes + runner = Runner(self.theory, operator_card) + output = runner.get_output() + + # To be used when the new version of EKO (0.11.1) will be available on conda + # pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) + # for j, pid in enumerate(output.rotations.inputpids): + # if pid == 22 : + # pdfs[j] = photon_100GeV + # ph_id = j + # if not self.qcd_pdfs.hasFlavor(pid): + # continue + # pdfs[j] = np.array( + # [ + # self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x + # for x in xgrid + # ] + # ) + + pdfs = np.zeros((len(output["inputpids"]), len(output["inputgrid"]))) + for j, pid in enumerate(output["inputpids"]): + if pid == 22 : + pdfs[j] = photon_100GeV + ph_id = j + if not self.qcd_pdfs.hasFlavor(pid): + continue + pdfs[j] = np.array( + [ + self.qcd_pdfs.xfxQ2(pid, x, output["q2_ref"]) / x + for x in output["inputgrid"] + ] + ) + + # To be used when the new version of EKO (0.11.1) will be available on conda + # for q2, elem in output.items(): + # pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) + # # error_final = np.einsum("ajbk,bk", elem.error, pdfs) - photon_fitting_scale = pdf_final[ph_id] + for q2, elem in output["Q2grid"].items(): + pdf_final = np.einsum("ajbk,bk", elem["operators"], pdfs) + #error_final = np.einsum("ajbk,bk", elem["operator_errors"], pdfs) - # we want x * gamma(x) - return xgrid * photon_fitting_scale + photon_fitting_scale = pdf_final[ph_id] + + # we want x * gamma(x) + photon_list.append( xgrid * photon_fitting_scale ) + + return np.concatenate(photon_list) \ No newline at end of file From 77667e637be1a5b9d6dbb2838d79a01e182971a3 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Sat, 10 Dec 2022 15:18:54 +0100 Subject: [PATCH 023/204] Rename photon_pdf -> photon and compute_photon -> compute --- n3fit/src/n3fit/model_trainer.py | 2 +- .../compute_photon.py => photon/compute.py} | 49 +++++++++++++++++-- .../structure_functions.py | 0 .../tests/test_structure_functions.py | 45 +++++++++++++++++ 4 files changed, 92 insertions(+), 4 deletions(-) rename validphys2/src/validphys/{photon_pdf/compute_photon.py => photon/compute.py} (85%) rename validphys2/src/validphys/{photon_pdf => photon}/structure_functions.py (100%) create mode 100644 validphys2/src/validphys/tests/test_structure_functions.py diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 7b8f6375ec..7c3dbdcf70 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -20,7 +20,7 @@ from n3fit.vpinterface import N3PDF import n3fit.hyper_optimization.penalties import n3fit.hyper_optimization.rewards -from validphys.photon_pdf.compute_photon import Photon +from validphys.photon.compute import Photon log = logging.getLogger(__name__) diff --git a/validphys2/src/validphys/photon_pdf/compute_photon.py b/validphys2/src/validphys/photon/compute.py similarity index 85% rename from validphys2/src/validphys/photon_pdf/compute_photon.py rename to validphys2/src/validphys/photon/compute.py index 7be1cecf6a..8a9ea87c02 100644 --- a/validphys2/src/validphys/photon_pdf/compute_photon.py +++ b/validphys2/src/validphys/photon/compute.py @@ -1,7 +1,4 @@ """Script that calls fiatlux to add the photon PDF.""" - -# from validphys import lhapdfset -from validphys.api import API import lhapdf import fiatlux import numpy as np @@ -38,6 +35,24 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.path_to_FL = fiatlux_runcard["path_to_FL"] def exctract_grids(self, xgrids): + r""" + Extract the subgrids inside xgrids. + + xgrids is the concatenation of different grids, i.e. + xgrid = np.array([xmin1, ..., xmax1, xmin2, ...,xmax2, xmin3, ...]). + The different grids are extracted and stored in a list: + xgrid_list = [np.array([xgrid1]), np.array([xgrid2]), ...] + + Parameters + ---------- + xgrids : nd.array + concatenation of the subgrids + + Returns + ------- + xgrid_list : list + list containing the different grids + """ xgrid_list = [] imin = 0 for i in range(1, len(xgrids)): @@ -48,6 +63,21 @@ def exctract_grids(self, xgrids): return xgrid_list def F2LO(self, x, Q): + r""" + Compute the LO DIS structure function F2. + + Parameters + ---------- + x : float + Bjorken's variable + Q : float + DIS hard scale + + Returns + ------- + F2_LO : float + Structure function F2 at LO + """ mcharm = self.theory["mc"] mbottom = self.theory["mb"] mtop = self.theory["mt"] @@ -69,6 +99,19 @@ def F2LO(self, x, Q): return res def alpha_em(self, q): + r""" + Compute the value of alpha_em. + + Parameters + ---------- + q: float + value in which the coupling is computed + + Returns + ------- + alpha_em: float + electromagnetic coupling + """ return self.couplings.a(q**2)[1] * 4 * np.pi def photon_fitting_scale(self, xgrids): diff --git a/validphys2/src/validphys/photon_pdf/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py similarity index 100% rename from validphys2/src/validphys/photon_pdf/structure_functions.py rename to validphys2/src/validphys/photon/structure_functions.py diff --git a/validphys2/src/validphys/tests/test_structure_functions.py b/validphys2/src/validphys/tests/test_structure_functions.py new file mode 100644 index 0000000000..be9cdcc845 --- /dev/null +++ b/validphys2/src/validphys/tests/test_structure_functions.py @@ -0,0 +1,45 @@ +import apfel +from validphys.photon.structure_functions import StructureFunction +import lhapdf +import numpy as np +from pathlib import Path +from eko.interpolation import make_grid +import pineappl +from scipy.interpolate import interp2d, RectBivariateSpline + +pdf_name = "NNPDF40_nnlo_as_01180" +pdfs = lhapdf.mkPDF(pdf_name, 0) +pdgid = int(pdfs.set().get_entry("Particle")) +path_to_F2 = '/home/laurenti/n3pdf/pineline/data/fktables/200/DIS_F2.pineappl.lz4' +path_to_FL = '/home/laurenti/n3pdf/pineline/data/fktables/200/DIS_FL.pineappl.lz4' + +fktablef2=pineappl.fk_table.FkTable.read(Path(path_to_F2)) +fktablefl=pineappl.fk_table.FkTable.read(Path(path_to_FL)) + +def produce_interpolator1(fktable): + x = np.unique(fktable.bin_left(1)) + q2 = np.unique(fktable.bin_left(0)) + predictions = fktable.convolute_with_one(pdgid, pdfs.xfxQ2) + grid2D = predictions.reshape(len(x),len(q2)).T + interpolator = interp2d(x, q2, grid2D) + return interpolator + +def produce_interpolator2(fktable): + x = np.unique(fktable.bin_left(1)) + q2 = np.unique(fktable.bin_left(0)) + predictions = fktable.convolute_with_one(pdgid, pdfs.xfxQ2) + grid2D = predictions.reshape(len(x),len(q2)) + print(grid2D) + interpolator = RectBivariateSpline(x, q2, grid2D) + return interpolator + +inter1_f2 = produce_interpolator1(fktablef2) +inter2_f2 = produce_interpolator2(fktablef2) +inter1_fl = produce_interpolator1(fktablefl) +inter2_fl = produce_interpolator2(fktablefl) + +for x in make_grid(10,10): + for q in [5, 10, 20, 50, 100]: + print(x, q) + np.testing.assert_allclose(inter1_f2(x, q**2)[0], inter2_f2(x, q**2)[0,0], rtol=21e-1) + np.testing.assert_allclose(inter1_fl(x, q**2)[0], inter2_fl(x, q**2)[0,0], rtol=21e-1) \ No newline at end of file From d11f7eb5492da16c39093a53ed5dddfb605c508b Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Sun, 11 Dec 2022 14:54:44 +0100 Subject: [PATCH 024/204] Remove test_structure_functions --- .../tests/test_structure_functions.py | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 validphys2/src/validphys/tests/test_structure_functions.py diff --git a/validphys2/src/validphys/tests/test_structure_functions.py b/validphys2/src/validphys/tests/test_structure_functions.py deleted file mode 100644 index be9cdcc845..0000000000 --- a/validphys2/src/validphys/tests/test_structure_functions.py +++ /dev/null @@ -1,45 +0,0 @@ -import apfel -from validphys.photon.structure_functions import StructureFunction -import lhapdf -import numpy as np -from pathlib import Path -from eko.interpolation import make_grid -import pineappl -from scipy.interpolate import interp2d, RectBivariateSpline - -pdf_name = "NNPDF40_nnlo_as_01180" -pdfs = lhapdf.mkPDF(pdf_name, 0) -pdgid = int(pdfs.set().get_entry("Particle")) -path_to_F2 = '/home/laurenti/n3pdf/pineline/data/fktables/200/DIS_F2.pineappl.lz4' -path_to_FL = '/home/laurenti/n3pdf/pineline/data/fktables/200/DIS_FL.pineappl.lz4' - -fktablef2=pineappl.fk_table.FkTable.read(Path(path_to_F2)) -fktablefl=pineappl.fk_table.FkTable.read(Path(path_to_FL)) - -def produce_interpolator1(fktable): - x = np.unique(fktable.bin_left(1)) - q2 = np.unique(fktable.bin_left(0)) - predictions = fktable.convolute_with_one(pdgid, pdfs.xfxQ2) - grid2D = predictions.reshape(len(x),len(q2)).T - interpolator = interp2d(x, q2, grid2D) - return interpolator - -def produce_interpolator2(fktable): - x = np.unique(fktable.bin_left(1)) - q2 = np.unique(fktable.bin_left(0)) - predictions = fktable.convolute_with_one(pdgid, pdfs.xfxQ2) - grid2D = predictions.reshape(len(x),len(q2)) - print(grid2D) - interpolator = RectBivariateSpline(x, q2, grid2D) - return interpolator - -inter1_f2 = produce_interpolator1(fktablef2) -inter2_f2 = produce_interpolator2(fktablef2) -inter1_fl = produce_interpolator1(fktablefl) -inter2_fl = produce_interpolator2(fktablefl) - -for x in make_grid(10,10): - for q in [5, 10, 20, 50, 100]: - print(x, q) - np.testing.assert_allclose(inter1_f2(x, q**2)[0], inter2_f2(x, q**2)[0,0], rtol=21e-1) - np.testing.assert_allclose(inter1_fl(x, q**2)[0], inter2_fl(x, q**2)[0,0], rtol=21e-1) \ No newline at end of file From 373f96d29e7675946f34da06c897fa62c1a968b6 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 12 Dec 2022 09:57:19 +0100 Subject: [PATCH 025/204] Remove temporarily replica_id from performfit --- n3fit/src/n3fit/model_gen.py | 1 + n3fit/src/n3fit/performfit.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index e4121702c8..ff98085a30 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -560,6 +560,7 @@ def pdfNN_layer_generator( # Evolution layer layer_evln = FkRotation(input_shape=(last_layer_nodes,), output_dim=out) + # Photon layer layer_photon = AddPhoton(pdf_ph=pdf_ph, input_shape=(out,), output_dim=out) # Basis rotation diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 7f84adf443..01d9459b83 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -197,7 +197,8 @@ def performfit( parallel_models=n_models, theoryid=theoryid, fiatlux_runcard=fiatlux, - replica_id=replica_idxs[0] + # replica_id=replica_idxs[0], + replica_id=None, ) # This is just to give a descriptive name to the fit function From 615405f67ce53f32d51686abc7c538de3c02aff4 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Mon, 12 Dec 2022 10:45:56 +0100 Subject: [PATCH 026/204] add a photon placeholder to n3fit --- n3fit/src/n3fit/layers/msr_normalization.py | 5 ++-- n3fit/src/n3fit/layers/rotations.py | 28 ++++++++++++------- n3fit/src/n3fit/model_gen.py | 22 ++++++++++----- n3fit/src/n3fit/model_trainer.py | 30 ++++++++++++++++----- n3fit/src/n3fit/msr.py | 9 +++++-- n3fit/src/n3fit/vpinterface.py | 17 ++++++++++++ 6 files changed, 86 insertions(+), 25 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index c31bf8fb41..1e3b2b5bf0 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -16,7 +16,8 @@ class MSR_Normalization(MetaLayer): _msr_enabled = False _vsr_enabled = False - def __init__(self, output_dim=14, mode="ALL", **kwargs): + def __init__(self, output_dim=14, mode="ALL", photon_contribution=0.0, **kwargs): + self._photon = photon_contribution if mode == True or mode.upper() == "ALL": self._msr_enabled = True self._vsr_enabled = True @@ -53,7 +54,7 @@ def call(self, pdf_integrated): norm_constants = [] if self._msr_enabled: - n_ag = [(1.0 - y[GLUON_IDX[0][0]-1]) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) + n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] + self._photon) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 50e5ba8f9a..48bdced56c 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -93,21 +93,31 @@ def call(self, pdf_raw): class AddPhoton(MetaLayer): """ - Changes the value of the photon to non-zero. + Changes the value of the photon component of the PDF to non-zero. + The photon idx of the PDF is always index 0. - Both input and output have shape (1, None, 14). However, - the input has a photon which is set to zero, while for the - output it is set to its real value. + In order to avoid bottlenecks, this layer can only compute the photon + for a given fixed shape. + In order to change the shape it is necessary to rebuild the photon. """ - def __init__(self, pdf_ph, output_dim, **kwargs): - # pdf_ph must be a tensor of shape (1, xgrid, 1) - self.pdf_ph = pdf_ph - self.output_dim = output_dim + def __init__(self, photon, **kwargs): + self._photon_generator = photon + self._pdf_ph = None super().__init__(**kwargs) + + def register_photon(self, xgrid): + """Compute the photon array and set the layer to be rebuilt""" + # TODO: maybe add here some caching mechanism so that the photon doesn't get + # recomputed if the grid hasn't changed! + if self._photon_generator is not None: + self._pdf_ph = self._photon_generator(xgrid) + self.built = False def call(self, pdfs): - return op.concatenate([self.pdf_ph, pdfs[:,:,1:]], axis=-1) + if self._pdf_ph is None: + return pdfs + return op.concatenate([self._pdf_ph, pdfs[:,:,1:]], axis=-1) class ObsRotation(MetaLayer): diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index ff98085a30..96176b7b2a 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -403,7 +403,7 @@ def pdfNN_layer_generator( impose_sumrule=None, scaler=None, parallel_models=1, - pdf_ph=None, + photon_computer=None, ): # pylint: disable=too-many-locals """ Generates the PDF model which takes as input a point in x (from 0 to 1) @@ -496,6 +496,10 @@ def pdfNN_layer_generator( will be a (1, None, 2) tensor where dim [:,:,0] is scaled parallel_models: int How many models should be trained in parallel + photon_computer: Function + If given, gives the `add_photon` a function to compute a photon which will be added at the + index 0 of the 14-size FK basis + This same function will also be used to compute the MSR component for the photon Returns ------- @@ -560,15 +564,16 @@ def pdfNN_layer_generator( # Evolution layer layer_evln = FkRotation(input_shape=(last_layer_nodes,), output_dim=out) + # Photon layer - layer_photon = AddPhoton(pdf_ph=pdf_ph, input_shape=(out,), output_dim=out) + layer_photon = AddPhoton(photon=photon_computer) # Basis rotation basis_rotation = FlavourToEvolution(flav_info=flav_info, fitbasis=fitbasis) # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler) + sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photon=photon_computer) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x @@ -642,12 +647,17 @@ def layer_fitbasis(x): def layer_pdf(x): return layer_evln(layer_fitbasis(x)) + # Final PDF (apply normalization) + normalized_pdf = sumrule_layer(layer_pdf) + # Photon layer, changes the photon from zero to non-zero def apply_photon(x): - return layer_photon(layer_pdf(x)) + return layer_photon(normalized_pdf(x)) - # Final PDF (apply normalization) - final_pdf = sumrule_layer(apply_photon) + if photon_computer is None: + final_pdf = normalized_pdf + else: + final_pdf = apply_photon # Create the model pdf_model = MetaModel( diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 7c3dbdcf70..3f1290d924 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -640,7 +640,7 @@ def _generate_pdf( regularizer, regularizer_args, seed, - pdf_ph, + photon_computer, ): """ Defines the internal variable layer_pdf @@ -667,6 +667,8 @@ def _generate_pdf( dictionary of arguments for the regularizer seed: int seed for the NN + photon_computer: function + function to compute the photon PDF see model_gen.pdfNN_layer_generator for more information Returns @@ -692,7 +694,7 @@ def _generate_pdf( impose_sumrule=self.impose_sumrule, scaler=self._scaler, parallel_models=self._parallel_models, - pdf_ph=pdf_ph, + photon_computer=photon_computer, ) return pdf_models @@ -865,9 +867,15 @@ def hyperparametrizable(self, params): xinput = self._xgrid_generation() # compute photon: - photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) - photon_array = photon.photon_fitting_scale(xinput.input_l.tensor_content[0,:,0]) - ph_pdf = op.batchit(op.numpy_to_tensor(photon_array[:, np.newaxis])) +# photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) +# photon_array = photon.photon_fitting_scale(xinput.input_l.tensor_content[0,:,0]) +# ph_pdf = op.batchit(op.numpy_to_tensor(photon_array[:, np.newaxis])) + + def photon_computer(x): + """Receives a grid with shape (1, n_x, 1) + and returns the photon PDF (1, n_x, 1) + """ + return np.zeros_like(x) ### Training loop for k, partition in enumerate(self.kpartitions): @@ -887,9 +895,19 @@ def hyperparametrizable(self, params): params.get("regularizer", None), # regularizer optional params.get("regularizer_args", None), seeds, - ph_pdf, + photon_computer, ) + # Register the fitting grid with the photon layer + try: + for m in pdf_models: + pl = m.get_layer("add_photon") + pl.register_photon(xinput.input_l.tensor_content) + except ValueError: + # There's no photon I guess + pass + + # Model generation joins all the different observable layers # together with pdf model generated above models = self._model_generation(xinput, pdf_models, partition, k) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index e99006de47..92bb801bfe 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -36,7 +36,7 @@ def gen_integration_input(nx): return xgrid, weights_array -def msr_impose(nx=int(2e3), mode='All', scaler=None): +def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): """ This function receives: Generates a function that applies a normalization layer to the fit. @@ -70,8 +70,13 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None): # 3. Now create the integration layer (the layer that will simply integrate, given some weight integrator = xIntegrator(weights_array, input_shape=(nx,)) + # 3.1 If a photon is given, compute the photon component of the MSR + photon_c = 0.0 + if photon is not None: + photon_c = np.sum(photon(np.linspace(0,1,100)))/100 + # 4. Now create the normalization by selecting the right integrations - normalizer = MSR_Normalization(mode=mode) + normalizer = MSR_Normalization(mode=mode, photon_contribution=photon_c) # 5. Make the xgrid array into a backend input layer so it can be given to the normalization xgrid_input = op.numpy_to_input(xgrid, name="integration_grid") diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 777cac91ac..c7ba392852 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -78,6 +78,20 @@ def xfxQ(self, x, Q, n, fl): ) return self.grid_values([fl], [x]).squeeze()[n] + def _register_photon(self, xgrid): + """If the PDF models contain photons, register the xgrid with them""" + try: + for m in self._lhapdf_set: + pl = m.get_layer("add_photon") + pl.register_photon(xgrid) + # Recompile the model if necessary + # (relies on the caching mechanism inside the add_photon layer) + if not pl.built: + m.compile() + except ValueError: + # There's no photon I guess + pass + def __call__(self, xarr, flavours=None, replica=None): """Uses the internal model to produce pdf values for the grid The output is on the evolution basis. @@ -103,6 +117,9 @@ def __call__(self, xarr, flavours=None, replica=None): # as the scaling is done by the model itself mod_xgrid = xarr.reshape(1, -1, 1) + # Try registering the grid with the photon + self._register_photon(mod_xgrid) + if replica is None or replica == 0: # We need generate output values for all replicas result = np.concatenate([m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0) From c3531c9f4c97dfdabab3d25f52b95c89ca53a311 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 12 Dec 2022 11:16:04 +0100 Subject: [PATCH 027/204] Change xinput.input_l -> xinput.input --- n3fit/src/n3fit/model_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 0185f8ce89..af3c7122be 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -879,7 +879,7 @@ def hyperparametrizable(self, params): # compute photon: photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) - photon_array = photon.photon_fitting_scale(xinput.input_l.tensor_content[0,:,0]) + photon_array = photon.photon_fitting_scale(xinput.input.tensor_content[0,:,0]) ph_pdf = op.batchit(op.numpy_to_tensor(photon_array[:, np.newaxis])) ### Training loop From c18b925accd1c57cd7b821c962374d5d12361ca4 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 12 Dec 2022 23:54:57 +0100 Subject: [PATCH 028/204] Implement photon_computer in n3fit --- n3fit/src/n3fit/layers/msr_normalization.py | 2 +- n3fit/src/n3fit/model_trainer.py | 12 +++++------- n3fit/src/n3fit/msr.py | 6 +++++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 1e3b2b5bf0..4a5ac4531d 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -54,7 +54,7 @@ def call(self, pdf_integrated): norm_constants = [] if self._msr_enabled: - n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] + self._photon) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) + n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - self._photon) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 01b92a3dcf..66d5fd78d7 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -879,16 +879,14 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() - # compute photon: -# photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) -# photon_array = photon.photon_fitting_scale(xinput.input_l.tensor_content[0,:,0]) -# ph_pdf = op.batchit(op.numpy_to_tensor(photon_array[:, np.newaxis])) + # Initialize photon class: + photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) def photon_computer(x): """Receives a grid with shape (1, n_x, 1) and returns the photon PDF (1, n_x, 1) """ - return np.zeros_like(x) + return photon.photon_fitting_scale(x[0,:,0])[np.newaxis,:,np.newaxis] ### Training loop for k, partition in enumerate(self.kpartitions): @@ -908,14 +906,14 @@ def photon_computer(x): params.get("regularizer", None), # regularizer optional params.get("regularizer_args", None), seeds, - photon_computer, + photon_computer if self.fiatlux_runcard is not None else None, ) # Register the fitting grid with the photon layer try: for m in pdf_models: pl = m.get_layer("add_photon") - pl.register_photon(xinput.input_l.tensor_content) + pl.register_photon(xinput.input.tensor_content) except ValueError: # There's no photon I guess pass diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 92bb801bfe..327d94b609 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -73,7 +73,11 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): # 3.1 If a photon is given, compute the photon component of the MSR photon_c = 0.0 if photon is not None: - photon_c = np.sum(photon(np.linspace(0,1,100)))/100 + photon_c = np.sum( + photon( + np.linspace(1e-5,1,100)[np.newaxis,:,np.newaxis] + ) + )/100 # 4. Now create the normalization by selecting the right integrations normalizer = MSR_Normalization(mode=mode, photon_contribution=photon_c) From f84ac3f2b099ae889e9dfad968746d65c0a8b7c3 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 13 Dec 2022 22:03:20 +0100 Subject: [PATCH 029/204] Move lux initialization functions in __init__ --- validphys2/src/validphys/photon/compute.py | 93 +++++++++------------- 1 file changed, 37 insertions(+), 56 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 8a9ea87c02..0c453d2583 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -8,6 +8,7 @@ from .structure_functions import StructureFunction import yaml +from os import remove class Photon: def __init__(self, theoryid, fiatlux_runcard, replica=0): @@ -15,24 +16,44 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: - # the commented objects should be passed from theory runcard, - # however EKO complains that he doesn't find them - self.theory["nfref"] = None + self.theory["nfref"] = 5 self.theory["nf0"] = None self.theory["fact_to_ren_scale_ratio"] = 1. self.theory["ModSV"] = None - self.theory["IC"] = 1 self.theory["IB"] = 0 self.theory["FNS"] = "VFNS" + self.theory["QED"] = 2 + self.theory["ModEv"] = "EXA" + self.theory["alphaem_running"] = True self.q_in = 100 self.q_in2 = self.q_in ** 2 self.q_fin = self.theory["Q0"] self.theory["Q0"]= self.q_in self.qref = self.theory["Qref"] + + theory_coupling = update_theory(self.theory) + theory_coupling["ModEv"] = "TRN" + self.couplings = Couplings.from_dict(theory_coupling) + self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) - self.couplings = Couplings.from_dict(update_theory(self.theory)) - self.path_to_F2 = fiatlux_runcard["path_to_F2"] - self.path_to_FL = fiatlux_runcard["path_to_FL"] + path_to_F2 = fiatlux_runcard["path_to_F2"] + path_to_FL = fiatlux_runcard["path_to_FL"] + f2 = StructureFunction(path_to_F2, self.qcd_pdfs) + fl = StructureFunction(path_to_FL, self.qcd_pdfs) + + # lux = fiatlux.FiatLux(fiatlux_runcard) + # we have a dict but fiatlux wants a yaml file + # TODO : remove this dirty trick + ff = open('fiatlux_runcard.yml', 'w+') + yaml.dump(self.fiatlux_runcard, ff) + self.lux = fiatlux.FiatLux('fiatlux_runcard.yml') + remove('fiatlux_runcard.yml') + self.lux.PlugAlphaQED(self.alpha_em, self.qref) + self.lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, self.F2LO) + self.lux.InsertInelasticSplitQ([4.18, 1e100]) + + self.cache = {} + def exctract_grids(self, xgrids): r""" @@ -131,36 +152,16 @@ def photon_fitting_scale(self, xgrids): ------- photon_fitting_scale: numpy.array photon PDF at the scale 1 GeV - """ - if self.fiatlux_runcard is None : - return np.zeros(len(xgrids)) - + """ xgrid_list = self.exctract_grids(xgrids) - # lux = fiatlux.FiatLux(fiatlux_runcard) - # we have a dict but fiatlux wants a yaml file - # TODO : remove this trick - ff = open('fiatlux_runcard.yml', 'w+') - yaml.dump(self.fiatlux_runcard, ff) - - lux = fiatlux.FiatLux('fiatlux_runcard.yml') - lux.PlugAlphaQED(self.alpha_em, self.qref) - - f2 = StructureFunction(self.path_to_F2, self.qcd_pdfs) - fl = StructureFunction(self.path_to_FL, self.qcd_pdfs) - - lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, self.F2LO) - - lux.InsertInelasticSplitQ([4.18, 1e100]) - grid_count = 0 photon_list = [] for xgrid in xgrid_list : photon_100GeV = np.zeros(len(xgrid)) for i, x in enumerate(xgrid): print("computing grid point", grid_count + i+1, "/", len(xgrids)) - photon_100GeV[i] = lux.EvaluatePhoton(x, self.q_in2).total / x - # TODO: fiatlux returns gamma(x) or x*gamma(x) ? + photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x grid_count += len(xgrid) operator_card = dict( @@ -188,23 +189,8 @@ def photon_fitting_scale(self, xgrids): runner = Runner(self.theory, operator_card) output = runner.get_output() - # To be used when the new version of EKO (0.11.1) will be available on conda - # pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) - # for j, pid in enumerate(output.rotations.inputpids): - # if pid == 22 : - # pdfs[j] = photon_100GeV - # ph_id = j - # if not self.qcd_pdfs.hasFlavor(pid): - # continue - # pdfs[j] = np.array( - # [ - # self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x - # for x in xgrid - # ] - # ) - - pdfs = np.zeros((len(output["inputpids"]), len(output["inputgrid"]))) - for j, pid in enumerate(output["inputpids"]): + pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) + for j, pid in enumerate(output.rotations.inputpids): if pid == 22 : pdfs[j] = photon_100GeV ph_id = j @@ -212,19 +198,14 @@ def photon_fitting_scale(self, xgrids): continue pdfs[j] = np.array( [ - self.qcd_pdfs.xfxQ2(pid, x, output["q2_ref"]) / x - for x in output["inputgrid"] + self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x + for x in xgrid ] ) - # To be used when the new version of EKO (0.11.1) will be available on conda - # for q2, elem in output.items(): - # pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) - # # error_final = np.einsum("ajbk,bk", elem.error, pdfs) - - for q2, elem in output["Q2grid"].items(): - pdf_final = np.einsum("ajbk,bk", elem["operators"], pdfs) - #error_final = np.einsum("ajbk,bk", elem["operator_errors"], pdfs) + for q2, elem in output.items(): + pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) + # error_final = np.einsum("ajbk,bk", elem.error, pdfs) photon_fitting_scale = pdf_final[ph_id] From 73972faff88491a903865a27163b2a0cf6305e4f Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 14 Dec 2022 12:39:37 +0100 Subject: [PATCH 030/204] Add caching mechanism in class Photon --- validphys2/src/validphys/photon/compute.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 0c453d2583..0d1cd338ef 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -13,6 +13,7 @@ class Photon: def __init__(self, theoryid, fiatlux_runcard, replica=0): # TODO : for the moment we do the 0-th replica then we change it + print("+++++++++++++++++SDIGHEDE SDAGHEDE+++++++++++++++++") self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: @@ -141,12 +142,8 @@ def photon_fitting_scale(self, xgrids): Parameters ---------- - xgrid: numpy.array + xgrids: numpy.array grid of the x points - pdf_name: string - name of the QCD set - replica: int - number of replica Returns ------- @@ -160,8 +157,15 @@ def photon_fitting_scale(self, xgrids): for xgrid in xgrid_list : photon_100GeV = np.zeros(len(xgrid)) for i, x in enumerate(xgrid): - print("computing grid point", grid_count + i+1, "/", len(xgrids)) - photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x + for x_cached in self.cache : + if np.isclose(x, x_cached): + print("using cache for grid point", grid_count + i+1, "/", len(xgrids)) + photon_100GeV[i] = self.cache[x_cached] + break + else : + print("computing grid point", grid_count + i+1, "/", len(xgrids)) + photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x + self.cache[x] = photon_100GeV[i] grid_count += len(xgrid) operator_card = dict( @@ -211,6 +215,6 @@ def photon_fitting_scale(self, xgrids): # we want x * gamma(x) photon_list.append( xgrid * photon_fitting_scale ) - + import ipdb; ipdb.set_trace() return np.concatenate(photon_list) \ No newline at end of file From 182fcea742a509a79b21aed79dd57183bf843845 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 14 Dec 2022 12:40:30 +0100 Subject: [PATCH 031/204] Use lambert grid for integrating the photon msr --- n3fit/src/n3fit/msr.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 327d94b609..d2c1b1253a 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -3,6 +3,8 @@ """ import logging import numpy as np +from scipy.special import lambertw +from scipy.integrate import trapezoid from n3fit.layers import xDivide, MSR_Normalization, xIntegrator from n3fit.backends import operations as op @@ -73,11 +75,19 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): # 3.1 If a photon is given, compute the photon component of the MSR photon_c = 0.0 if photon is not None: - photon_c = np.sum( - photon( - np.linspace(1e-5,1,100)[np.newaxis,:,np.newaxis] - ) - )/100 + + def make_lambert_grid(n_pts, x_min, x_max): + def direct_relation(x): + return 5 * (1 - x) - np.log(x) + def inverse_relation(y): + return np.real(1 / 5 * lambertw(5 * np.exp(5 - y))) + y_min = direct_relation(x_min) + y_max = direct_relation(x_max) + return np.array([inverse_relation(y) for y in np.linspace(y_min, y_max, n_pts)]) + + xgrid_ph = make_lambert_grid(1e-6, 1, 100) + photon_array = photon(xgrid_ph[np.newaxis,:,np.newaxis]) + photon_c = trapezoid(photon_array, xgrid_ph, axis = 1) # 4. Now create the normalization by selecting the right integrations normalizer = MSR_Normalization(mode=mode, photon_contribution=photon_c) From abe5e51cbda408ddb7bb4bf1b9d3c8c8a08b202e Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 14 Dec 2022 12:41:15 +0100 Subject: [PATCH 032/204] Also add trapezoidal rule to photon msr --- n3fit/src/n3fit/msr.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index d2c1b1253a..32a691fb6b 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -75,7 +75,7 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): # 3.1 If a photon is given, compute the photon component of the MSR photon_c = 0.0 if photon is not None: - + def make_lambert_grid(n_pts, x_min, x_max): def direct_relation(x): return 5 * (1 - x) - np.log(x) @@ -85,9 +85,10 @@ def inverse_relation(y): y_max = direct_relation(x_max) return np.array([inverse_relation(y) for y in np.linspace(y_min, y_max, n_pts)]) - xgrid_ph = make_lambert_grid(1e-6, 1, 100) - photon_array = photon(xgrid_ph[np.newaxis,:,np.newaxis]) - photon_c = trapezoid(photon_array, xgrid_ph, axis = 1) + xgrid_ph = make_lambert_grid(100, 1e-6, 1) + photon_array = photon(xgrid_ph[np.newaxis,:,np.newaxis])[0,:,0] + # numerically integrate with the trapeizodal rule + photon_c = trapezoid(photon_array, xgrid_ph) # 4. Now create the normalization by selecting the right integrations normalizer = MSR_Normalization(mode=mode, photon_contribution=photon_c) From 3db6e63671464323dc6db3758487a80d4b8e3a0c Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 14 Dec 2022 13:00:59 +0100 Subject: [PATCH 033/204] Remove ipdb --- validphys2/src/validphys/photon/compute.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 0d1cd338ef..4d5dd4723c 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -13,7 +13,6 @@ class Photon: def __init__(self, theoryid, fiatlux_runcard, replica=0): # TODO : for the moment we do the 0-th replica then we change it - print("+++++++++++++++++SDIGHEDE SDAGHEDE+++++++++++++++++") self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: @@ -215,6 +214,6 @@ def photon_fitting_scale(self, xgrids): # we want x * gamma(x) photon_list.append( xgrid * photon_fitting_scale ) - import ipdb; ipdb.set_trace() + return np.concatenate(photon_list) \ No newline at end of file From fe6c134a4e882157fc9c128fe807dbe75a3867e7 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 14 Dec 2022 22:58:38 +0100 Subject: [PATCH 034/204] Load precomputed eko instead of computing it --- validphys2/src/validphys/photon/compute.py | 68 ++++++---------------- 1 file changed, 19 insertions(+), 49 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 4d5dd4723c..0ef885746b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -4,7 +4,9 @@ import numpy as np from eko.couplings import Couplings from eko.compatibility import update_theory -from eko.runner import Runner +from eko.output.legacy import load_tar +from eko.interpolation import XGrid +from eko.output.manipulate import xgrid_reshape from .structure_functions import StructureFunction import yaml @@ -16,23 +18,13 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: - self.theory["nfref"] = 5 - self.theory["nf0"] = None - self.theory["fact_to_ren_scale_ratio"] = 1. - self.theory["ModSV"] = None - self.theory["IB"] = 0 - self.theory["FNS"] = "VFNS" - self.theory["QED"] = 2 - self.theory["ModEv"] = "EXA" - self.theory["alphaem_running"] = True - self.q_in = 100 - self.q_in2 = self.q_in ** 2 - self.q_fin = self.theory["Q0"] - self.theory["Q0"]= self.q_in self.qref = self.theory["Qref"] - + self.q_in2 = 100**2 theory_coupling = update_theory(self.theory) theory_coupling["ModEv"] = "TRN" + theory_coupling["alphaem_running"] = True + theory_coupling["nfref"] = 5 + theory_coupling['fact_to_ren_scale_ratio'] = 1. self.couplings = Couplings.from_dict(theory_coupling) self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) @@ -99,15 +91,15 @@ def F2LO(self, x, Q): F2_LO : float Structure function F2 at LO """ - mcharm = self.theory["mc"] - mbottom = self.theory["mb"] - mtop = self.theory["mt"] + thres_charm = self.theory["Qmc"] + thres_bottom = self.theory["Qmb"] + thres_top = self.theory["Qmt"] # at LO we use ZM-VFS - if Q < mcharm : + if Q < thres_charm : hq = 3 - elif Q < mbottom : + elif Q < thres_bottom : hq = 4 - elif Q < mtop : + elif Q < thres_top : hq = 5 else : hq = 6 @@ -148,7 +140,7 @@ def photon_fitting_scale(self, xgrids): ------- photon_fitting_scale: numpy.array photon PDF at the scale 1 GeV - """ + """ xgrid_list = self.exctract_grids(xgrids) grid_count = 0 @@ -167,33 +159,11 @@ def photon_fitting_scale(self, xgrids): self.cache[x] = photon_100GeV[i] grid_count += len(xgrid) - operator_card = dict( - sorted( - dict( - interpolation_xgrid=xgrid.tolist(), - interpolation_polynomial_degree=4, - interpolation_is_log=True, - ev_op_max_order=10, - ev_op_iterations=10, - backward_inversion="expanded", - n_integration_cores=0, - debug_skip_non_singlet=False, - debug_skip_singlet=False, - Q2grid=[self.q_fin**2], - inputgrid=None, - targetgrid=None, - inputpids=None, - targetpids=None, - ).items() - ) - ) - - # TODO : this EKO should be precomputed and stored since it never changes - runner = Runner(self.theory, operator_card) - output = runner.get_output() + eko=load_tar(self.fiatlux_runcard['path_to_eko']) + xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) - pdfs = np.zeros((len(output.rotations.inputpids), len(xgrid))) - for j, pid in enumerate(output.rotations.inputpids): + pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) + for j, pid in enumerate(eko.rotations.inputpids): if pid == 22 : pdfs[j] = photon_100GeV ph_id = j @@ -206,7 +176,7 @@ def photon_fitting_scale(self, xgrids): ] ) - for q2, elem in output.items(): + for q2, elem in eko.items(): pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) # error_final = np.einsum("ajbk,bk", elem.error, pdfs) From a3af93ee5e92a9da6000d795e7c32fb145516647 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 14 Dec 2022 23:19:41 +0100 Subject: [PATCH 035/204] Use gen_integration_input in msr for photon --- n3fit/src/n3fit/msr.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 32a691fb6b..5a73d97827 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -3,7 +3,6 @@ """ import logging import numpy as np -from scipy.special import lambertw from scipy.integrate import trapezoid from n3fit.layers import xDivide, MSR_Normalization, xIntegrator @@ -75,20 +74,9 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): # 3.1 If a photon is given, compute the photon component of the MSR photon_c = 0.0 if photon is not None: - - def make_lambert_grid(n_pts, x_min, x_max): - def direct_relation(x): - return 5 * (1 - x) - np.log(x) - def inverse_relation(y): - return np.real(1 / 5 * lambertw(5 * np.exp(5 - y))) - y_min = direct_relation(x_min) - y_max = direct_relation(x_max) - return np.array([inverse_relation(y) for y in np.linspace(y_min, y_max, n_pts)]) - - xgrid_ph = make_lambert_grid(100, 1e-6, 1) - photon_array = photon(xgrid_ph[np.newaxis,:,np.newaxis])[0,:,0] - # numerically integrate with the trapeizodal rule - photon_c = trapezoid(photon_array, xgrid_ph) + xgrid_ph, wgt = gen_integration_input(100) + photon_array = photon(np.expand_dims(xgrid_ph, axis=0))[0] + photon_c = np.sum(photon_array * wgt) # 4. Now create the normalization by selecting the right integrations normalizer = MSR_Normalization(mode=mode, photon_contribution=photon_c) From 9253857435ee6a709afd939d09a99233d7c148c5 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 15 Dec 2022 11:47:31 +0100 Subject: [PATCH 036/204] Add function compute_eko in photon class --- n3fit/src/n3fit/msr.py | 7 ++- validphys2/src/validphys/photon/compute.py | 54 ++++++++++++++++++++-- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 5a73d97827..23e019bd8f 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -3,7 +3,6 @@ """ import logging import numpy as np -from scipy.integrate import trapezoid from n3fit.layers import xDivide, MSR_Normalization, xIntegrator from n3fit.backends import operations as op @@ -12,7 +11,7 @@ log = logging.getLogger(__name__) -def gen_integration_input(nx): +def gen_integration_input(nx, min=-9): """ Generates a np.array (shaped (nx,1)) of nx elements where the nx/2 first elements are a logspace between 0 and 0.1 @@ -20,7 +19,7 @@ def gen_integration_input(nx): """ lognx = int(nx / 2) linnx = int(nx - lognx) - xgrid_log = np.logspace(-9, -1, lognx + 1) + xgrid_log = np.logspace(min, -1, lognx + 1) xgrid_lin = np.linspace(0.1, 1, linnx) xgrid = np.concatenate([xgrid_log[:-1], xgrid_lin]).reshape(nx, 1) @@ -74,7 +73,7 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): # 3.1 If a photon is given, compute the photon component of the MSR photon_c = 0.0 if photon is not None: - xgrid_ph, wgt = gen_integration_input(100) + xgrid_ph, wgt = gen_integration_input(100, min=-6) photon_array = photon(np.expand_dims(xgrid_ph, axis=0))[0] photon_c = np.sum(photon_array * wgt) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 0ef885746b..97cf91c389 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -7,6 +7,7 @@ from eko.output.legacy import load_tar from eko.interpolation import XGrid from eko.output.manipulate import xgrid_reshape +from eko.runner import Runner from .structure_functions import StructureFunction import yaml @@ -149,7 +150,7 @@ def photon_fitting_scale(self, xgrids): photon_100GeV = np.zeros(len(xgrid)) for i, x in enumerate(xgrid): for x_cached in self.cache : - if np.isclose(x, x_cached): + if np.isclose(x, x_cached, atol=0, rtol=1e-3): print("using cache for grid point", grid_count + i+1, "/", len(xgrids)) photon_100GeV[i] = self.cache[x_cached] break @@ -159,8 +160,13 @@ def photon_fitting_scale(self, xgrids): self.cache[x] = photon_100GeV[i] grid_count += len(xgrid) - eko=load_tar(self.fiatlux_runcard['path_to_eko']) - xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) + # if the grid has less than 5 points we cannot interpolate the precomputed grid + # since it has interpolation_polynomial_degree = 4. Therefore we compute the EKO. + if len(xgrid) > 4: + eko=load_tar(self.fiatlux_runcard['path_to_eko']) + xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) + else : + eko = self.compute_eko(xgrid) pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) for j, pid in enumerate(eko.rotations.inputpids): @@ -186,4 +192,44 @@ def photon_fitting_scale(self, xgrids): photon_list.append( xgrid * photon_fitting_scale ) return np.concatenate(photon_list) - \ No newline at end of file + + def compute_eko(self, xgrid): + + theory = self.theory.copy() + theory["nfref"] = 5 + theory["nf0"] = None + theory["fact_to_ren_scale_ratio"] = 1. + theory["ModSV"] = None + theory["IB"] = 0 + theory["FNS"] = "VFNS" + theory["QED"] = 2 + theory["ModEv"] = "EXA" + theory["alphaem_running"] = True + q_in = 100 + q_fin = self.theory["Q0"] + theory["Q0"]= q_in + + operator_card = dict( + sorted( + dict( + interpolation_xgrid=xgrid.tolist(), + interpolation_polynomial_degree= 4 if len(xgrid) > 4 else len(xgrid) - 1, + interpolation_is_log=True, + ev_op_max_order=10, + ev_op_iterations=10, + backward_inversion="exact", + n_integration_cores=0, + debug_skip_non_singlet=False, + debug_skip_singlet=False, + Q2grid=[q_fin**2], + inputgrid=None, + targetgrid=None, + inputpids=None, + targetpids=None, + ).items() + ) + ) + + runner = Runner(theory, operator_card) + return runner.get_output() + \ No newline at end of file From 7cb116d6ab357b708af40897ff8d3254d75d35b0 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 15 Dec 2022 23:14:20 +0100 Subject: [PATCH 037/204] Remove photon_computer and pass Photon to model gen --- n3fit/src/n3fit/layers/rotations.py | 2 +- n3fit/src/n3fit/model_gen.py | 10 +++++----- n3fit/src/n3fit/model_trainer.py | 19 ++++++++----------- n3fit/src/n3fit/msr.py | 8 +++----- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 48bdced56c..cc76c59010 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -111,7 +111,7 @@ def register_photon(self, xgrid): # TODO: maybe add here some caching mechanism so that the photon doesn't get # recomputed if the grid hasn't changed! if self._photon_generator is not None: - self._pdf_ph = self._photon_generator(xgrid) + self._pdf_ph = self._photon_generator.compute(xgrid) self.built = False def call(self, pdfs): diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 96176b7b2a..4cd85dfacf 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -403,7 +403,7 @@ def pdfNN_layer_generator( impose_sumrule=None, scaler=None, parallel_models=1, - photon_computer=None, + photon=None, ): # pylint: disable=too-many-locals """ Generates the PDF model which takes as input a point in x (from 0 to 1) @@ -496,7 +496,7 @@ def pdfNN_layer_generator( will be a (1, None, 2) tensor where dim [:,:,0] is scaled parallel_models: int How many models should be trained in parallel - photon_computer: Function + photon: Function If given, gives the `add_photon` a function to compute a photon which will be added at the index 0 of the 14-size FK basis This same function will also be used to compute the MSR component for the photon @@ -566,14 +566,14 @@ def pdfNN_layer_generator( layer_evln = FkRotation(input_shape=(last_layer_nodes,), output_dim=out) # Photon layer - layer_photon = AddPhoton(photon=photon_computer) + layer_photon = AddPhoton(photon=photon) # Basis rotation basis_rotation = FlavourToEvolution(flav_info=flav_info, fitbasis=fitbasis) # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photon=photon_computer) + sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photon=photon) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x @@ -654,7 +654,7 @@ def layer_pdf(x): def apply_photon(x): return layer_photon(normalized_pdf(x)) - if photon_computer is None: + if photon is None: final_pdf = normalized_pdf else: final_pdf = apply_photon diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 66d5fd78d7..379c234788 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -653,7 +653,7 @@ def _generate_pdf( regularizer, regularizer_args, seed, - photon_computer, + photon, ): """ Defines the internal variable layer_pdf @@ -680,7 +680,7 @@ def _generate_pdf( dictionary of arguments for the regularizer seed: int seed for the NN - photon_computer: function + photon: Photon function to compute the photon PDF see model_gen.pdfNN_layer_generator for more information @@ -707,7 +707,7 @@ def _generate_pdf( impose_sumrule=self.impose_sumrule, scaler=self._scaler, parallel_models=self._parallel_models, - photon_computer=photon_computer, + photon=photon, ) return pdf_models @@ -880,13 +880,10 @@ def hyperparametrizable(self, params): xinput = self._xgrid_generation() # Initialize photon class: - photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) - - def photon_computer(x): - """Receives a grid with shape (1, n_x, 1) - and returns the photon PDF (1, n_x, 1) - """ - return photon.photon_fitting_scale(x[0,:,0])[np.newaxis,:,np.newaxis] + if self.fiatlux_runcard is not None: + photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) + else: + photon=None ### Training loop for k, partition in enumerate(self.kpartitions): @@ -906,7 +903,7 @@ def photon_computer(x): params.get("regularizer", None), # regularizer optional params.get("regularizer_args", None), seeds, - photon_computer if self.fiatlux_runcard is not None else None, + photon, ) # Register the fitting grid with the photon layer diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 23e019bd8f..4dd58b886f 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) -def gen_integration_input(nx, min=-9): +def gen_integration_input(nx): """ Generates a np.array (shaped (nx,1)) of nx elements where the nx/2 first elements are a logspace between 0 and 0.1 @@ -19,7 +19,7 @@ def gen_integration_input(nx, min=-9): """ lognx = int(nx / 2) linnx = int(nx - lognx) - xgrid_log = np.logspace(min, -1, lognx + 1) + xgrid_log = np.logspace(-9, -1, lognx + 1) xgrid_lin = np.linspace(0.1, 1, linnx) xgrid = np.concatenate([xgrid_log[:-1], xgrid_lin]).reshape(nx, 1) @@ -73,9 +73,7 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): # 3.1 If a photon is given, compute the photon component of the MSR photon_c = 0.0 if photon is not None: - xgrid_ph, wgt = gen_integration_input(100, min=-6) - photon_array = photon(np.expand_dims(xgrid_ph, axis=0))[0] - photon_c = np.sum(photon_array * wgt) + photon_c = photon.integrate() # 4. Now create the normalization by selecting the right integrations normalizer = MSR_Normalization(mode=mode, photon_contribution=photon_c) From f4a674e1ab0daacfc4d64b2542d65344c9eb5553 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 15 Dec 2022 23:18:58 +0100 Subject: [PATCH 038/204] Rewrite function alpha_em --- validphys2/src/validphys/photon/compute.py | 166 ++++++++++----------- 1 file changed, 75 insertions(+), 91 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 97cf91c389..6827a2334a 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -2,13 +2,13 @@ import lhapdf import fiatlux import numpy as np -from eko.couplings import Couplings -from eko.compatibility import update_theory from eko.output.legacy import load_tar from eko.interpolation import XGrid from eko.output.manipulate import xgrid_reshape -from eko.runner import Runner +from eko.interpolation import make_grid from .structure_functions import StructureFunction +from scipy.interpolate import interp1d +from scipy.integrate import trapezoid import yaml from os import remove @@ -19,14 +19,13 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: - self.qref = self.theory["Qref"] self.q_in2 = 100**2 - theory_coupling = update_theory(self.theory) - theory_coupling["ModEv"] = "TRN" - theory_coupling["alphaem_running"] = True - theory_coupling["nfref"] = 5 - theory_coupling['fact_to_ren_scale_ratio'] = 1. - self.couplings = Couplings.from_dict(theory_coupling) + self.alpha_em_ref = self.theory["alphaqed"] + self.qref = self.theory["Qref"] + self.eu2 = 4. / 9 + self.ed2 = 1. / 9 + self.e2q = [self.ed2, self.eu2, self.ed2, self.eu2, self.ed2, self.eu2] # d u s c b t + self.set_thresholds_a_em() self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) path_to_F2 = fiatlux_runcard["path_to_F2"] @@ -45,7 +44,7 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, self.F2LO) self.lux.InsertInelasticSplitQ([4.18, 1e100]) - self.cache = {} + self.produce_interpolator() def exctract_grids(self, xgrids): @@ -92,24 +91,18 @@ def F2LO(self, x, Q): F2_LO : float Structure function F2 at LO """ - thres_charm = self.theory["Qmc"] - thres_bottom = self.theory["Qmb"] - thres_top = self.theory["Qmt"] # at LO we use ZM-VFS - if Q < thres_charm : - hq = 3 - elif Q < thres_bottom : - hq = 4 - elif Q < thres_top : - hq = 5 + if Q < self.theory["Qmc"] : + nf = 3 + elif Q < self.theory["Qmb"] : + nf = 4 + elif Q < self.theory["Qmt"] : + nf = 5 else : - hq = 6 - e2u = 4/9 - e2d = 1/9 - e2q = [e2d, e2u, e2d, e2u, e2d, e2u] # d u s c b t + nf = 6 res = 0 - for i in range(1, hq+1): - res += e2q[i-1] * (self.qcd_pdfs.xfxQ(x, Q)[i] + self.qcd_pdfs.xfxQ(x, Q)[-i]) + for i in range(1, nf+1): + res += self.e2q[i-1] * (self.qcd_pdfs.xfxQ(x, Q)[i] + self.qcd_pdfs.xfxQ(x, Q)[-i]) return res def alpha_em(self, q): @@ -126,9 +119,45 @@ def alpha_em(self, q): alpha_em: float electromagnetic coupling """ - return self.couplings.a(q**2)[1] * 4 * np.pi + # return self.couplings.a(q**2)[1] * 4 * np.pi + if q < self.theory["Qmc"] : + nf = 3 + elif q < self.theory["Qmb"] : + nf = 4 + elif q < self.theory["Qmt"] : + nf = 5 + else : + nf = 6 + return self.a_em_nlo( + q, + self.a_thresh[nf], + self.thresh[nf], + nf + ) * (4 * np.pi) + + def a_em_nlo(self, q, a_ref, qref, nf): + nl = 3 + nc = 3 + nu = nf // 2 + nd = nf - nu + beta0 = ( -4.0 / 3 * (nl + nc * (nu * self.eu2 + nd * self.ed2)) ) + beta1 = -4.0 * ( nl + nc * (nu * self.eu2**2 + nd * self.ed2**2) ) + lmu = np.log(q / qref) + den = 1.0 + beta0 * a_ref * lmu + a_LO = a_ref / den + as_NLO = a_LO * (1 - beta1 / beta0 * a_LO * np.log(den)) + return as_NLO + + def set_thresholds_a_em(self): + a_ref = self.alpha_em_ref / (4 * np.pi) + self.a_em_mt = self.a_em_nlo(self.theory["Qmt"], a_ref, self.qref, 5) + self.a_em_mb = self.a_em_nlo(self.theory["Qmb"], a_ref, self.qref, 5) + self.a_em_mc = self.a_em_nlo(self.theory["Qmc"], self.a_em_mb, self.theory["Qmb"], 4) - def photon_fitting_scale(self, xgrids): + self.thresh = {3: self.theory["Qmc"], 4: self.theory["Qmb"], 5: self.qref, 6:self.theory["Qmt"]} + self.a_thresh = {3: self.a_em_mc, 4:self.a_em_mb, 5:self.alpha_em_ref/(4*np.pi), 6:self.a_em_mt} + + def compute_photon_array(self, xgrids): r""" Compute the photon PDF for every point in the grid xgrid. @@ -139,34 +168,20 @@ def photon_fitting_scale(self, xgrids): Returns ------- - photon_fitting_scale: numpy.array + compute_photon_array: numpy.array photon PDF at the scale 1 GeV """ xgrid_list = self.exctract_grids(xgrids) - grid_count = 0 photon_list = [] for xgrid in xgrid_list : photon_100GeV = np.zeros(len(xgrid)) for i, x in enumerate(xgrid): - for x_cached in self.cache : - if np.isclose(x, x_cached, atol=0, rtol=1e-3): - print("using cache for grid point", grid_count + i+1, "/", len(xgrids)) - photon_100GeV[i] = self.cache[x_cached] - break - else : - print("computing grid point", grid_count + i+1, "/", len(xgrids)) - photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x - self.cache[x] = photon_100GeV[i] - grid_count += len(xgrid) - - # if the grid has less than 5 points we cannot interpolate the precomputed grid - # since it has interpolation_polynomial_degree = 4. Therefore we compute the EKO. - if len(xgrid) > 4: - eko=load_tar(self.fiatlux_runcard['path_to_eko']) - xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) - else : - eko = self.compute_eko(xgrid) + print("computing grid point", i+1, "/", len(xgrids)) + photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x + + eko=load_tar(self.fiatlux_runcard['path_to_eko']) + xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) for j, pid in enumerate(eko.rotations.inputpids): @@ -186,50 +201,19 @@ def photon_fitting_scale(self, xgrids): pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) # error_final = np.einsum("ajbk,bk", elem.error, pdfs) - photon_fitting_scale = pdf_final[ph_id] - + photon_Q0 = pdf_final[ph_id] # we want x * gamma(x) - photon_list.append( xgrid * photon_fitting_scale ) + photon_list.append( xgrid * photon_Q0 ) return np.concatenate(photon_list) - def compute_eko(self, xgrid): - - theory = self.theory.copy() - theory["nfref"] = 5 - theory["nf0"] = None - theory["fact_to_ren_scale_ratio"] = 1. - theory["ModSV"] = None - theory["IB"] = 0 - theory["FNS"] = "VFNS" - theory["QED"] = 2 - theory["ModEv"] = "EXA" - theory["alphaem_running"] = True - q_in = 100 - q_fin = self.theory["Q0"] - theory["Q0"]= q_in - - operator_card = dict( - sorted( - dict( - interpolation_xgrid=xgrid.tolist(), - interpolation_polynomial_degree= 4 if len(xgrid) > 4 else len(xgrid) - 1, - interpolation_is_log=True, - ev_op_max_order=10, - ev_op_iterations=10, - backward_inversion="exact", - n_integration_cores=0, - debug_skip_non_singlet=False, - debug_skip_singlet=False, - Q2grid=[q_fin**2], - inputgrid=None, - targetgrid=None, - inputpids=None, - targetpids=None, - ).items() - ) - ) - - runner = Runner(theory, operator_card) - return runner.get_output() - \ No newline at end of file + def produce_interpolator(self): + self.xgrid = make_grid(98, 99, x_min=1.e-9) # TODO : use the output grid so the EKO will not be reshaped + self.photon_array = self.compute_photon_array(self.xgrid) + self.interpolator = interp1d(self.xgrid, self.photon_array, fill_value=0.) + + def compute(self, xgrid): + return self.interpolator(xgrid[0,:,0])[np.newaxis,:,np.newaxis] + + def integrate(self): + return trapezoid(self.photon_array, self.xgrid) \ No newline at end of file From aabc9a10c466121bf05082966a6cf6578667e2f9 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 16 Dec 2022 11:01:26 +0100 Subject: [PATCH 039/204] Copmute photon grid on the output grid --- validphys2/src/validphys/photon/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 6827a2334a..2453b9aed0 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -24,7 +24,7 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.qref = self.theory["Qref"] self.eu2 = 4. / 9 self.ed2 = 1. / 9 - self.e2q = [self.ed2, self.eu2, self.ed2, self.eu2, self.ed2, self.eu2] # d u s c b t + self.eq2 = [self.ed2, self.eu2, self.ed2, self.eu2, self.ed2, self.eu2] # d u s c b t self.set_thresholds_a_em() self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) @@ -46,7 +46,7 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.produce_interpolator() - + # since we interpolate this is useless now def exctract_grids(self, xgrids): r""" Extract the subgrids inside xgrids. @@ -102,7 +102,7 @@ def F2LO(self, x, Q): nf = 6 res = 0 for i in range(1, nf+1): - res += self.e2q[i-1] * (self.qcd_pdfs.xfxQ(x, Q)[i] + self.qcd_pdfs.xfxQ(x, Q)[-i]) + res += self.eq2[i-1] * (self.qcd_pdfs.xfxQ(x, Q)[i] + self.qcd_pdfs.xfxQ(x, Q)[-i]) return res def alpha_em(self, q): @@ -119,7 +119,6 @@ def alpha_em(self, q): alpha_em: float electromagnetic coupling """ - # return self.couplings.a(q**2)[1] * 4 * np.pi if q < self.theory["Qmc"] : nf = 3 elif q < self.theory["Qmb"] : @@ -208,7 +207,8 @@ def compute_photon_array(self, xgrids): return np.concatenate(photon_list) def produce_interpolator(self): - self.xgrid = make_grid(98, 99, x_min=1.e-9) # TODO : use the output grid so the EKO will not be reshaped + # TODO : pass the grid in a more clever way + self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) self.photon_array = self.compute_photon_array(self.xgrid) self.interpolator = interp1d(self.xgrid, self.photon_array, fill_value=0.) From ba413b3ca00d24b9bc416437492c7c635641b5e9 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 16 Dec 2022 12:58:12 +0100 Subject: [PATCH 040/204] Clean photon class --- validphys2/src/validphys/photon/compute.py | 97 +++++++--------------- 1 file changed, 32 insertions(+), 65 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 2453b9aed0..29ece2123b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -14,7 +14,7 @@ from os import remove class Photon: - def __init__(self, theoryid, fiatlux_runcard, replica=0): + def __init__(self, theoryid, fiatlux_runcard, replica_id): # TODO : for the moment we do the 0-th replica then we change it self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard @@ -27,7 +27,7 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.eq2 = [self.ed2, self.eu2, self.ed2, self.eu2, self.ed2, self.eu2] # d u s c b t self.set_thresholds_a_em() - self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica) + self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica_id) path_to_F2 = fiatlux_runcard["path_to_F2"] path_to_FL = fiatlux_runcard["path_to_FL"] f2 = StructureFunction(path_to_F2, self.qcd_pdfs) @@ -45,35 +45,6 @@ def __init__(self, theoryid, fiatlux_runcard, replica=0): self.lux.InsertInelasticSplitQ([4.18, 1e100]) self.produce_interpolator() - - # since we interpolate this is useless now - def exctract_grids(self, xgrids): - r""" - Extract the subgrids inside xgrids. - - xgrids is the concatenation of different grids, i.e. - xgrid = np.array([xmin1, ..., xmax1, xmin2, ...,xmax2, xmin3, ...]). - The different grids are extracted and stored in a list: - xgrid_list = [np.array([xgrid1]), np.array([xgrid2]), ...] - - Parameters - ---------- - xgrids : nd.array - concatenation of the subgrids - - Returns - ------- - xgrid_list : list - list containing the different grids - """ - xgrid_list = [] - imin = 0 - for i in range(1, len(xgrids)): - if xgrids[i-1] > xgrids[i] : - xgrid_list.append(xgrids[imin:i]) - imin = i - xgrid_list.append(xgrids[imin:]) - return xgrid_list def F2LO(self, x, Q): r""" @@ -156,7 +127,7 @@ def set_thresholds_a_em(self): self.thresh = {3: self.theory["Qmc"], 4: self.theory["Qmb"], 5: self.qref, 6:self.theory["Qmt"]} self.a_thresh = {3: self.a_em_mc, 4:self.a_em_mb, 5:self.alpha_em_ref/(4*np.pi), 6:self.a_em_mt} - def compute_photon_array(self, xgrids): + def compute_photon_array(self, xgrid): r""" Compute the photon PDF for every point in the grid xgrid. @@ -169,42 +140,38 @@ def compute_photon_array(self, xgrids): ------- compute_photon_array: numpy.array photon PDF at the scale 1 GeV - """ - xgrid_list = self.exctract_grids(xgrids) - - photon_list = [] - for xgrid in xgrid_list : - photon_100GeV = np.zeros(len(xgrid)) - for i, x in enumerate(xgrid): - print("computing grid point", i+1, "/", len(xgrids)) - photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x - - eko=load_tar(self.fiatlux_runcard['path_to_eko']) - xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) + """ + photon_100GeV = np.zeros(len(xgrid)) + for i, x in enumerate(xgrid): + print("computing grid point", i+1, "/", len(xgrid)) + # photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x + photon_100GeV[i] = np.exp(-x) + + eko=load_tar(self.fiatlux_runcard['path_to_eko']) + xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) - pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) - for j, pid in enumerate(eko.rotations.inputpids): - if pid == 22 : - pdfs[j] = photon_100GeV - ph_id = j - if not self.qcd_pdfs.hasFlavor(pid): - continue - pdfs[j] = np.array( - [ - self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x - for x in xgrid - ] - ) + pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) + for j, pid in enumerate(eko.rotations.inputpids): + if pid == 22 : + pdfs[j] = photon_100GeV + ph_id = j + if not self.qcd_pdfs.hasFlavor(pid): + continue + pdfs[j] = np.array( + [ + self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x + for x in xgrid + ] + ) - for q2, elem in eko.items(): - pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) - # error_final = np.einsum("ajbk,bk", elem.error, pdfs) + for q2, elem in eko.items(): + pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) + # error_final = np.einsum("ajbk,bk", elem.error, pdfs) + + photon_Q0 = pdf_final[ph_id] - photon_Q0 = pdf_final[ph_id] - # we want x * gamma(x) - photon_list.append( xgrid * photon_Q0 ) - - return np.concatenate(photon_list) + # we want x * gamma(x) + return xgrid * photon_Q0 def produce_interpolator(self): # TODO : pass the grid in a more clever way From fed3221c2468eb9fe80bead3b0179842520af5b5 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 16 Dec 2022 14:22:34 +0100 Subject: [PATCH 041/204] Remove line used for debugging --- validphys2/src/validphys/photon/compute.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 29ece2123b..5ac1c981a9 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -144,8 +144,7 @@ def compute_photon_array(self, xgrid): photon_100GeV = np.zeros(len(xgrid)) for i, x in enumerate(xgrid): print("computing grid point", i+1, "/", len(xgrid)) - # photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x - photon_100GeV[i] = np.exp(-x) + photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x eko=load_tar(self.fiatlux_runcard['path_to_eko']) xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) From b616523cf61fcc6ee3c608a42b5e6a04454a912a Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 16 Dec 2022 14:24:15 +0100 Subject: [PATCH 042/204] Handle different replicas --- n3fit/src/n3fit/layers/msr_normalization.py | 13 +++++++++---- n3fit/src/n3fit/layers/rotations.py | 12 ++++++------ n3fit/src/n3fit/model_gen.py | 12 ++++++------ n3fit/src/n3fit/model_trainer.py | 21 +++++++++++++-------- n3fit/src/n3fit/msr.py | 14 +++++++------- n3fit/src/n3fit/performfit.py | 3 +-- 6 files changed, 42 insertions(+), 33 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 4a5ac4531d..39445cd1cb 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -16,8 +16,8 @@ class MSR_Normalization(MetaLayer): _msr_enabled = False _vsr_enabled = False - def __init__(self, output_dim=14, mode="ALL", photon_contribution=0.0, **kwargs): - self._photon = photon_contribution + def __init__(self, output_dim=14, mode="ALL", photons_contribution=None, **kwargs): + self._photons = photons_contribution if mode == True or mode.upper() == "ALL": self._msr_enabled = True self._vsr_enabled = True @@ -40,7 +40,7 @@ def __init__(self, output_dim=14, mode="ALL", photon_contribution=0.0, **kwargs) super().__init__(**kwargs, name="normalizer") - def call(self, pdf_integrated): + def call(self, pdf_integrated, id): """Imposes the valence and momentum sum rules: A_g = (1-sigma)/g A_v = A_v24 = A_v35 = 3/V @@ -52,9 +52,14 @@ def call(self, pdf_integrated): """ y = op.flatten(pdf_integrated) norm_constants = [] + + if self._photons is not None: + photon_integral = self._photons[id] + else : + photon_integral = 0. if self._msr_enabled: - n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - self._photon) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) + n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - photon_integral) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index cc76c59010..bd386e4b0d 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -101,8 +101,8 @@ class AddPhoton(MetaLayer): In order to change the shape it is necessary to rebuild the photon. """ - def __init__(self, photon, **kwargs): - self._photon_generator = photon + def __init__(self, photons, **kwargs): + self._photons_generator = photons self._pdf_ph = None super().__init__(**kwargs) @@ -110,14 +110,14 @@ def register_photon(self, xgrid): """Compute the photon array and set the layer to be rebuilt""" # TODO: maybe add here some caching mechanism so that the photon doesn't get # recomputed if the grid hasn't changed! - if self._photon_generator is not None: - self._pdf_ph = self._photon_generator.compute(xgrid) + if self._photons_generator is not None: + self._pdf_ph = [photon.compute(xgrid) for photon in self._photons_generator] self.built = False - def call(self, pdfs): + def call(self, pdfs, i): if self._pdf_ph is None: return pdfs - return op.concatenate([self._pdf_ph, pdfs[:,:,1:]], axis=-1) + return op.concatenate([self._pdf_ph[i], pdfs[:,:,1:]], axis=-1) class ObsRotation(MetaLayer): diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 4cd85dfacf..ecdea276a6 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -403,7 +403,7 @@ def pdfNN_layer_generator( impose_sumrule=None, scaler=None, parallel_models=1, - photon=None, + photons=None, ): # pylint: disable=too-many-locals """ Generates the PDF model which takes as input a point in x (from 0 to 1) @@ -566,14 +566,14 @@ def pdfNN_layer_generator( layer_evln = FkRotation(input_shape=(last_layer_nodes,), output_dim=out) # Photon layer - layer_photon = AddPhoton(photon=photon) + layer_photon = AddPhoton(photons=photons) # Basis rotation basis_rotation = FlavourToEvolution(flav_info=flav_info, fitbasis=fitbasis) # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photon=photon) + sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photons=photons) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x @@ -648,13 +648,13 @@ def layer_pdf(x): return layer_evln(layer_fitbasis(x)) # Final PDF (apply normalization) - normalized_pdf = sumrule_layer(layer_pdf) + normalized_pdf = sumrule_layer(layer_pdf, i) # Photon layer, changes the photon from zero to non-zero def apply_photon(x): - return layer_photon(normalized_pdf(x)) + return layer_photon(normalized_pdf(x), i) - if photon is None: + if photons is None: final_pdf = normalized_pdf else: final_pdf = apply_photon diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 379c234788..489a9e4f4a 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -102,7 +102,7 @@ def __init__( parallel_models=1, theoryid=None, fiatlux_runcard=None, - replica_id=None, + replicas_id=None, ): """ Parameters @@ -157,7 +157,7 @@ def __init__( self._parallel_models = parallel_models self.theoryid = theoryid self.fiatlux_runcard = fiatlux_runcard - self.replica_id = replica_id + self.replicas_id = replicas_id # Initialise internal variables which define behaviour if debug: @@ -653,7 +653,7 @@ def _generate_pdf( regularizer, regularizer_args, seed, - photon, + photons, ): """ Defines the internal variable layer_pdf @@ -707,7 +707,7 @@ def _generate_pdf( impose_sumrule=self.impose_sumrule, scaler=self._scaler, parallel_models=self._parallel_models, - photon=photon, + photons=photons, ) return pdf_models @@ -879,11 +879,16 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() - # Initialize photon class: + # Initialize all photon classes for the different replicas: if self.fiatlux_runcard is not None: - photon=Photon(theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard) + # TODO : implement the different replicas inside the Photon class + photons=[Photon( + theoryid=self.theoryid, + fiatlux_runcard=self.fiatlux_runcard, + replica_id = id + ) for id in self.replicas_id] else: - photon=None + photons=None ### Training loop for k, partition in enumerate(self.kpartitions): @@ -903,7 +908,7 @@ def hyperparametrizable(self, params): params.get("regularizer", None), # regularizer optional params.get("regularizer_args", None), seeds, - photon, + photons, ) # Register the fitting grid with the photon layer diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 4dd58b886f..4b6e1f0fe9 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -36,7 +36,7 @@ def gen_integration_input(nx): return xgrid, weights_array -def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): +def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): """ This function receives: Generates a function that applies a normalization layer to the fit. @@ -71,24 +71,24 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photon=None): integrator = xIntegrator(weights_array, input_shape=(nx,)) # 3.1 If a photon is given, compute the photon component of the MSR - photon_c = 0.0 - if photon is not None: - photon_c = photon.integrate() + photons_c = None + if photons is not None: + photons_c = [photon.integrate() for photon in photons] # 4. Now create the normalization by selecting the right integrations - normalizer = MSR_Normalization(mode=mode, photon_contribution=photon_c) + normalizer = MSR_Normalization(mode=mode, photons_contribution=photons_c) # 5. Make the xgrid array into a backend input layer so it can be given to the normalization xgrid_input = op.numpy_to_input(xgrid, name="integration_grid") # Finally prepare a function which will take as input the output of the PDF model # and will return it appropiately normalized. - def apply_normalization(layer_pdf): + def apply_normalization(layer_pdf, id): """ layer_pdf: output of the PDF, unnormalized, ready for the fktable """ x_original = op.op_gather_keep_dims(xgrid_input, -1, axis=-1) pdf_integrand = op.op_multiply([division_by_x(x_original), layer_pdf(xgrid_input)]) - normalization = normalizer(integrator(pdf_integrand)) + normalization = normalizer(integrator(pdf_integrand), id) def ultimate_pdf(x): return layer_pdf(x)*normalization diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 01d9459b83..3f4c1e7de7 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -197,8 +197,7 @@ def performfit( parallel_models=n_models, theoryid=theoryid, fiatlux_runcard=fiatlux, - # replica_id=replica_idxs[0], - replica_id=None, + replicas_id=replica_idxs, ) # This is just to give a descriptive name to the fit function From 36ad12d918c740ea117c4f93b0bc86c92a5df5df Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 16 Dec 2022 14:46:42 +0100 Subject: [PATCH 043/204] Upgrade docstrings in photon class --- n3fit/src/n3fit/layers/msr_normalization.py | 6 ++-- n3fit/src/n3fit/msr.py | 4 +-- validphys2/src/validphys/photon/compute.py | 38 +++++++++++++++++++-- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 39445cd1cb..97111e83c6 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -40,7 +40,7 @@ def __init__(self, output_dim=14, mode="ALL", photons_contribution=None, **kwarg super().__init__(**kwargs, name="normalizer") - def call(self, pdf_integrated, id): + def call(self, pdf_integrated, ph_replica): """Imposes the valence and momentum sum rules: A_g = (1-sigma)/g A_v = A_v24 = A_v35 = 3/V @@ -52,9 +52,9 @@ def call(self, pdf_integrated, id): """ y = op.flatten(pdf_integrated) norm_constants = [] - + if self._photons is not None: - photon_integral = self._photons[id] + photon_integral = self._photons[ph_replica] else : photon_integral = 0. diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 4b6e1f0fe9..b5f9e4b88c 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -82,13 +82,13 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): xgrid_input = op.numpy_to_input(xgrid, name="integration_grid") # Finally prepare a function which will take as input the output of the PDF model # and will return it appropiately normalized. - def apply_normalization(layer_pdf, id): + def apply_normalization(layer_pdf, ph_replica): """ layer_pdf: output of the PDF, unnormalized, ready for the fktable """ x_original = op.op_gather_keep_dims(xgrid_input, -1, axis=-1) pdf_integrand = op.op_multiply([division_by_x(x_original), layer_pdf(xgrid_input)]) - normalization = normalizer(integrator(pdf_integrand), id) + normalization = normalizer(integrator(pdf_integrand), ph_replica) def ultimate_pdf(x): return layer_pdf(x)*normalization diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 5ac1c981a9..a849bc296b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -5,7 +5,6 @@ from eko.output.legacy import load_tar from eko.interpolation import XGrid from eko.output.manipulate import xgrid_reshape -from eko.interpolation import make_grid from .structure_functions import StructureFunction from scipy.interpolate import interp1d from scipy.integrate import trapezoid @@ -15,7 +14,6 @@ class Photon: def __init__(self, theoryid, fiatlux_runcard, replica_id): - # TODO : for the moment we do the 0-th replica then we change it self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: @@ -106,6 +104,25 @@ def alpha_em(self, q): ) * (4 * np.pi) def a_em_nlo(self, q, a_ref, qref, nf): + """ + Compute the alpha_em running for FFS at NLO. + + Parameters + ---------- + q : float + target scale + a_ref : float + reference value of a = alpha_em/(4*pi) + qref: float + reference scale + nf: int + number of flavors + + Returns + ------- + as_NLO : float + target value of a + """ nl = 3 nc = 3 nu = nf // 2 @@ -119,6 +136,7 @@ def a_em_nlo(self, q, a_ref, qref, nf): return as_NLO def set_thresholds_a_em(self): + """Compute and store the couplings at thresholds""" a_ref = self.alpha_em_ref / (4 * np.pi) self.a_em_mt = self.a_em_nlo(self.theory["Qmt"], a_ref, self.qref, 5) self.a_em_mb = self.a_em_nlo(self.theory["Qmb"], a_ref, self.qref, 5) @@ -147,6 +165,8 @@ def compute_photon_array(self, xgrid): photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x eko=load_tar(self.fiatlux_runcard['path_to_eko']) + # If we make sure that the grid of the precomputed EKO is the same of + # self.xgrid then we don't need to reshape xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) @@ -179,7 +199,21 @@ def produce_interpolator(self): self.interpolator = interp1d(self.xgrid, self.photon_array, fill_value=0.) def compute(self, xgrid): + """ + Compute the photon interpolating the values of self.photon_array. + + Parameters + ---------- + xgrid : nd.array + array of x values with shape (1,xgrid,1) + + Returns + ------- + photon values : nd.array + array of photon values with shape (1,xgrid,1) + """ return self.interpolator(xgrid[0,:,0])[np.newaxis,:,np.newaxis] def integrate(self): + """Compute the integral of the photon on the x range""" return trapezoid(self.photon_array, self.xgrid) \ No newline at end of file From 59b2aec996c2773cfd12df7d1ea7ee806ef62e4a Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 20 Dec 2022 15:11:32 +0100 Subject: [PATCH 044/204] Move different replicas inside Photon class --- n3fit/src/n3fit/layers/rotations.py | 2 +- n3fit/src/n3fit/model_trainer.py | 7 +- n3fit/src/n3fit/msr.py | 2 +- validphys2/src/validphys/photon/compute.py | 88 ++++++++----------- .../validphys/photon/structure_functions.py | 46 ++++++++++ 5 files changed, 87 insertions(+), 58 deletions(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index bd386e4b0d..511fd35717 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -111,7 +111,7 @@ def register_photon(self, xgrid): # TODO: maybe add here some caching mechanism so that the photon doesn't get # recomputed if the grid hasn't changed! if self._photons_generator is not None: - self._pdf_ph = [photon.compute(xgrid) for photon in self._photons_generator] + self._pdf_ph = self._photons_generator.compute(xgrid) self.built = False def call(self, pdfs, i): diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 489a9e4f4a..848a9dc7b0 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -881,12 +881,11 @@ def hyperparametrizable(self, params): # Initialize all photon classes for the different replicas: if self.fiatlux_runcard is not None: - # TODO : implement the different replicas inside the Photon class - photons=[Photon( + photons=Photon( theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard, - replica_id = id - ) for id in self.replicas_id] + replicas_id=self.replicas_id, + ) else: photons=None diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index b5f9e4b88c..6325d4afbb 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -73,7 +73,7 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): # 3.1 If a photon is given, compute the photon component of the MSR photons_c = None if photons is not None: - photons_c = [photon.integrate() for photon in photons] + photons_c = photons.integrate() # 4. Now create the normalization by selecting the right integrations normalizer = MSR_Normalization(mode=mode, photons_contribution=photons_c) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index a849bc296b..400ea5937b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -5,7 +5,7 @@ from eko.output.legacy import load_tar from eko.interpolation import XGrid from eko.output.manipulate import xgrid_reshape -from .structure_functions import StructureFunction +from .structure_functions import StructureFunction, F2LO from scipy.interpolate import interp1d from scipy.integrate import trapezoid @@ -13,67 +13,48 @@ from os import remove class Photon: - def __init__(self, theoryid, fiatlux_runcard, replica_id): + def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard if fiatlux_runcard is not None: + self.replicas_id = replicas_id self.q_in2 = 100**2 self.alpha_em_ref = self.theory["alphaqed"] self.qref = self.theory["Qref"] + self.Qmc = self.theory["Qmc"] + self.Qmb = self.theory["Qmb"] + self.Qmt = self.theory["Qmt"] + if self.theory["MaxNfAs"] <= 5 : + self.Qmt = np.inf + if self.theory["MaxNfAs"] <= 4 : + self.Qmb = np.inf + if self.theory["MaxNfAs"] <= 3 : + self.Qmc = np.inf self.eu2 = 4. / 9 self.ed2 = 1. / 9 - self.eq2 = [self.ed2, self.eu2, self.ed2, self.eu2, self.ed2, self.eu2] # d u s c b t self.set_thresholds_a_em() - self.qcd_pdfs = lhapdf.mkPDF(fiatlux_runcard["pdf_name"], replica_id) + self.qcd_pdfs = [lhapdf.mkPDF(fiatlux_runcard["pdf_name"], id) for id in replicas_id] path_to_F2 = fiatlux_runcard["path_to_F2"] path_to_FL = fiatlux_runcard["path_to_FL"] - f2 = StructureFunction(path_to_F2, self.qcd_pdfs) - fl = StructureFunction(path_to_FL, self.qcd_pdfs) + f2 = [StructureFunction(path_to_F2, pdfs) for pdfs in self.qcd_pdfs] + fl = [StructureFunction(path_to_FL, pdfs) for pdfs in self.qcd_pdfs] + f2lo = [F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] # lux = fiatlux.FiatLux(fiatlux_runcard) # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick ff = open('fiatlux_runcard.yml', 'w+') yaml.dump(self.fiatlux_runcard, ff) - self.lux = fiatlux.FiatLux('fiatlux_runcard.yml') + self.lux = [fiatlux.FiatLux('fiatlux_runcard.yml') for i in range(len(replicas_id))] remove('fiatlux_runcard.yml') - self.lux.PlugAlphaQED(self.alpha_em, self.qref) - self.lux.PlugStructureFunctions(f2.FxQ, fl.FxQ, self.F2LO) - self.lux.InsertInelasticSplitQ([4.18, 1e100]) + for i in range(len(replicas_id)): + self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) + self.lux[i].InsertInelasticSplitQ([4.18, 1e100]) + self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) self.produce_interpolator() - def F2LO(self, x, Q): - r""" - Compute the LO DIS structure function F2. - - Parameters - ---------- - x : float - Bjorken's variable - Q : float - DIS hard scale - - Returns - ------- - F2_LO : float - Structure function F2 at LO - """ - # at LO we use ZM-VFS - if Q < self.theory["Qmc"] : - nf = 3 - elif Q < self.theory["Qmb"] : - nf = 4 - elif Q < self.theory["Qmt"] : - nf = 5 - else : - nf = 6 - res = 0 - for i in range(1, nf+1): - res += self.eq2[i-1] * (self.qcd_pdfs.xfxQ(x, Q)[i] + self.qcd_pdfs.xfxQ(x, Q)[-i]) - return res - def alpha_em(self, q): r""" Compute the value of alpha_em. @@ -88,11 +69,11 @@ def alpha_em(self, q): alpha_em: float electromagnetic coupling """ - if q < self.theory["Qmc"] : + if q < self.Qmc : nf = 3 - elif q < self.theory["Qmb"] : + elif q < self.Qmb : nf = 4 - elif q < self.theory["Qmt"] : + elif q < self.Qmt : nf = 5 else : nf = 6 @@ -145,7 +126,7 @@ def set_thresholds_a_em(self): self.thresh = {3: self.theory["Qmc"], 4: self.theory["Qmb"], 5: self.qref, 6:self.theory["Qmt"]} self.a_thresh = {3: self.a_em_mc, 4:self.a_em_mb, 5:self.alpha_em_ref/(4*np.pi), 6:self.a_em_mt} - def compute_photon_array(self, xgrid): + def compute_photon_array(self, xgrid, id): r""" Compute the photon PDF for every point in the grid xgrid. @@ -162,23 +143,23 @@ def compute_photon_array(self, xgrid): photon_100GeV = np.zeros(len(xgrid)) for i, x in enumerate(xgrid): print("computing grid point", i+1, "/", len(xgrid)) - photon_100GeV[i] = self.lux.EvaluatePhoton(x, self.q_in2).total / x + photon_100GeV[i] = self.lux[id].EvaluatePhoton(x, self.q_in2).total / x eko=load_tar(self.fiatlux_runcard['path_to_eko']) # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) - + pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) for j, pid in enumerate(eko.rotations.inputpids): if pid == 22 : pdfs[j] = photon_100GeV ph_id = j - if not self.qcd_pdfs.hasFlavor(pid): + if not self.qcd_pdfs[id].hasFlavor(pid): continue pdfs[j] = np.array( [ - self.qcd_pdfs.xfxQ2(pid, x, self.q_in2) / x + self.qcd_pdfs[id].xfxQ2(pid, x, self.q_in2) / x for x in xgrid ] ) @@ -195,8 +176,8 @@ def compute_photon_array(self, xgrid): def produce_interpolator(self): # TODO : pass the grid in a more clever way self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) - self.photon_array = self.compute_photon_array(self.xgrid) - self.interpolator = interp1d(self.xgrid, self.photon_array, fill_value=0.) + self.photons_array = [self.compute_photon_array(self.xgrid, i) for i in range(len(self.replicas_id))] + self.interpolator = [interp1d(self.xgrid, photon_array, fill_value=0.) for photon_array in self.photons_array] def compute(self, xgrid): """ @@ -212,8 +193,11 @@ def compute(self, xgrid): photon values : nd.array array of photon values with shape (1,xgrid,1) """ - return self.interpolator(xgrid[0,:,0])[np.newaxis,:,np.newaxis] + return [ + self.interpolator[id](xgrid[0,:,0])[np.newaxis,:,np.newaxis] + for id in range(len(self.replicas_id)) + ] def integrate(self): """Compute the integral of the photon on the x range""" - return trapezoid(self.photon_array, self.xgrid) \ No newline at end of file + return [trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id))] \ No newline at end of file diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index e5b392b0fc..57ee93576c 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -25,3 +25,49 @@ def produce_interpolator(self): def FxQ(self, x, Q): return self.interpolator(x, Q**2)[0,0] + +class F2LO : + def __init__(self, pdfs, theory): + self.pdfs = pdfs + self.Qmc = theory["Qmc"] + self.Qmb = theory["Qmb"] + self.Qmt = theory["Qmt"] + if theory["MaxNfPdf"] <= 5 : + self.Qmt = np.inf + if theory["MaxNfPdf"] <= 4 : + self.Qmb = np.inf + if theory["MaxNfPdf"] <= 3 : + self.Qmc = np.inf + eu2 = 4. / 9 + ed2 = 1. / 9 + self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t + + def FxQ(self, x, Q): + r""" + Compute the LO DIS structure function F2. + + Parameters + ---------- + x : float + Bjorken's variable + Q : float + DIS hard scale + + Returns + ------- + F2_LO : float + Structure function F2 at LO + """ + # at LO we use ZM-VFS + if Q < self.Qmc : + nf = 3 + elif Q < self.Qmb : + nf = 4 + elif Q < self.Qmt : + nf = 5 + else : + nf = 6 + res = 0 + for i in range(1, nf+1): + res += self.eq2[i-1] * (self.pdfs.xfxQ(x, Q)[i] + self.pdfs.xfxQ(x, Q)[-i]) + return res From 94acfc873f88922d3c89b82769c0f9770746e589 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 21 Dec 2022 14:56:18 +0100 Subject: [PATCH 045/204] Clean Photon class --- validphys2/src/validphys/photon/compute.py | 80 +++++++++++----------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 400ea5937b..e963fca44e 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -16,44 +16,46 @@ class Photon: def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard - if fiatlux_runcard is not None: - self.replicas_id = replicas_id - self.q_in2 = 100**2 - self.alpha_em_ref = self.theory["alphaqed"] - self.qref = self.theory["Qref"] - self.Qmc = self.theory["Qmc"] - self.Qmb = self.theory["Qmb"] - self.Qmt = self.theory["Qmt"] - if self.theory["MaxNfAs"] <= 5 : - self.Qmt = np.inf - if self.theory["MaxNfAs"] <= 4 : - self.Qmb = np.inf - if self.theory["MaxNfAs"] <= 3 : - self.Qmc = np.inf - self.eu2 = 4. / 9 - self.ed2 = 1. / 9 - self.set_thresholds_a_em() - - self.qcd_pdfs = [lhapdf.mkPDF(fiatlux_runcard["pdf_name"], id) for id in replicas_id] - path_to_F2 = fiatlux_runcard["path_to_F2"] - path_to_FL = fiatlux_runcard["path_to_FL"] - f2 = [StructureFunction(path_to_F2, pdfs) for pdfs in self.qcd_pdfs] - fl = [StructureFunction(path_to_FL, pdfs) for pdfs in self.qcd_pdfs] - f2lo = [F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] - - # lux = fiatlux.FiatLux(fiatlux_runcard) - # we have a dict but fiatlux wants a yaml file - # TODO : remove this dirty trick - ff = open('fiatlux_runcard.yml', 'w+') - yaml.dump(self.fiatlux_runcard, ff) - self.lux = [fiatlux.FiatLux('fiatlux_runcard.yml') for i in range(len(replicas_id))] - remove('fiatlux_runcard.yml') - for i in range(len(replicas_id)): - self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) - self.lux[i].InsertInelasticSplitQ([4.18, 1e100]) - self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) - - self.produce_interpolator() + self.replicas_id = replicas_id + self.q_in2 = 100**2 + + # parameters for the alphaem running + self.alpha_em_ref = self.theory["alphaqed"] + self.qref = self.theory["Qref"] + self.Qmc = self.theory["Qmc"] + self.Qmb = self.theory["Qmb"] + self.Qmt = self.theory["Qmt"] + if self.theory["MaxNfAs"] <= 5 : + self.Qmt = np.inf + if self.theory["MaxNfAs"] <= 4 : + self.Qmb = np.inf + if self.theory["MaxNfAs"] <= 3 : + self.Qmc = np.inf + self.eu2 = 4. / 9 + self.ed2 = 1. / 9 + self.set_thresholds_a_em() + + # structure functions + self.qcd_pdfs = [lhapdf.mkPDF(fiatlux_runcard["pdf_name"], id) for id in replicas_id] + path_to_F2 = fiatlux_runcard["path_to_F2"] + path_to_FL = fiatlux_runcard["path_to_FL"] + f2 = [StructureFunction(path_to_F2, pdfs) for pdfs in self.qcd_pdfs] + fl = [StructureFunction(path_to_FL, pdfs) for pdfs in self.qcd_pdfs] + f2lo = [F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] + + # set fiatlux + ff = open('fiatlux_runcard.yml', 'w+') + yaml.dump(self.fiatlux_runcard, ff) + self.lux = [fiatlux.FiatLux('fiatlux_runcard.yml') for i in range(len(replicas_id))] + remove('fiatlux_runcard.yml') + # we have a dict but fiatlux wants a yaml file + # TODO : remove this dirty trick + for i in range(len(replicas_id)): + self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) + self.lux[i].InsertInelasticSplitQ([4.18, 1e100]) + self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) + + self.produce_interpolators() def alpha_em(self, q): r""" @@ -173,7 +175,7 @@ def compute_photon_array(self, xgrid, id): # we want x * gamma(x) return xgrid * photon_Q0 - def produce_interpolator(self): + def produce_interpolators(self): # TODO : pass the grid in a more clever way self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) self.photons_array = [self.compute_photon_array(self.xgrid, i) for i in range(len(self.replicas_id))] From 08255ecb1e1d0d8954e721f70f90182596a6cc99 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 9 Jan 2023 17:03:53 +0100 Subject: [PATCH 046/204] Fix error in alphaem running --- validphys2/src/validphys/photon/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index e963fca44e..b1f27d36db 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -112,7 +112,7 @@ def a_em_nlo(self, q, a_ref, qref, nf): nd = nf - nu beta0 = ( -4.0 / 3 * (nl + nc * (nu * self.eu2 + nd * self.ed2)) ) beta1 = -4.0 * ( nl + nc * (nu * self.eu2**2 + nd * self.ed2**2) ) - lmu = np.log(q / qref) + lmu = np.log(q**2 / qref**2) den = 1.0 + beta0 * a_ref * lmu a_LO = a_ref / den as_NLO = a_LO * (1 - beta1 / beta0 * a_LO * np.log(den)) From 36414128a5f4aafef91ce0c83f8a90f9422e6ea2 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 9 Jan 2023 19:32:05 +0100 Subject: [PATCH 047/204] Precompute beta(nf) in the constructor --- validphys2/src/validphys/photon/compute.py | 50 ++++++++++++---------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index b1f27d36db..33d19d9f2a 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -20,6 +20,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.q_in2 = 100**2 # parameters for the alphaem running + self.set_betas() self.alpha_em_ref = self.theory["alphaqed"] self.qref = self.theory["Qref"] self.Qmc = self.theory["Qmc"] @@ -31,8 +32,6 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.Qmb = np.inf if self.theory["MaxNfAs"] <= 3 : self.Qmc = np.inf - self.eu2 = 4. / 9 - self.ed2 = 1. / 9 self.set_thresholds_a_em() # structure functions @@ -79,14 +78,14 @@ def alpha_em(self, q): nf = 5 else : nf = 6 - return self.a_em_nlo( + return self.alpha_em_nlo( q, - self.a_thresh[nf], + self.alpha_thresh[nf], self.thresh[nf], nf - ) * (4 * np.pi) + ) - def a_em_nlo(self, q, a_ref, qref, nf): + def alpha_em_nlo(self, q, alpha_ref, qref, nf): """ Compute the alpha_em running for FFS at NLO. @@ -106,27 +105,34 @@ def a_em_nlo(self, q, a_ref, qref, nf): as_NLO : float target value of a """ - nl = 3 - nc = 3 - nu = nf // 2 - nd = nf - nu - beta0 = ( -4.0 / 3 * (nl + nc * (nu * self.eu2 + nd * self.ed2)) ) - beta1 = -4.0 * ( nl + nc * (nu * self.eu2**2 + nd * self.ed2**2) ) - lmu = np.log(q**2 / qref**2) - den = 1.0 + beta0 * a_ref * lmu - a_LO = a_ref / den - as_NLO = a_LO * (1 - beta1 / beta0 * a_LO * np.log(den)) - return as_NLO + lmu = 2 * np.log(q / qref) + den = 1.0 + self.beta0[nf] * alpha_ref * lmu + alpha_LO = alpha_ref / den + alpha_NLO = alpha_LO * (1 - self.b1[nf] * alpha_LO * np.log(den)) + return alpha_NLO def set_thresholds_a_em(self): """Compute and store the couplings at thresholds""" - a_ref = self.alpha_em_ref / (4 * np.pi) - self.a_em_mt = self.a_em_nlo(self.theory["Qmt"], a_ref, self.qref, 5) - self.a_em_mb = self.a_em_nlo(self.theory["Qmb"], a_ref, self.qref, 5) - self.a_em_mc = self.a_em_nlo(self.theory["Qmc"], self.a_em_mb, self.theory["Qmb"], 4) + self.alpha_em_mt = self.alpha_em_nlo(self.theory["Qmt"], self.alpha_em_ref, self.qref, 5) + self.alpha_em_mb = self.alpha_em_nlo(self.theory["Qmb"], self.alpha_em_ref, self.qref, 5) + self.alpha_em_mc = self.alpha_em_nlo(self.theory["Qmc"], self.alpha_em_mb, self.theory["Qmb"], 4) self.thresh = {3: self.theory["Qmc"], 4: self.theory["Qmb"], 5: self.qref, 6:self.theory["Qmt"]} - self.a_thresh = {3: self.a_em_mc, 4:self.a_em_mb, 5:self.alpha_em_ref/(4*np.pi), 6:self.a_em_mt} + self.alpha_thresh = {3: self.alpha_em_mc, 4:self.alpha_em_mb, 5:self.alpha_em_ref, 6:self.alpha_em_mt} + + def set_betas(self): + """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" + nl = 3 + nc = 3 + eu2 = 4. / 9 + ed2 = 1. / 9 + self.beta0 = {} + self.b1 = {} + for nf in range(3, 6+1): + nu = nf // 2 + nd = nf - nu + self.beta0[nf] = ( -4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2)) ) / (4 * np.pi) + self.b1[nf] = -4.0 * ( nl + nc * (nu * eu2**2 + nd * ed2**2) ) / self.beta0[nf] / (4 * np.pi)**2 def compute_photon_array(self, xgrid, id): r""" From ee492baf0b0e41e6ebf46fc4d1872003ff111fa0 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Mon, 9 Jan 2023 20:13:27 +0100 Subject: [PATCH 048/204] Rename set_thresholds_a_em -> set_thresholds_alpha_em --- validphys2/src/validphys/photon/compute.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 33d19d9f2a..9049a951e4 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -32,7 +32,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.Qmb = np.inf if self.theory["MaxNfAs"] <= 3 : self.Qmc = np.inf - self.set_thresholds_a_em() + self.set_thresholds_alpha_em() # structure functions self.qcd_pdfs = [lhapdf.mkPDF(fiatlux_runcard["pdf_name"], id) for id in replicas_id] @@ -111,14 +111,24 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): alpha_NLO = alpha_LO * (1 - self.b1[nf] * alpha_LO * np.log(den)) return alpha_NLO - def set_thresholds_a_em(self): + def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" self.alpha_em_mt = self.alpha_em_nlo(self.theory["Qmt"], self.alpha_em_ref, self.qref, 5) self.alpha_em_mb = self.alpha_em_nlo(self.theory["Qmb"], self.alpha_em_ref, self.qref, 5) self.alpha_em_mc = self.alpha_em_nlo(self.theory["Qmc"], self.alpha_em_mb, self.theory["Qmb"], 4) - self.thresh = {3: self.theory["Qmc"], 4: self.theory["Qmb"], 5: self.qref, 6:self.theory["Qmt"]} - self.alpha_thresh = {3: self.alpha_em_mc, 4:self.alpha_em_mb, 5:self.alpha_em_ref, 6:self.alpha_em_mt} + self.thresh = { + 3: self.theory["Qmc"], + 4: self.theory["Qmb"], + 5: self.qref, + 6: self.theory["Qmt"] + } + self.alpha_thresh = { + 3: self.alpha_em_mc, + 4: self.alpha_em_mb, + 5: self.alpha_em_ref, + 6: self.alpha_em_mt + } def set_betas(self): """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" From bcf788a2691a769a428a5fb1f9fbf332a4f741ac Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 12 Jan 2023 16:22:39 +0100 Subject: [PATCH 049/204] Adding init for photon module --- validphys2/src/validphys/photon/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 validphys2/src/validphys/photon/__init__.py diff --git a/validphys2/src/validphys/photon/__init__.py b/validphys2/src/validphys/photon/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From d75ce5ecc8f64323f02133b7fc13f2b8a39a341b Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 12 Jan 2023 17:03:14 +0100 Subject: [PATCH 050/204] Modify convolution between eko and pdf --- validphys2/src/validphys/photon/compute.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 9049a951e4..0b5f58c2c0 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -20,7 +20,6 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.q_in2 = 100**2 # parameters for the alphaem running - self.set_betas() self.alpha_em_ref = self.theory["alphaqed"] self.qref = self.theory["Qref"] self.Qmc = self.theory["Qmc"] @@ -32,6 +31,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.Qmb = np.inf if self.theory["MaxNfAs"] <= 3 : self.Qmc = np.inf + self.set_betas() self.set_thresholds_alpha_em() # structure functions @@ -182,7 +182,8 @@ def compute_photon_array(self, xgrid, id): ] ) - for q2, elem in eko.items(): + q2 = eko.Q2grid[0] + with eko.operator(q2) as elem: pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) # error_final = np.einsum("ajbk,bk", elem.error, pdfs) @@ -192,7 +193,9 @@ def compute_photon_array(self, xgrid, id): return xgrid * photon_Q0 def produce_interpolators(self): - # TODO : pass the grid in a more clever way + # TODO : once that #1537 is merged do: + # fom evolven3fit_new.cli import XGRID + # self.xgrid = XGRID self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) self.photons_array = [self.compute_photon_array(self.xgrid, i) for i in range(len(self.replicas_id))] self.interpolator = [interp1d(self.xgrid, photon_array, fill_value=0.) for photon_array in self.photons_array] From ba6555777093add696473ff7e0ce4f82e5fa63c7 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Thu, 12 Jan 2023 18:30:00 +0100 Subject: [PATCH 051/204] Init tests for photon module --- .../validphys/photon/structure_functions.py | 3 ++- .../tests/test_structurefunctions.py | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 validphys2/src/validphys/tests/test_structurefunctions.py diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 57ee93576c..8a8331d8f3 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -68,6 +68,7 @@ def FxQ(self, x, Q): else : nf = 6 res = 0 + pdfs_values = self.pdfs.xfxQ(x, Q) for i in range(1, nf+1): - res += self.eq2[i-1] * (self.pdfs.xfxQ(x, Q)[i] + self.pdfs.xfxQ(x, Q)[-i]) + res += self.eq2[i-1] * (pdfs_values[i] + pdfs_values[-i]) return res diff --git a/validphys2/src/validphys/tests/test_structurefunctions.py b/validphys2/src/validphys/tests/test_structurefunctions.py new file mode 100644 index 0000000000..a33207aa14 --- /dev/null +++ b/validphys2/src/validphys/tests/test_structurefunctions.py @@ -0,0 +1,27 @@ +import pytest +from validphys.photon.structure_functions import F2LO +import numpy as np +import lhapdf + +def test_zero_pdfs(): + class fake_pdfs: + def xfxQ(self, x, Q): + res = {} + for i in range(1, 6+1): + res[i] = res[-i] = 0. + return res + + pdfs = fake_pdfs() + + fake_theory = { + "Qmc": 1.3, + "Qmb": 5. , + "Qmt": 172., + "MaxNfPdf": 5, + } + + f2lo = F2LO(pdfs, fake_theory) + + for x in np.geomspace(1e-4, 1., 10): + for Q in np.geomspace(10, 1000000, 10): + np.testing.assert_allclose(f2lo.FxQ(x, Q), 0.) From 1c4f4d53421fd680ed2509a8286d5ef420c0e6df Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 13 Jan 2023 10:10:37 +0100 Subject: [PATCH 052/204] Add fiatlux to conda-recipe --- conda-recipe/conda_build_config.yaml | 1 + conda-recipe/meta.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 327e710892..ff12f3c6d5 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -9,3 +9,4 @@ pin_run_as_build: gsl: x.x.x yaml-cpp: x.x.x libarchive: x.x + fiatlux: x.x.x diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index adc3c6328e..abf5fa46fa 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -53,6 +53,7 @@ requirements: - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 - curio >=1.0 - pineappl >=0.5.2 + - fiatlux test: requires: From a4a15540d5563d1af89fd9a4735e8228907a736b Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 13 Jan 2023 10:35:52 +0100 Subject: [PATCH 053/204] Fix previous commit --- conda-recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index abf5fa46fa..76675072c5 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -23,6 +23,7 @@ requirements: - apfel >=3 # see https://github.com/scarrazza/apfel - python - numpy + - fiatlux run: # Only install tensorflow on linux. Select eigen build - tensorflow >=2 *eigen* # [linux] From a21afe8f8d6791609bc967496565e2e9b3e63bf9 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Fri, 13 Jan 2023 11:11:41 +0100 Subject: [PATCH 054/204] Remove fiatlux from pin_run_as_build --- conda-recipe/conda_build_config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index ff12f3c6d5..327e710892 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -9,4 +9,3 @@ pin_run_as_build: gsl: x.x.x yaml-cpp: x.x.x libarchive: x.x - fiatlux: x.x.x From 25f99b235cf9a454c23ad0f35d6a2e0781bd4227 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 17 Jan 2023 10:33:54 +0100 Subject: [PATCH 055/204] Remove fiatlux from host --- conda-recipe/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 76675072c5..abf5fa46fa 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -23,7 +23,6 @@ requirements: - apfel >=3 # see https://github.com/scarrazza/apfel - python - numpy - - fiatlux run: # Only install tensorflow on linux. Select eigen build - tensorflow >=2 *eigen* # [linux] From 533719e5b10236eca4d7e13ed5560c58b159710a Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 17 Jan 2023 11:01:10 +0100 Subject: [PATCH 056/204] Remove fiatlux from conda recipe --- conda-recipe/meta.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index abf5fa46fa..adc3c6328e 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -53,7 +53,6 @@ requirements: - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 - curio >=1.0 - pineappl >=0.5.2 - - fiatlux test: requires: From 1ff8b4cf54ab141b7fcd024cdc9381491fa124ec Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 17 Jan 2023 14:08:17 +0100 Subject: [PATCH 057/204] Revert "Remove fiatlux from conda recipe" This reverts commit 533719e5b10236eca4d7e13ed5560c58b159710a. --- conda-recipe/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index adc3c6328e..abf5fa46fa 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -53,6 +53,7 @@ requirements: - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 - curio >=1.0 - pineappl >=0.5.2 + - fiatlux test: requires: From eaa5ae9d512bebfd06a35948bd5b46fe390ec098 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Tue, 17 Jan 2023 21:24:58 +0100 Subject: [PATCH 058/204] Modify eko.output -> eko.io --- validphys2/src/validphys/photon/compute.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 0b5f58c2c0..a12597e04a 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -2,9 +2,9 @@ import lhapdf import fiatlux import numpy as np -from eko.output.legacy import load_tar +from eko.io.legacy import load_tar from eko.interpolation import XGrid -from eko.output.manipulate import xgrid_reshape +from eko.io.manipulate import xgrid_reshape from .structure_functions import StructureFunction, F2LO from scipy.interpolate import interp1d from scipy.integrate import trapezoid @@ -113,15 +113,15 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" - self.alpha_em_mt = self.alpha_em_nlo(self.theory["Qmt"], self.alpha_em_ref, self.qref, 5) - self.alpha_em_mb = self.alpha_em_nlo(self.theory["Qmb"], self.alpha_em_ref, self.qref, 5) - self.alpha_em_mc = self.alpha_em_nlo(self.theory["Qmc"], self.alpha_em_mb, self.theory["Qmb"], 4) + self.alpha_em_mt = self.alpha_em_nlo(self.Qmt, self.alpha_em_ref, self.qref, 5) + self.alpha_em_mb = self.alpha_em_nlo(self.Qmb, self.alpha_em_ref, self.qref, 5) + self.alpha_em_mc = self.alpha_em_nlo(self.Qmc, self.alpha_em_mb, self.Qmb, 4) self.thresh = { - 3: self.theory["Qmc"], - 4: self.theory["Qmb"], + 3: self.Qmc, + 4: self.Qmb, 5: self.qref, - 6: self.theory["Qmt"] + 6: self.Qmt } self.alpha_thresh = { 3: self.alpha_em_mc, From e69dca3bb6c7637d8eecafb50a20325784541e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 18 Jan 2023 12:50:03 +0100 Subject: [PATCH 059/204] Move import of fiatlux and eko and install fiatlux only for linux --- conda-recipe/meta.yaml | 4 ++-- validphys2/src/validphys/photon/compute.py | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index abf5fa46fa..7b79fbc8e8 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -24,7 +24,7 @@ requirements: - python - numpy run: - # Only install tensorflow on linux. Select eigen build + # Only install tensorflow and fiatlux on linux. Select eigen build - tensorflow >=2 *eigen* # [linux] - psutil # to ensure n3fit affinity is with the right processors - hyperopt @@ -53,7 +53,7 @@ requirements: - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 - curio >=1.0 - pineappl >=0.5.2 - - fiatlux + - fiatlux *eigen* # [linux] test: requires: diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index a12597e04a..791eddeaaa 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -1,10 +1,9 @@ """Script that calls fiatlux to add the photon PDF.""" import lhapdf -import fiatlux + import numpy as np -from eko.io.legacy import load_tar + from eko.interpolation import XGrid -from eko.io.manipulate import xgrid_reshape from .structure_functions import StructureFunction, F2LO from scipy.interpolate import interp1d from scipy.integrate import trapezoid @@ -43,6 +42,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2lo = [F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] # set fiatlux + import fiatlux ff = open('fiatlux_runcard.yml', 'w+') yaml.dump(self.fiatlux_runcard, ff) self.lux = [fiatlux.FiatLux('fiatlux_runcard.yml') for i in range(len(replicas_id))] @@ -163,9 +163,12 @@ def compute_photon_array(self, xgrid, id): print("computing grid point", i+1, "/", len(xgrid)) photon_100GeV[i] = self.lux[id].EvaluatePhoton(x, self.q_in2).total / x + from eko.io.legacy import load_tar eko=load_tar(self.fiatlux_runcard['path_to_eko']) + # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape + from eko.io.manipulate import xgrid_reshapes xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) From 2bb62638900dabc3320acfd3d69d7848afaa8a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 18 Jan 2023 14:06:10 +0100 Subject: [PATCH 060/204] Fix conda recipe --- conda-recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 7b79fbc8e8..e966cabd28 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -53,7 +53,7 @@ requirements: - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 - curio >=1.0 - pineappl >=0.5.2 - - fiatlux *eigen* # [linux] + - fiatlux # [linux] test: requires: From d2f7f620c493ae38ae088affd8b427f92af36535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 18 Jan 2023 14:36:47 +0100 Subject: [PATCH 061/204] Move exo.interpolation --- validphys2/src/validphys/photon/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 791eddeaaa..4d9fecbcbe 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -3,7 +3,6 @@ import numpy as np -from eko.interpolation import XGrid from .structure_functions import StructureFunction, F2LO from scipy.interpolate import interp1d from scipy.integrate import trapezoid @@ -169,6 +168,7 @@ def compute_photon_array(self, xgrid, id): # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape from eko.io.manipulate import xgrid_reshapes + from eko.interpolation import XGrid xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) From 62f63d96cc2f8804934ba0cc9c44d9e516a1477b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 18 Jan 2023 17:08:02 +0100 Subject: [PATCH 062/204] Fix typo in xgrid_reshape --- validphys2/src/validphys/photon/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 4d9fecbcbe..67c7fc7363 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -167,7 +167,7 @@ def compute_photon_array(self, xgrid, id): # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape - from eko.io.manipulate import xgrid_reshapes + from eko.io.manipulate import xgrid_reshape from eko.interpolation import XGrid xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) From 6919be36ab711403e6f2ad1c27f4f7cce1d9dc90 Mon Sep 17 00:00:00 2001 From: niclaurenti Date: Wed, 18 Jan 2023 17:34:44 +0100 Subject: [PATCH 063/204] Remove comment --- n3fit/src/n3fit/layers/rotations.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 511fd35717..f98720e220 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -108,8 +108,6 @@ def __init__(self, photons, **kwargs): def register_photon(self, xgrid): """Compute the photon array and set the layer to be rebuilt""" - # TODO: maybe add here some caching mechanism so that the photon doesn't get - # recomputed if the grid hasn't changed! if self._photons_generator is not None: self._pdf_ph = self._photons_generator.compute(xgrid) self.built = False From b4efda9af171a884ade16b2ec3cc8e658829316b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 23 Jan 2023 18:09:20 +0100 Subject: [PATCH 064/204] Implement additional errors --- validphys2/src/validphys/photon/compute.py | 70 +++++++++++++++------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 67c7fc7363..20ce50659d 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -6,6 +6,7 @@ from .structure_functions import StructureFunction, F2LO from scipy.interpolate import interp1d from scipy.integrate import trapezoid +from pathlib import Path import yaml from os import remove @@ -52,6 +53,13 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) self.lux[i].InsertInelasticSplitQ([4.18, 1e100]) self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) + + # TODO : once that #1537 is merged do: + # from evolven3fit_new.cli import XGRID + # self.xgrid = XGRID + self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) + + self.generate_error_matrix() self.produce_interpolators() @@ -112,9 +120,9 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" - self.alpha_em_mt = self.alpha_em_nlo(self.Qmt, self.alpha_em_ref, self.qref, 5) - self.alpha_em_mb = self.alpha_em_nlo(self.Qmb, self.alpha_em_ref, self.qref, 5) - self.alpha_em_mc = self.alpha_em_nlo(self.Qmc, self.alpha_em_mb, self.Qmb, 4) + self.alpha_em_Qmt = self.alpha_em_nlo(self.Qmt, self.alpha_em_ref, self.qref, 5) + self.alpha_em_Qmb = self.alpha_em_nlo(self.Qmb, self.alpha_em_ref, self.qref, 5) + self.alpha_em_Qmc = self.alpha_em_nlo(self.Qmc, self.alpha_em_Qmb, self.Qmb, 4) self.thresh = { 3: self.Qmc, @@ -123,10 +131,10 @@ def set_thresholds_alpha_em(self): 6: self.Qmt } self.alpha_thresh = { - 3: self.alpha_em_mc, - 4: self.alpha_em_mb, + 3: self.alpha_em_Qmc, + 4: self.alpha_em_Qmb, 5: self.alpha_em_ref, - 6: self.alpha_em_mt + 6: self.alpha_em_Qmt } def set_betas(self): @@ -143,7 +151,7 @@ def set_betas(self): self.beta0[nf] = ( -4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2)) ) / (4 * np.pi) self.b1[nf] = -4.0 * ( nl + nc * (nu * eu2**2 + nd * ed2**2) ) / self.beta0[nf] / (4 * np.pi)**2 - def compute_photon_array(self, xgrid, id): + def compute_photon_array(self, id): r""" Compute the photon PDF for every point in the grid xgrid. @@ -157,21 +165,23 @@ def compute_photon_array(self, xgrid, id): compute_photon_array: numpy.array photon PDF at the scale 1 GeV """ - photon_100GeV = np.zeros(len(xgrid)) - for i, x in enumerate(xgrid): - print("computing grid point", i+1, "/", len(xgrid)) - photon_100GeV[i] = self.lux[id].EvaluatePhoton(x, self.q_in2).total / x + photon_100GeV = np.zeros(len(self.xgrid)) + for i, x in enumerate(self.xgrid): + print("computing grid point", i+1, "/", len(self.xgrid)) + # photon_100GeV[i] = self.lux[id].EvaluatePhoton(x, self.q_in2).total / x + photon_100GeV[i] = np.exp(-x) + photon_100GeV += self.generate_errors() - from eko.io.legacy import load_tar - eko=load_tar(self.fiatlux_runcard['path_to_eko']) + from eko.io import EKO + eko=EKO.read(Path(self.fiatlux_runcard['path_to_eko'])) # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape from eko.io.manipulate import xgrid_reshape from eko.interpolation import XGrid - xgrid_reshape(eko, targetgrid = XGrid(xgrid), inputgrid = XGrid(xgrid)) + xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) - pdfs = np.zeros((len(eko.rotations.inputpids), len(xgrid))) + pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) for j, pid in enumerate(eko.rotations.inputpids): if pid == 22 : pdfs[j] = photon_100GeV @@ -181,7 +191,7 @@ def compute_photon_array(self, xgrid, id): pdfs[j] = np.array( [ self.qcd_pdfs[id].xfxQ2(pid, x, self.q_in2) / x - for x in xgrid + for x in self.xgrid ] ) @@ -193,14 +203,10 @@ def compute_photon_array(self, xgrid, id): photon_Q0 = pdf_final[ph_id] # we want x * gamma(x) - return xgrid * photon_Q0 + return self.xgrid * photon_Q0 def produce_interpolators(self): - # TODO : once that #1537 is merged do: - # fom evolven3fit_new.cli import XGRID - # self.xgrid = XGRID - self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) - self.photons_array = [self.compute_photon_array(self.xgrid, i) for i in range(len(self.replicas_id))] + self.photons_array = [self.compute_photon_array(i) for i in range(len(self.replicas_id))] self.interpolator = [interp1d(self.xgrid, photon_array, fill_value=0.) for photon_array in self.photons_array] def compute(self, xgrid): @@ -224,4 +230,22 @@ def compute(self, xgrid): def integrate(self): """Compute the integral of the photon on the x range""" - return [trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id))] \ No newline at end of file + return [trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id))] + + def generate_error_matrix(self): + if not self.fiatlux_runcard["additional_errors"] : + self.error_matrix = None + extra_set = lhapdf.mkPDFs("LUXqed17_plus_PDF4LHC15_nnlo_100") + self.error_matrix = np.array( + [ + [(extra_set[i].xfxQ2(22, x, self.q_in2) - extra_set[0].xfxQ2(22, x, self.q_in2)) / x for x in self.xgrid] + for i in range(101, 107+1) + ] + ).T + + def generate_errors(self): + if self.error_matrix is None : + return np.zeros_like(self.xgrid) + u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) + errors = u @ (s * np.random.normal(size=7)) + return errors From e6934f29be7af02ff97efbf766d01cbee20001d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 24 Jan 2023 15:36:47 +0100 Subject: [PATCH 065/204] Upgrade eko to 0.12 --- validphys2/src/validphys/photon/compute.py | 67 +++++++++++----------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 20ce50659d..59fe8f5daf 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -164,41 +164,43 @@ def compute_photon_array(self, id): ------- compute_photon_array: numpy.array photon PDF at the scale 1 GeV - """ + """ + # Compute photon PDF photon_100GeV = np.zeros(len(self.xgrid)) for i, x in enumerate(self.xgrid): print("computing grid point", i+1, "/", len(self.xgrid)) - # photon_100GeV[i] = self.lux[id].EvaluatePhoton(x, self.q_in2).total / x - photon_100GeV[i] = np.exp(-x) + photon_100GeV[i] = self.lux[id].EvaluatePhoton(x, self.q_in2).total / x photon_100GeV += self.generate_errors() + # Load eko and reshape it from eko.io import EKO - eko=EKO.read(Path(self.fiatlux_runcard['path_to_eko'])) - - # If we make sure that the grid of the precomputed EKO is the same of - # self.xgrid then we don't need to reshape - from eko.io.manipulate import xgrid_reshape - from eko.interpolation import XGrid - xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) - - pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) - for j, pid in enumerate(eko.rotations.inputpids): - if pid == 22 : - pdfs[j] = photon_100GeV - ph_id = j - if not self.qcd_pdfs[id].hasFlavor(pid): - continue - pdfs[j] = np.array( - [ - self.qcd_pdfs[id].xfxQ2(pid, x, self.q_in2) / x - for x in self.xgrid - ] - ) + with EKO.edit(Path(self.fiatlux_runcard['path_to_eko'])) as eko: + # If we make sure that the grid of the precomputed EKO is the same of + # self.xgrid then we don't need to reshape + from eko.io.manipulate import xgrid_reshape + from eko.interpolation import XGrid + xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) + + # construct PDFs + pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) + for j, pid in enumerate(eko.rotations.inputpids): + if pid == 22 : + pdfs[j] = photon_100GeV + ph_id = j + if not self.qcd_pdfs[id].hasFlavor(pid): + continue + pdfs[j] = np.array( + [ + self.qcd_pdfs[id].xfxQ2(pid, x, self.q_in2) / x + for x in self.xgrid + ] + ) - q2 = eko.Q2grid[0] - with eko.operator(q2) as elem: - pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) - # error_final = np.einsum("ajbk,bk", elem.error, pdfs) + # Apply EKO to PDFs + q2 = eko.mu2grid[0] + with eko.operator(q2) as elem: + pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) + # error_final = np.einsum("ajbk,bk", elem.error, pdfs) photon_Q0 = pdf_final[ph_id] @@ -207,7 +209,7 @@ def compute_photon_array(self, id): def produce_interpolators(self): self.photons_array = [self.compute_photon_array(i) for i in range(len(self.replicas_id))] - self.interpolator = [interp1d(self.xgrid, photon_array, fill_value=0.) for photon_array in self.photons_array] + self.interpolator = [interp1d(self.xgrid, photon_array, fill_value=0., kind='cubic') for photon_array in self.photons_array] def compute(self, xgrid): """ @@ -236,12 +238,13 @@ def generate_error_matrix(self): if not self.fiatlux_runcard["additional_errors"] : self.error_matrix = None extra_set = lhapdf.mkPDFs("LUXqed17_plus_PDF4LHC15_nnlo_100") + # TODO : maybe doing it on f instead of xf isn't the same self.error_matrix = np.array( [ - [(extra_set[i].xfxQ2(22, x, self.q_in2) - extra_set[0].xfxQ2(22, x, self.q_in2)) / x for x in self.xgrid] - for i in range(101, 107+1) + [(extra_set[i].xfxQ2(22, x, self.q_in2) - extra_set[0].xfxQ2(22, x, self.q_in2)) / x for i in range(101, 107+1)] + for x in self.xgrid ] - ).T + ) # first index must be x, while second one must be replica index def generate_errors(self): if self.error_matrix is None : From 9efaf707b8453d45b737b106c394848153c9fbc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 13 Feb 2023 10:34:17 +0100 Subject: [PATCH 066/204] Add tests --- validphys2/src/validphys/photon/structure_functions.py | 9 ++++----- validphys2/src/validphys/tests/photon/__init__.py | 0 .../tests/{ => photon}/test_structurefunctions.py | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 validphys2/src/validphys/tests/photon/__init__.py rename validphys2/src/validphys/tests/{ => photon}/test_structurefunctions.py (99%) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 8a8331d8f3..ac335b2521 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -1,7 +1,7 @@ from pathlib import Path import pineappl import numpy as np -from scipy.interpolate import RectBivariateSpline +from scipy.interpolate import interp2d class StructureFunction : def __init__(self, path_to_fktable, pdfs): @@ -19,12 +19,11 @@ def produce_interpolator(self): # here we require that the (x,Q2) couples that we passed # to pinefarm is a rectangular matrix - grid2D = predictions.reshape(len(x),len(q2)) - # RectBivariateSpline is faster than interp2d - self.interpolator = RectBivariateSpline(x, q2, grid2D) + grid2D = predictions.reshape(len(x),len(q2)).T + self.interpolator = interp2d(x, q2, grid2D, kind='cubic', fill_value=0) def FxQ(self, x, Q): - return self.interpolator(x, Q**2)[0,0] + return self.interpolator(x, Q**2)[0] class F2LO : def __init__(self, pdfs, theory): diff --git a/validphys2/src/validphys/tests/photon/__init__.py b/validphys2/src/validphys/tests/photon/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/validphys2/src/validphys/tests/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py similarity index 99% rename from validphys2/src/validphys/tests/test_structurefunctions.py rename to validphys2/src/validphys/tests/photon/test_structurefunctions.py index a33207aa14..777ec8c050 100644 --- a/validphys2/src/validphys/tests/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -25,3 +25,4 @@ def xfxQ(self, x, Q): for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(f2lo.FxQ(x, Q), 0.) + From dcbf977adb5fd05f13fc54a404e781fc24b753f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 13 Feb 2023 15:42:05 +0100 Subject: [PATCH 067/204] Add test for AddPhoton --- n3fit/src/n3fit/tests/test_layers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/n3fit/src/n3fit/tests/test_layers.py b/n3fit/src/n3fit/tests/test_layers.py index b1d662450f..a42203d34f 100644 --- a/n3fit/src/n3fit/tests/test_layers.py +++ b/n3fit/src/n3fit/tests/test_layers.py @@ -259,3 +259,22 @@ def test_mask(): masker = layers.Mask(bool_mask=np_mask, c=rn_val) ret = masker(fi) np.testing.assert_allclose(ret, masked_fi * rn_val, rtol=1e-5) + +def test_addphoton_init(): + """Test AddPhoton class.""" + addphoton = layers.AddPhoton(photons=None) + np.testing.assert_equal(addphoton._photons_generator, None) + addphoton = layers.AddPhoton(photons=1234) + np.testing.assert_equal(addphoton._photons_generator, 1234) + np.testing.assert_equal(addphoton._pdf_ph, None) + +class fakePhoton(): + def compute(self, xgrid): + return [np.exp(-xgrid)] + +def test_compute_photon(): + photon = fakePhoton() + addphoton = layers.AddPhoton(photons=photon) + xgrid = np.geomspace(1e-4, 1., 10) + addphoton.register_photon(xgrid) + np.testing.assert_allclose(addphoton._pdf_ph, [np.exp(-xgrid)]) From 11120e5accaa634a685740eb912f0a92a838637e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 14 Feb 2023 13:01:27 +0100 Subject: [PATCH 068/204] Start testing --- validphys2/src/validphys/photon/compute.py | 11 +++++++---- .../validphys/tests/photon/test_structurefunctions.py | 3 ++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 59fe8f5daf..8c9b64013e 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -3,7 +3,8 @@ import numpy as np -from .structure_functions import StructureFunction, F2LO +# from .structure_functions import StructureFunction, F2LO +from . import structure_functions as sf from scipy.interpolate import interp1d from scipy.integrate import trapezoid from pathlib import Path @@ -37,9 +38,9 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.qcd_pdfs = [lhapdf.mkPDF(fiatlux_runcard["pdf_name"], id) for id in replicas_id] path_to_F2 = fiatlux_runcard["path_to_F2"] path_to_FL = fiatlux_runcard["path_to_FL"] - f2 = [StructureFunction(path_to_F2, pdfs) for pdfs in self.qcd_pdfs] - fl = [StructureFunction(path_to_FL, pdfs) for pdfs in self.qcd_pdfs] - f2lo = [F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] + f2 = [sf.StructureFunction(path_to_F2, pdfs) for pdfs in self.qcd_pdfs] + fl = [sf.StructureFunction(path_to_FL, pdfs) for pdfs in self.qcd_pdfs] + f2lo = [sf.F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] # set fiatlux import fiatlux @@ -120,6 +121,8 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" + # TODO : this is only for qref=91.2 + # Generalize it self.alpha_em_Qmt = self.alpha_em_nlo(self.Qmt, self.alpha_em_ref, self.qref, 5) self.alpha_em_Qmb = self.alpha_em_nlo(self.Qmb, self.alpha_em_ref, self.qref, 5) self.alpha_em_Qmc = self.alpha_em_nlo(self.Qmc, self.alpha_em_Qmb, self.Qmb, 4) diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index 777ec8c050..1fb4904318 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -1,7 +1,6 @@ import pytest from validphys.photon.structure_functions import F2LO import numpy as np -import lhapdf def test_zero_pdfs(): class fake_pdfs: @@ -22,6 +21,8 @@ def xfxQ(self, x, Q): f2lo = F2LO(pdfs, fake_theory) + np.testing.assert_equal(f2lo.Qmt, np.inf) + for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(f2lo.FxQ(x, Q), 0.) From 92bf8c666f349aac7382ef63c71940e4666bda30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 15 Feb 2023 09:46:37 +0100 Subject: [PATCH 069/204] Remove commented line --- validphys2/src/validphys/photon/compute.py | 1 - 1 file changed, 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 8c9b64013e..b124887c02 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -3,7 +3,6 @@ import numpy as np -# from .structure_functions import StructureFunction, F2LO from . import structure_functions as sf from scipy.interpolate import interp1d from scipy.integrate import trapezoid From 37b65539b7952d9e22bd05c513712bc76542e69c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 15 Feb 2023 12:06:17 +0100 Subject: [PATCH 070/204] Generalize function set_thresholds_alpha_em --- validphys2/src/validphys/photon/compute.py | 36 +++---- .../validphys/tests/photon/test_compute.py | 95 +++++++++++++++++++ 2 files changed, 114 insertions(+), 17 deletions(-) create mode 100644 validphys2/src/validphys/tests/photon/test_compute.py diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index b124887c02..6fd49a9ffb 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -120,24 +120,26 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" - # TODO : this is only for qref=91.2 - # Generalize it - self.alpha_em_Qmt = self.alpha_em_nlo(self.Qmt, self.alpha_em_ref, self.qref, 5) - self.alpha_em_Qmb = self.alpha_em_nlo(self.Qmb, self.alpha_em_ref, self.qref, 5) - self.alpha_em_Qmc = self.alpha_em_nlo(self.Qmc, self.alpha_em_Qmb, self.Qmb, 4) + thresh_list = [self.Qmc, self.Qmb, self.Qmt] + if self.qref < self.Qmc : + nfref = 3 + elif self.qref < self.Qmb : + nfref = 4 + elif self.qref < self.Qmt : + nfref = 5 + else : + nfref = 6 + thresh_list.insert(nfref - 3, self.qref) + + self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} - self.thresh = { - 3: self.Qmc, - 4: self.Qmb, - 5: self.qref, - 6: self.Qmt - } - self.alpha_thresh = { - 3: self.alpha_em_Qmc, - 4: self.alpha_em_Qmb, - 5: self.alpha_em_ref, - 6: self.alpha_em_Qmt - } + self.alpha_thresh = {nfref: self.alpha_em_ref} + + for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): + self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1) + + for nf in reversed(range(3, nfref)): + self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1) def set_betas(self): """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py new file mode 100644 index 0000000000..2be7c9ae7f --- /dev/null +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -0,0 +1,95 @@ +import pytest +from validphys.photon.compute import Photon +from validphys.photon import structure_functions +import lhapdf +import fiatlux +import numpy as np +from collections import namedtuple + +class faketheory(): + def get_description(self): + return { + "alphaqed": 0.01, + "Qref": 91.2, + "Qmc": 1.3, + "Qmb": 4.92, + "Qmt": 173., + "MaxNfAs": 5, + + } + +fiatlux_runcard = { + "pdf_name": "no_pdf", + "path_to_F2": "/", + "path_to_FL": "/", + "additional_errors": False, +} + +photon = namedtuple('photon', ['total', 'elastic', 'inelastic']) + +class fakeFiatlux(): + def __init__(self, runcard): + self.runcard = runcard + self.alphaem = None + self.qref = None + self.trash1 = None + self.trash2 = None + self.f2 = None + self.fl = None + self.f2lo = None + self.res = photon(0, 0, 0) + + def PlugAlphaQED(self, alphaem, qref): + self.alphaem = alphaem + self.qref = qref + + def InsertInelasticSplitQ(self, args): + self.trash1 = args[0] + self.trash2 = args[1] + + def PlugStructureFunctions(self, f2, fl, f2lo): + self.f2 = f2 + self.fl = fl + self.f2lo = f2lo + + def EvaluatePhoton(self, x, q): + return self.res + +class fakeStructureFunction(): + def __init__(self, path, pdfs): + self.path = path + self.pdfs = pdfs + + def FxQ(self): + return 0 + +class fakeF2LO(): + def __init__(self, pdfs, theory): + self.pdfs = pdfs + self.theory = theory + + def FxQ(self): + return 0 + + +def test_init(monkeypatch): + monkeypatch.setattr(structure_functions, "StructureFunction", fakeStructureFunction) + monkeypatch.setattr(structure_functions, "F2LO", fakeF2LO) + monkeypatch.setattr(lhapdf, "mkPDF", lambda *args: 1) + monkeypatch.setattr(fiatlux, "FiatLux", fakeFiatlux) + monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) + photon = Photon(faketheory(), fiatlux_runcard, [1,2,3]) + np.testing.assert_equal(photon.replicas_id, [1,2,3]) + np.testing.assert_equal(photon.Qmt, np.inf) + np.testing.assert_almost_equal(photon.Qmb, 4.92) + np.testing.assert_almost_equal(photon.Qmc, 1.3) + np.testing.assert_almost_equal(photon.thresh[5], 91.2) + np.testing.assert_almost_equal(photon.thresh[4], 4.92) + np.testing.assert_almost_equal(photon.thresh[3], 1.3) + np.testing.assert_almost_equal(photon.alpha_thresh[5], 0.01) + np.testing.assert_almost_equal(photon.alpha_thresh[4], photon.alpha_em_nlo(4.92, 0.01, 91.2, 5)) + np.testing.assert_almost_equal(photon.alpha_thresh[3], photon.alpha_em_nlo(1.3, photon.alpha_thresh[4], 4.92, 4)) + np.testing.assert_equal(len((photon.alpha_thresh)), 3) + np.testing.assert_equal(len((photon.thresh)), 3) + + From 8591cf7b3f3d8c8989de1e5dfc0cb4f91a672738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 15 Feb 2023 12:07:30 +0100 Subject: [PATCH 071/204] Remove parenthesis --- validphys2/src/validphys/tests/photon/test_compute.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 2be7c9ae7f..2af723dc2d 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -89,7 +89,7 @@ def test_init(monkeypatch): np.testing.assert_almost_equal(photon.alpha_thresh[5], 0.01) np.testing.assert_almost_equal(photon.alpha_thresh[4], photon.alpha_em_nlo(4.92, 0.01, 91.2, 5)) np.testing.assert_almost_equal(photon.alpha_thresh[3], photon.alpha_em_nlo(1.3, photon.alpha_thresh[4], 4.92, 4)) - np.testing.assert_equal(len((photon.alpha_thresh)), 3) - np.testing.assert_equal(len((photon.thresh)), 3) + np.testing.assert_equal(len(photon.alpha_thresh), 3) + np.testing.assert_equal(len(photon.thresh), 3) From 6e17e6334bcc86fb012edaed168b7e2b24b749e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 15 Feb 2023 14:39:26 +0100 Subject: [PATCH 072/204] Fix bug in generate_error_matrix --- validphys2/src/validphys/photon/compute.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 6fd49a9ffb..6281a09ec0 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -59,7 +59,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # self.xgrid = XGRID self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) - self.generate_error_matrix() + self.error_matrix = self.generate_error_matrix() self.produce_interpolators() @@ -240,10 +240,10 @@ def integrate(self): def generate_error_matrix(self): if not self.fiatlux_runcard["additional_errors"] : - self.error_matrix = None + return None extra_set = lhapdf.mkPDFs("LUXqed17_plus_PDF4LHC15_nnlo_100") # TODO : maybe doing it on f instead of xf isn't the same - self.error_matrix = np.array( + return np.array( [ [(extra_set[i].xfxQ2(22, x, self.q_in2) - extra_set[0].xfxQ2(22, x, self.q_in2)) / x for i in range(101, 107+1)] for x in self.xgrid From a8c80e1336ea1d3dbf7efa9685171a83e56e1c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 15 Feb 2023 18:31:11 +0100 Subject: [PATCH 073/204] add docstrings --- validphys2/src/validphys/photon/compute.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 6281a09ec0..cf43509096 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -235,10 +235,11 @@ def compute(self, xgrid): ] def integrate(self): - """Compute the integral of the photon on the x range""" + """Compute the integral of the photon on the x range.""" return [trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id))] def generate_error_matrix(self): + """generate error matrix to be used for the additional errors.""" if not self.fiatlux_runcard["additional_errors"] : return None extra_set = lhapdf.mkPDFs("LUXqed17_plus_PDF4LHC15_nnlo_100") @@ -251,6 +252,7 @@ def generate_error_matrix(self): ) # first index must be x, while second one must be replica index def generate_errors(self): + """generate LUX additional errors.""" if self.error_matrix is None : return np.zeros_like(self.xgrid) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) From e712984108fbdbac0224d89114386c7a2f5d12b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 22 Feb 2023 15:10:30 +0100 Subject: [PATCH 074/204] Fix additional errors --- validphys2/src/validphys/photon/compute.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index cf43509096..6ff2ff5773 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -170,11 +170,9 @@ def compute_photon_array(self, id): photon PDF at the scale 1 GeV """ # Compute photon PDF - photon_100GeV = np.zeros(len(self.xgrid)) - for i, x in enumerate(self.xgrid): - print("computing grid point", i+1, "/", len(self.xgrid)) - photon_100GeV[i] = self.lux[id].EvaluatePhoton(x, self.q_in2).total / x + photon_100GeV = np.array([self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid]) photon_100GeV += self.generate_errors() + photon_100GeV /= self.xgrid # Load eko and reshape it from eko.io import EKO @@ -243,10 +241,9 @@ def generate_error_matrix(self): if not self.fiatlux_runcard["additional_errors"] : return None extra_set = lhapdf.mkPDFs("LUXqed17_plus_PDF4LHC15_nnlo_100") - # TODO : maybe doing it on f instead of xf isn't the same return np.array( [ - [(extra_set[i].xfxQ2(22, x, self.q_in2) - extra_set[0].xfxQ2(22, x, self.q_in2)) / x for i in range(101, 107+1)] + [(extra_set[i].xfxQ2(22, x, self.q_in2) - extra_set[0].xfxQ2(22, x, self.q_in2)) for i in range(101, 107+1)] for x in self.xgrid ] ) # first index must be x, while second one must be replica index From 42bc035e74afdfb2e78b49b4ef26704513bf8661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 23 Feb 2023 17:39:59 +0100 Subject: [PATCH 075/204] Revert "Merge branch 'master' into add-photon" This reverts commit 5c6156912598bd23a0d5d3b57d4186b48d2eead3, reversing changes made to 1f3ec4189aed9746da9ed42bdca9d2c521cc04ed. --- .github/workflows/fitbot.yml | 4 +- .github/workflows/tests.yml | 5 +- .gitignore | 3 - conda-recipe/conda_build_config.yaml | 2 +- conda-recipe/meta.yaml | 7 +- doc/sphinx/source/releases.rst | 3 - doc/sphinx/source/tutorials/pseudodata.rst | 94 ++-- n3fit/runcards/examples/developing.yml | 1 - n3fit/setup.py | 1 - n3fit/src/evolven3fit_new/__init__.py | 0 n3fit/src/evolven3fit_new/cli.py | 244 --------- n3fit/src/evolven3fit_new/eko_utils.py | 83 ---- n3fit/src/evolven3fit_new/evolve.py | 220 -------- n3fit/src/evolven3fit_new/utils.py | 201 -------- .../src/n3fit/hyper_optimization/penalties.py | 16 +- n3fit/src/n3fit/io/writer.py | 6 +- n3fit/src/n3fit/scripts/evolven3fit_new.py | 159 ------ n3fit/src/n3fit/scripts/n3fit_exec.py | 4 +- n3fit/src/n3fit/tests/regressions/filter.yml | 88 ---- n3fit/src/n3fit/tests/test_evolven3fit.py | 70 --- n3fit/src/n3fit/tests/test_penalties.py | 39 -- validphys2/src/validphys/commondataparser.py | 107 +--- validphys2/src/validphys/commondatawriter.py | 83 ---- validphys2/src/validphys/core.py | 79 +-- validphys2/src/validphys/coredata.py | 45 +- validphys2/src/validphys/dataplots.py | 10 +- validphys2/src/validphys/filters.py | 207 +++----- validphys2/src/validphys/fitdata.py | 6 +- validphys2/src/validphys/mc_gen.py | 1 - validphys2/src/validphys/n3fit_data.py | 9 +- validphys2/src/validphys/paramfits/dataops.py | 2 +- validphys2/src/validphys/pdfbases.py | 35 +- validphys2/src/validphys/photon/compute.py | 7 +- validphys2/src/validphys/plotoptions/core.py | 15 +- validphys2/src/validphys/pseudodata.py | 139 +----- validphys2/src/validphys/tests/conftest.py | 3 - .../test_filter_rebuild_closure_data.csv | 470 +++++++++--------- .../src/validphys/tests/test_fitdata.py | 23 +- validphys2/src/validphys/tests/test_loader.py | 27 +- .../src/validphys/tests/test_pseudodata.py | 33 +- .../src/validphys/tests/test_weights.py | 12 +- .../src/validphys/theorycovariance/tests.py | 2 +- 42 files changed, 465 insertions(+), 2100 deletions(-) delete mode 100644 n3fit/src/evolven3fit_new/__init__.py delete mode 100644 n3fit/src/evolven3fit_new/cli.py delete mode 100644 n3fit/src/evolven3fit_new/eko_utils.py delete mode 100644 n3fit/src/evolven3fit_new/evolve.py delete mode 100644 n3fit/src/evolven3fit_new/utils.py delete mode 100644 n3fit/src/n3fit/scripts/evolven3fit_new.py delete mode 100644 n3fit/src/n3fit/tests/regressions/filter.yml delete mode 100644 n3fit/src/n3fit/tests/test_evolven3fit.py delete mode 100644 n3fit/src/n3fit/tests/test_penalties.py delete mode 100644 validphys2/src/validphys/commondatawriter.py diff --git a/.github/workflows/fitbot.yml b/.github/workflows/fitbot.yml index e859b520be..17f8162877 100644 --- a/.github/workflows/fitbot.yml +++ b/.github/workflows/fitbot.yml @@ -11,7 +11,7 @@ env: N3FIT_MAXNREP: 20 # total number of replicas to fit POSTFIT_NREP: 16 # requested replicas for postfit REFERENCE_SET: NNBOT-90875c07e-2022-11-30 # reference set for vp - CONDA_PY: 310 + CONDA_PY: 39 jobs: build: @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.10"] + python-version: [3.9] runs-on: ${{ matrix.os }} env: NETRC_FILE: ${{ secrets.NETRC_FILE }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ca417bb12c..e12b15d9d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,13 +12,12 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: ["3.9", "3.10"] + python-version: [3.8, 3.9] include: - os: ubuntu-latest CONDA_OS: linux-64 - os: macos-latest CONDA_OS: osx-64 - fail-fast: false runs-on: ${{ matrix.os }} env: NETRC_FILE: ${{ secrets.NETRC_FILE }} @@ -68,7 +67,7 @@ jobs: $CONDA_PREFIX/conda-bld/${{matrix.CONDA_OS}}/*.tar.bz2 \ dummy@packages.nnpdf.science:~/packages/conda/${{matrix.CONDA_OS}} - name: Build and upload sphinx documentation to NNPDF server - if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master' && matrix.python-version == '3.9' + if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master' shell: bash -l {0} run: | KEY=$( mktemp ) diff --git a/.gitignore b/.gitignore index 0fbe0bcd81..14960be3b1 100644 --- a/.gitignore +++ b/.gitignore @@ -378,9 +378,6 @@ ENV/ env.bak/ venv.bak/ -conda-build -conda_bld/ - # Spyder project settings .spyderproject .spyproject diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index a15ac5fdef..327e710892 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,7 +1,7 @@ #For some reason the resolver decides to use an old version of numpy #without this numpy: - - 1.21 + - 1.19 pin_run_as_build: lhapdf: x.x.x diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index d31a3ba3d5..e966cabd28 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -50,14 +50,11 @@ requirements: - recommonmark - sphinx_rtd_theme >0.5 - sphinxcontrib-bibtex + - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 - curio >=1.0 - - pineappl >=0.5.8 - - eko ==0.10.2 - - banana-hep ==0.6.6 - - lz4 >=3.1.10 # see https://github.com/conda-forge/eko-feedstock/issues/10 + - pineappl >=0.5.2 - fiatlux # [linux] - test: requires: - hypothesis diff --git a/doc/sphinx/source/releases.rst b/doc/sphinx/source/releases.rst index f451f9b843..d4b6b7f957 100644 --- a/doc/sphinx/source/releases.rst +++ b/doc/sphinx/source/releases.rst @@ -21,9 +21,6 @@ The code is tagged to contextualize the versioning, mark significant developments and to mark versions used to produce main results. The significant releases since the code was made public are: -`Version 4.0.6 `_ - The last version that uses C++ objects in n3fit and validphys, and apfel for - PDF evolution. `Version 4.0.5 `_ The last version to support legacy genetic algorithms fits based on C++. `Version 4.0.4 `_ diff --git a/doc/sphinx/source/tutorials/pseudodata.rst b/doc/sphinx/source/tutorials/pseudodata.rst index 9bfe25eab8..da35664a69 100644 --- a/doc/sphinx/source/tutorials/pseudodata.rst +++ b/doc/sphinx/source/tutorials/pseudodata.rst @@ -2,58 +2,9 @@ Obtaining the pseudodata used by an ``n3fit`` fit ================================================= -Since version 4.0.7 the Monte Carlo data replicas are saved to disk by default as the fit is performed. -This can be deactivated by setting the ``savepseudodata`` flag to ``False`` under the ``fitting`` namespace in the fit runcard: - -.. code-block :: yaml - - fitting: - savepseudodata: False - -If the ``savepseudodata`` flag is not set to ``False``, the training and validation splits to disk -under files named ``datacuts_theory_fitting_training_pseudodata.csv`` and similarly for the validation -split. These can then be loaded within validphys by leveraging the -:py:func:`validphys.pseudodata.read_fit_pseudodata` action: - -.. code-block :: python - - >>> from validphys.api import API - >>> pseudodata = API.read_fit_pseudodata(fit="pseudodata_test_fit_n3fit") - >>> replica1_info = pseudodata[0] - >>> replica1_info.pseudodata.loc[replica1_info.tr_idx] - replica 1 - group dataset id - ATLAS ATLASZPT8TEVMDIST 1 29.856281 - 3 14.686290 - 4 8.568288 - 5 2.848544 - 6 0.704977 - ... ... - NMC NMCPD 247 0.688019 - 249 0.713272 - 255 0.673997 - 256 0.751973 - 259 0.750572 - - [223 rows x 1 columns] - -With the postfit reshuffling handled instead by :py:func:`validphys.pseudodata.read_pdf_pseudodata`. - - -Reconstructing pseudodata --------------------------- - -.. warning:: - - The functionality described here is not guaranteed to work between different versions of the code - or its dependencies. Specifically, if anything breaks the pseudodata generation between commits, - e.g. changes to the theory predictions or settings or the random number generator, it is **not - possible to reconstruct previously generated pseudodata** for the code state at such different - commits. - Suppose one has obtained a fit using the ``n3fit`` framework and wants to do some analysis that requires -knowing exactly the data that the neural networks saw during the fitting procedure while this has not been stored. -The information is reproducible given the various seeds in the fit runcard. +knowing exactly the data that the neural networks saw during the fitting procedure. The +information is reproducible given the various seeds in the fit runcard. The 3 random seeds used in the fit are ``trvlseed`` which determines the training/validation splitting, ``nnseed`` which concerns the initialization of the neural netowrks themselves, and finally ``mcseed`` which is the @@ -111,4 +62,43 @@ is a ``namedtuple`` containing the entire dataset, alongside the training and va When running this action from a runcard, it may be worthwhile to use the ``--parallel`` flag when calling validphys. This flag parallelizes dependencies which will compute the pseudodata replicas in an asynchronous manner. This is - advantageous since the MC replica generation is computationally intensive. \ No newline at end of file + advantageous since the MC replica generation is computationally intensive. + +Saving and loading pseudodata +----------------------------- + +It is possible to instead save the Monte Carlo replicas to disk as the fit is performed. This can be achieved by setting +the ``savepseudodata`` flag to ``true`` under the ``fitting`` namespace in the fit runcard: + +.. code-block :: yaml + + fitting: + savepseudodata: true + +This will save the training and validation splits to disk under files named ``datacuts_theory_fitting_training_pseudodata.csv`` +and similarly for the validation split. These can then be loaded within validphys be leveraging the +:py:func:`validphys.pseudodata.read_fit_pseudodata` action: + +.. code-block :: python + + >>> from validphys.api import API + >>> pseudodata = API.read_fit_pseudodata(fit="pseudodata_test_fit_n3fit") + >>> replica1_info = pseudodata[0] + >>> replica1_info.pseudodata.loc[replica1_info.tr_idx] + replica 1 + group dataset id + ATLAS ATLASZPT8TEVMDIST 1 29.856281 + 3 14.686290 + 4 8.568288 + 5 2.848544 + 6 0.704977 + ... ... + NMC NMCPD 247 0.688019 + 249 0.713272 + 255 0.673997 + 256 0.751973 + 259 0.750572 + + [223 rows x 1 columns] + +With the postfit reshuffling be handled instead by the :py:func:`validphys.pseudodata.read_pdf_pseudodata`. diff --git a/n3fit/runcards/examples/developing.yml b/n3fit/runcards/examples/developing.yml index 2db98eacd5..57c10236bc 100644 --- a/n3fit/runcards/examples/developing.yml +++ b/n3fit/runcards/examples/developing.yml @@ -92,7 +92,6 @@ parameters: # This defines the parameter dictionary that is passed to the Model fitting: fitbasis: EVOL # EVOL (7), EVOLQED (8), etc. - savepseudodata: False basis: - {fl: sng, trainable: false, smallx: [1.093, 1.121], largex: [1.486, 3.287]} - {fl: g, trainable: false, smallx: [0.8329, 1.071], largex: [3.084, 6.767]} diff --git a/n3fit/setup.py b/n3fit/setup.py index e81c54809b..e68a67f4fe 100644 --- a/n3fit/setup.py +++ b/n3fit/setup.py @@ -13,7 +13,6 @@ entry_points = {'console_scripts': ['n3fit = n3fit.scripts.n3fit_exec:main', - 'evolven3fit_new = n3fit.scripts.evolven3fit_new:main', 'vp-setupfit = n3fit.scripts.vp_setupfit:main', 'varflavors = n3fit.scripts.varflavors:main', ] diff --git a/n3fit/src/evolven3fit_new/__init__.py b/n3fit/src/evolven3fit_new/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/n3fit/src/evolven3fit_new/cli.py b/n3fit/src/evolven3fit_new/cli.py deleted file mode 100644 index 92ecbdf231..0000000000 --- a/n3fit/src/evolven3fit_new/cli.py +++ /dev/null @@ -1,244 +0,0 @@ -from . import evolve, utils - -import numpy as np - -XGRID = np.array( - [ - 1.00000000000000e-09, - 1.29708482343957e-09, - 1.68242903474257e-09, - 2.18225315420583e-09, - 2.83056741739819e-09, - 3.67148597892941e-09, - 4.76222862935315e-09, - 6.17701427376180e-09, - 8.01211109898438e-09, - 1.03923870607245e-08, - 1.34798064073805e-08, - 1.74844503691778e-08, - 2.26788118881103e-08, - 2.94163370300835e-08, - 3.81554746595878e-08, - 4.94908707232129e-08, - 6.41938295708371e-08, - 8.32647951986859e-08, - 1.08001422993829e-07, - 1.40086873081130e-07, - 1.81704331793772e-07, - 2.35685551545377e-07, - 3.05703512595323e-07, - 3.96522309841747e-07, - 5.14321257236570e-07, - 6.67115245136676e-07, - 8.65299922973143e-07, - 1.12235875241487e-06, - 1.45577995547683e-06, - 1.88824560514613e-06, - 2.44917352454946e-06, - 3.17671650028717e-06, - 4.12035415232797e-06, - 5.34425265752090e-06, - 6.93161897806315e-06, - 8.99034258238145e-06, - 1.16603030112258e-05, - 1.51228312288769e-05, - 1.96129529349212e-05, - 2.54352207134502e-05, - 3.29841683435992e-05, - 4.27707053972016e-05, - 5.54561248105849e-05, - 7.18958313632514e-05, - 9.31954227979614e-05, - 1.20782367731330e-04, - 1.56497209466554e-04, - 2.02708936328495e-04, - 2.62459799331951e-04, - 3.39645244168985e-04, - 4.39234443000422e-04, - 5.67535660104533e-04, - 7.32507615725537e-04, - 9.44112105452451e-04, - 1.21469317686978e-03, - 1.55935306118224e-03, - 1.99627451141338e-03, - 2.54691493736552e-03, - 3.23597510213126e-03, - 4.09103436509565e-03, - 5.14175977083962e-03, - 6.41865096062317e-03, - 7.95137940306351e-03, - 9.76689999624100e-03, - 1.18876139251364e-02, - 1.43298947643919e-02, - 1.71032279460271e-02, - 2.02100733925079e-02, - 2.36463971369542e-02, - 2.74026915728357e-02, - 3.14652506132444e-02, - 3.58174829282429e-02, - 4.04411060163317e-02, - 4.53171343973807e-02, - 5.04266347950069e-02, - 5.57512610084339e-02, - 6.12736019390519e-02, - 6.69773829498255e-02, - 7.28475589986517e-02, - 7.88703322292727e-02, - 8.50331197801452e-02, - 9.13244910278679e-02, - 9.77340879783772e-02, - 1.04252538208639e-01, - 1.10871366547237e-01, - 1.17582909372878e-01, - 1.24380233801599e-01, - 1.31257062945031e-01, - 1.38207707707289e-01, - 1.45227005135651e-01, - 1.52310263065985e-01, - 1.59453210652156e-01, - 1.66651954293987e-01, - 1.73902938455578e-01, - 1.81202910873333e-01, - 1.88548891679097e-01, - 1.95938145999193e-01, - 2.03368159629765e-01, - 2.10836617429103e-01, - 2.18341384106561e-01, - 2.25880487124065e-01, - 2.33452101459503e-01, - 2.41054536011681e-01, - 2.48686221452762e-01, - 2.56345699358723e-01, - 2.64031612468684e-01, - 2.71742695942783e-01, - 2.79477769504149e-01, - 2.87235730364833e-01, - 2.95015546847664e-01, - 3.02816252626866e-01, - 3.10636941519503e-01, - 3.18476762768082e-01, - 3.26334916761672e-01, - 3.34210651149156e-01, - 3.42103257303627e-01, - 3.50012067101685e-01, - 3.57936449985571e-01, - 3.65875810279643e-01, - 3.73829584735962e-01, - 3.81797240286494e-01, - 3.89778271981947e-01, - 3.97772201099286e-01, - 4.05778573402340e-01, - 4.13796957540671e-01, - 4.21826943574548e-01, - 4.29868141614175e-01, - 4.37920180563205e-01, - 4.45982706956990e-01, - 4.54055383887562e-01, - 4.62137890007651e-01, - 4.70229918607142e-01, - 4.78331176755675e-01, - 4.86441384506059e-01, - 4.94560274153348e-01, - 5.02687589545177e-01, - 5.10823085439086e-01, - 5.18966526903235e-01, - 5.27117688756998e-01, - 5.35276355048428e-01, - 5.43442318565661e-01, - 5.51615380379768e-01, - 5.59795349416641e-01, - 5.67982042055800e-01, - 5.76175281754088e-01, - 5.84374898692498e-01, - 5.92580729444440e-01, - 6.00792616663950e-01, - 6.09010408792398e-01, - 6.17233959782450e-01, - 6.25463128838069e-01, - 6.33697780169485e-01, - 6.41937782762089e-01, - 6.50183010158361e-01, - 6.58433340251944e-01, - 6.66688655093089e-01, - 6.74948840704708e-01, - 6.83213786908386e-01, - 6.91483387159697e-01, - 6.99757538392251e-01, - 7.08036140869916e-01, - 7.16319098046733e-01, - 7.24606316434025e-01, - 7.32897705474271e-01, - 7.41193177421404e-01, - 7.49492647227008e-01, - 7.57796032432224e-01, - 7.66103253064927e-01, - 7.74414231541921e-01, - 7.82728892575836e-01, - 7.91047163086478e-01, - 7.99368972116378e-01, - 8.07694250750291e-01, - 8.16022932038457e-01, - 8.24354950923382e-01, - 8.32690244169987e-01, - 8.41028750298844e-01, - 8.49370409522600e-01, - 8.57715163684985e-01, - 8.66062956202683e-01, - 8.74413732009721e-01, - 8.82767437504206e-01, - 8.91124020497459e-01, - 8.99483430165226e-01, - 9.07845617001021e-01, - 9.16210532771399e-01, - 9.24578130473112e-01, - 9.32948364292029e-01, - 9.41321189563734e-01, - 9.49696562735755e-01, - 9.58074441331298e-01, - 9.66454783914439e-01, - 9.74837550056705e-01, - 9.83222700304978e-01, - 9.91610196150662e-01, - 1.00000000000000e00, - ] -) - - -def cli_evolven3fit_new( - configuration_folder, - q_fin, - q_points, - op_card_info, - theory_card_info, - dump, - load, - force, -): - """Evolves the fitted PDFs. - - The q_grid starts at the Q0 given by the theory but - the last point is q_fin and its number of - points can be specified by q_points. If just one of the - two is not specified by the user, the default grid - will be used. - - If a path is given for the dump option, the eko - will be dumped in that path after the computation. - - If a path is given for the load option, the eko - to be used for the evolution will be loaded from that - path. - - The two options are mutually exclusive. - """ - utils.check_is_a_fit(configuration_folder) - return evolve.evolve_fit( - configuration_folder, - q_fin, - q_points, - op_card_info, - theory_card_info, - force, - load, - dump, - ) diff --git a/n3fit/src/evolven3fit_new/eko_utils.py b/n3fit/src/evolven3fit_new/eko_utils.py deleted file mode 100644 index 076e38144d..0000000000 --- a/n3fit/src/evolven3fit_new/eko_utils.py +++ /dev/null @@ -1,83 +0,0 @@ -from ekobox import gen_theory, gen_op -from eko import run_dglap - -from validphys.loader import Loader -from . import utils - -from typing import Any, Dict, Optional -import logging - -_logger = logging.getLogger(__name__) - - -def construct_eko_cards( - theoryID, - q_fin, - q_points, - x_grid, - op_card_dict: Optional[Dict[str, Any]] = None, - theory_card_dict: Optional[Dict[str, Any]] = None, -): - """ - Return the theory and operator cards used to construct the eko. - theoryID is the ID of the theory for which we are computing the theory and operator card. - q_fin is the final point of the q grid while q_points is the number of points of the grid. - x_grid is the x grid to be used. - op_card_dict and theory_card_dict are optional updates that can be provided respectively to the - operator card and to the theory card. - """ - if theory_card_dict is None: - theory_card_dict = {} - if op_card_dict is None: - op_card_dict = {} - # theory_card construction - theory = Loader().check_theoryID(theoryID).get_description() - theory.pop("FNS") - theory.update(theory_card_dict) - theory_card = gen_theory.gen_theory_card(theory["PTO"], theory["Q0"], update=theory) - # construct operator card - q2_grid = utils.generate_q2grid( - theory["Q0"], - q_fin, - q_points, - {theory["mb"]: theory["kbThr"], theory["mt"]: theory["ktThr"]}, - ) - op_card = gen_op.gen_op_card(q2_grid, update={"interpolation_xgrid": x_grid}) - op_card.update(op_card_dict) - return theory_card, op_card - - -def construct_eko_for_fit(theory_card, op_card, save_path=None): - """ - Construct the eko operator needed for evolution of fitted pdfs - - Parameters - ---------- - theory_card: dict - theory card to use for the eko - op_card: dict - operator card to use for the eko - save_path: pathlib.Path - path where the eko will be saved (the eko - won't be saved if save_path is None) - Returns - ------- - : eko.output.Output - eko operator - """ - # generate eko operator - if save_path is not None: - if not save_path.parent.exists(): - raise FileNotFoundError( - f"Path where eko should be dumped does not exist: {save_path}" - ) - eko_op = run_dglap(theory_card, op_card) - if save_path is not None: - # Here we want to catch all possible exceptions in order to avoid losing the computed eko - try: - _logger.info(f"Saving computed eko to : {save_path}") - eko_op.dump_tar(save_path) - except: - _logger.error(f"Error saving the eko to : {save_path}") - pass - return eko_op diff --git a/n3fit/src/evolven3fit_new/evolve.py b/n3fit/src/evolven3fit_new/evolve.py deleted file mode 100644 index 2cf0180040..0000000000 --- a/n3fit/src/evolven3fit_new/evolve.py +++ /dev/null @@ -1,220 +0,0 @@ -import pathlib -import logging -import sys - -import numpy as np -from reportengine.compat import yaml - -from ekobox import genpdf, gen_info -from ekomark import apply -from eko import basis_rotation as br -from eko import output -from validphys.loader import Loader - -from . import utils, eko_utils - - -_logger = logging.getLogger(__name__) - -LOG_FILE = "evolven3fit_new.log" - -LOGGING_SETTINGS = {"formatter" : "%(asctime)s %(name)s/%(levelname)s: %(message)s", - "level" : logging.INFO -} - -def evolve_fit( - fit_folder, - q_fin, - q_points, - op_card_dict, - theory_card_dict, - force, - eko_path=None, - dump_eko=None, -): - """ - Evolves all the fitted replica in fit_folder/nnfit - - Parameters - ---------- - - fit_folder: str or pathlib.Path - path to the folder containing the fit - q_fin: float - final point of the q_grid - q_points: int - number of points in the q_grid - op_card_dict: dict - user settings for the op_card - theory_card_dict: dict - user settings for the t_card - force: bool - whether to force the evolution to be done again - eko_path: str or pathlib.Path - path where the eko is stored (if None the eko will be - recomputed) - dump_eko: str or pathlib.Path - path where the eko is dumped (if None the eko won't be - stored) - """ - log_file = pathlib.Path(fit_folder) / LOG_FILE - if log_file.exists(): - if force: - log_file.unlink() - else: - raise FileExistsError( - f"Log file already exists: {log_file} evolven3fit_new has already been run?" - ) - log_file = logging.FileHandler(log_file) - stdout_log = logging.StreamHandler(sys.stdout) - for log in [log_file, stdout_log]: - log.setLevel(LOGGING_SETTINGS["level"]) - log.setFormatter( - logging.Formatter(LOGGING_SETTINGS["formatter"]) - ) - for logger in (_logger, *[logging.getLogger("eko")]): - logger.handlers = [] - logger.setLevel(LOGGING_SETTINGS["level"]) - logger.addHandler(log_file) - logger.addHandler(stdout_log) - - usr_path = pathlib.Path(fit_folder) - initial_PDFs_dict = load_fit(usr_path) - x_grid = np.array( - initial_PDFs_dict[list(initial_PDFs_dict.keys())[0]]["xgrid"] - ).astype(np.float) - theoryID = utils.get_theoryID_from_runcard(usr_path) - theory, op = eko_utils.construct_eko_cards( - theoryID, q_fin, q_points, x_grid, op_card_dict, theory_card_dict - ) - if eko_path is not None: - eko_path = pathlib.Path(eko_path) - _logger.info(f"Loading eko from : {eko_path}") - eko_op = output.Output.load_tar(eko_path) - else: - try: - _logger.info(f"Loading eko from theory {theoryID}") - theory_eko_path = (Loader().check_theoryID(theoryID).path)/'eko.tar' - eko_op = output.Output.load_tar(theory_eko_path) - except FileNotFoundError: - _logger.info(f"eko not found in theory {theoryID}, we will construct it") - eko_op = eko_utils.construct_eko_for_fit(theory, op, dump_eko) - eko_op.xgrid_reshape(targetgrid=x_grid, inputgrid=x_grid) - info = gen_info.create_info_file(theory, op, 1, info_update={}) - info["NumMembers"] = "REPLACE_NREP" - info["ErrorType"] = "replicas" - info["AlphaS_Qs"] = list(eko_op["Q2grid"]) - info["XMin"] = float(x_grid[0]) - info["XMax"] = float(x_grid[-1]) - dump_info_file(usr_path, info) - for replica in initial_PDFs_dict.keys(): - evolved_block = evolve_exportgrid(initial_PDFs_dict[replica], eko_op, x_grid) - dump_evolved_replica( - evolved_block, usr_path, int(replica.removeprefix("replica_")) - ) - # remove folder: - # The function dump_evolved_replica dumps the replica files in a temporary folder - # We need then to remove it after fixing the position of those replica files - (usr_path / "nnfit" / usr_path.stem).rmdir() - - -def load_fit(usr_path): - """ - Loads all the replica pdfs at fitting scale in usr_path and return the exportgrids - - Parameters - ---------- - - usr_path: pathlib.Path - path to the folder containing the fit - - Returns - ------- - - : dict - exportgrids info - """ - nnfitpath = usr_path / "nnfit" - pdf_dict = {} - for yaml_file in nnfitpath.glob("replica_*/*.exportgrid"): - data = yaml.safe_load(yaml_file.read_text(encoding="UTF-8")) - pdf_dict[yaml_file.parent.stem] = data - return pdf_dict - - -def evolve_exportgrid(exportgrid, eko, x_grid): - """ - Evolves the provided exportgrid for the desired replica with the eko and returns the evolved block - - Parameters - ---------- - exportgrid: dict - exportgrid of pdf at fitting scale - eko: eko object - eko operator for evolution - xgrid: list - xgrid to be used as the targetgrid - Returns - ------- - : np.array - evolved block - """ - # construct LhapdfLike object - pdf_grid = np.array(exportgrid["pdfgrid"]).transpose() - pdf_to_evolve = utils.LhapdfLike(pdf_grid, exportgrid["q20"], x_grid) - # evolve pdf - evolved_pdf = apply.apply_pdf(eko, pdf_to_evolve) - # generate block to dump - targetgrid = list(eko["targetgrid"]) - - def ev_pdf(pid, x, Q2): - return x * evolved_pdf[Q2]["pdfs"][pid][targetgrid.index(x)] - - block = genpdf.generate_block( - ev_pdf, - xgrid=targetgrid, - Q2grid=list(eko["Q2grid"]), - pids=br.flavor_basis_pids, - ) - return block - - -def dump_evolved_replica(evolved_block, usr_path, replica_num): - """ - Dump the evolved replica given by evolved_block as the replica num "replica_num" in - the folder usr_path/nnfit/replica_/usr_path.stem.dat - - Parameters - ---------- - evolved_block: numpy.array - block of an evolved PDF - usr_path: pathlib.Path - path of the fit folder - replica_num: int - replica number - """ - path_where_dump = usr_path / "nnfit" / usr_path.stem - # create folder to dump the evolved replica if it does not exist - path_where_dump.mkdir(exist_ok=True) - to_write_in_head = f"PdfType: replica\nFromMCReplica: {replica_num}\n" - genpdf.export.dump_blocks( - path_where_dump, replica_num, [evolved_block], pdf_type=to_write_in_head - ) - # fixing_replica_path - utils.fix_replica_path(usr_path, replica_num) - - -def dump_info_file(usr_path, info): - """ - Dump the info file given by info in the folder usr_path/nnfit/usr_path.stem.info. - - Parameters - ---------- - usr_path: pathlib.Path - path of the fit folder - info: dict - info of the fit - """ - path_where_dump = usr_path / "nnfit" / usr_path.stem - genpdf.export.dump_info(path_where_dump, info) - utils.fix_info_path(usr_path) diff --git a/n3fit/src/evolven3fit_new/utils.py b/n3fit/src/evolven3fit_new/utils.py deleted file mode 100644 index afbfa6f445..0000000000 --- a/n3fit/src/evolven3fit_new/utils.py +++ /dev/null @@ -1,201 +0,0 @@ -import shutil -import pathlib -from scipy.interpolate import interp1d -import numpy as np -from reportengine.compat import yaml -from validphys.pdfbases import PIDS_DICT - -DEFAULT_Q2GRID = np.array( - [ - 1.6500000e00, - 1.7874388e00, - 1.9429053e00, - 2.1193749e00, - 2.3204100e00, - 2.5502944e00, - 2.8142025e00, - 3.1184122e00, - 3.4705775e00, - 3.8800751e00, - 4.3584516e00, - 4.9200000e00, - 4.9200000e00, - 5.5493622e00, - 6.2897452e00, - 7.1650687e00, - 8.2052867e00, - 9.4481248e00, - 1.0941378e01, - 1.2745972e01, - 1.4940062e01, - 1.7624572e01, - 2.0930715e01, - 2.5030298e01, - 3.0149928e01, - 3.6590777e01, - 4.4756282e01, - 5.5191298e01, - 6.8637940e01, - 8.6115921e01, - 1.0903923e02, - 1.3938725e02, - 1.7995815e02, - 2.3474820e02, - 3.0952544e02, - 4.1270732e02, - 5.5671861e02, - 7.6011795e02, - 1.0509694e03, - 1.4722574e03, - 2.0906996e03, - 3.0112909e03, - 4.4016501e03, - 6.5333918e03, - 9.8535186e03, - 1.5109614e04, - 2.3573066e04, - 3.7444017e04, - 6.0599320e04, - 1.0000000e05, - ] - ) **2 - -class LhapdfLike: - """ - Class which emulates lhapdf but only for an initial condition PDF (i.e. with only one q2 value). - - Q20 is the fitting scale fo the pdf and it is the only available scale for the objects of this class. - - X_GRID is the grid of x values on top of which the pdf is interpolated. - - PDF_GRID is a dictionary containing the pdf grids at fitting scale for each pid. - """ - - def __init__(self, pdf_grid, q20, x_grid): - self.pdf_grid = pdf_grid - self.q20 = q20 - self.x_grid = x_grid - self.funcs = [ - interp1d(self.x_grid, self.pdf_grid[pid], kind="cubic") - for pid in range(len(PIDS_DICT)) - ] - - def xfxQ2(self, pid, x, q2): - """Return the value of the PDF for the requested pid, x value and, whatever the requested - q2 value, for the fitting q2. - - Parameters - ---------- - - pid: int - pid index of particle - x: float - x-value - q2: float - Q square value - - Returns - ------- - : float - x * PDF value - """ - return self.funcs[list(PIDS_DICT.values()).index(PIDS_DICT[pid])](x) - - def hasFlavor(self, pid): - """Check if the requested pid is in the PDF.""" - return pid in PIDS_DICT - - -def read_runcard(usr_path): - """Read the runcard and return the relevant information for evolven3fit_new""" - return yaml.safe_load((usr_path / "filter.yml").read_text(encoding="UTF-8")) - - -def get_theoryID_from_runcard(usr_path): - """Return the theoryID from the runcard""" - # read the runcard - my_runcard = read_runcard(usr_path) - return my_runcard["theory"]["theoryid"] - - -def generate_q2grid(Q0, Qfin, Q_points, match_dict): - """Generate the q2grid used in the final evolved pdfs or use the default grid if Qfin or Q_points is - not provided. - - match_dict contains the couples (mass : factor) where factor is the number to be multiplied to mass - in order to obtain the relative matching scale. - """ - if Qfin is None and Q_points is None: - return DEFAULT_Q2GRID - elif Qfin is None or Q_points is None: - raise ValueError( - "q_fin and q_points must be specified either both or none of them" - ) - else: - grids = [] - Q_ini = Q0 - num_points_list = [] - for masses in match_dict: - match_scale = masses * match_dict[masses] - # Fraction of the total points to be included in this batch is proportional - # to the log of the ratio between the initial scale and final scale of the - # batch itself (normalized to the same log of the global initial and final - # scales) - if match_scale < Qfin: - frac_of_point = np.log(match_scale / Q_ini) / np.log(Qfin / Q0) - num_points = int( - Q_points * frac_of_point - ) - num_points_list.append(num_points) - grids.append(np.geomspace(Q_ini**2, match_scale**2, num=num_points)) - Q_ini = match_scale - num_points = Q_points - sum(num_points_list) - grids.append(np.geomspace(Q_ini**2, Qfin**2, num=num_points)) - return np.concatenate(grids).tolist() - - -def fix_info_path(usr_path): - """Fix the location of the info file from the folder nnfit/usr_path to - just nnfit - - Examples - -------- - Starting from the info path - initial_info_file_path = "/myfolder/myfit/nnfit/myfit/myfit.info" - and using this function with usr_path = "/myfolder/myfit", one gets - final_info_file_path = "/myfolder/myfit/nnfit/myfit.info" - """ - nnfit = usr_path / "nnfit" - info_file = usr_path.stem + ".info" - info_file_path = nnfit / usr_path.stem / info_file - dest_path_info = nnfit / info_file - shutil.move(info_file_path, dest_path_info) - - -def fix_replica_path(usr_path, replica_num): - """Fix the location of the dat file of the replica from the folder nnfit/usr_path to - just nnfit/replica_ - - Examples - -------- - Starting from the replica 5 path - initial_replica_file_path = "/myfolder/myfit/nnfit/myfit/myfit_5.dat" - and using this function with usr_path = "/myfolder/myfit", one gets - final_replica_file_path = "/myfolder/myfit/nnfit/replica_5/myfit.dat" - """ - nnfit = usr_path / "nnfit" - replica_file_path = nnfit / usr_path.stem / f"{usr_path.stem}_{replica_num:04d}.dat" - dest_path_replica = nnfit / f"replica_{replica_num}" / f"{usr_path.stem}.dat" - shutil.move(replica_file_path, dest_path_replica) - - -def check_is_a_fit(config_folder): - """Check if config_folder is a fit folder, i.e. if it contains the filter.yml file - and the nnfit folder.""" - usr_path = pathlib.Path(config_folder) - filter_path = usr_path / "filter.yml" - if not filter_path.is_file(): - raise ValueError("filter.yaml file not found: the path" + str(filter_path.absolute()) + " is not valid") - nnfitpath = usr_path / "nnfit" - if not nnfitpath.is_dir(): - raise ValueError("nnfit folder not found: provided path is not valid") diff --git a/n3fit/src/n3fit/hyper_optimization/penalties.py b/n3fit/src/n3fit/hyper_optimization/penalties.py index 6e2e2ab12f..a17067edaf 100644 --- a/n3fit/src/n3fit/hyper_optimization/penalties.py +++ b/n3fit/src/n3fit/hyper_optimization/penalties.py @@ -20,7 +20,7 @@ """ import numpy as np from validphys import fitveto -from n3fit.vpinterface import N3PDF, integrability_numbers +from n3fit.vpinterface import N3PDF def saturation(pdf_models=None, n=100, min_x=1e-6, max_x=1e-4, flavors=None, **_kwargs): @@ -43,8 +43,8 @@ def saturation(pdf_models=None, n=100, min_x=1e-6, max_x=1e-4, flavors=None, **_ ------- >>> from n3fit.hyper_optimization.penalties import saturation >>> from n3fit.model_gen import pdfNN_layer_generator - >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'g', 's', 'sbar']] - >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR") + >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'cbar', 's', 'sbar']] + >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl) >>> isinstance(saturation(pdf_model, 5), float) True @@ -82,7 +82,7 @@ def patience(stopping_object=None, alpha=1e-4, **_kwargs): >>> from n3fit.hyper_optimization.penalties import patience >>> from types import SimpleNamespace >>> fake_stopping = SimpleNamespace(e_best_chi2=1000, stopping_patience=500, total_epochs=5000, vl_loss=2.42) - >>> patience(fake_stopping, alpha=1e-4) + >>> patience(None, fake_stopping, alpha=1e-4) 3.434143467595683 """ @@ -105,14 +105,14 @@ def integrability(pdf_models=None, **_kwargs): ------- >>> from n3fit.hyper_optimization.penalties import integrability >>> from n3fit.model_gen import pdfNN_layer_generator - >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'g', 's', 'sbar']] - >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR") - >>> isinstance(integrability(pdf_model), float) + >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'cbar', 's', 'sbar']] + >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl) + >>> isinstance(integrability([pdf_model], None), float) True """ pdf_instance = N3PDF(pdf_models) - integ_values = integrability_numbers(pdf_instance) + integ_values = pdf_instance.integrability_numbers() integ_overflow = np.sum(integ_values[integ_values > fitveto.INTEG_THRESHOLD]) if integ_overflow > 50.0: # before reaching an overflow, just give a stupidly big number diff --git a/n3fit/src/n3fit/io/writer.py b/n3fit/src/n3fit/io/writer.py index a670a9e365..9eb81068ce 100644 --- a/n3fit/src/n3fit/io/writer.py +++ b/n3fit/src/n3fit/io/writer.py @@ -11,7 +11,7 @@ import validphys import n3fit from n3fit import vpinterface -from evolven3fit_new.cli import XGRID + class WriterWrapper: def __init__(self, replica_number, pdf_object, stopping_object, q2, timings): @@ -249,8 +249,8 @@ def storefit( q_0^2 """ # build exportgrid - xgrid = XGRID.reshape(-1, 1) - + xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]).reshape(-1, 1) + result = pdf_object(xgrid, flavours="n3fit").squeeze() lha = evln2lha(result.T).T diff --git a/n3fit/src/n3fit/scripts/evolven3fit_new.py b/n3fit/src/n3fit/scripts/evolven3fit_new.py deleted file mode 100644 index a1bfe29370..0000000000 --- a/n3fit/src/n3fit/scripts/evolven3fit_new.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -This module contains the CLI for evolven3fit_new -""" -import logging -import pathlib -from argparse import ArgumentParser -import sys -import numpy as np - -from evolven3fit_new import eko_utils, evolve, cli - -_logger = logging.getLogger(__name__) - - -def construct_eko_parser(subparsers): - parser = subparsers.add_parser( - "produce_eko", - help=( - """Produce the eko for the specified theory. - The q_grid starts at the Q0 given by the theory but the last point is q_fin - and its number of points can be specified by q_points. - The x_grid starts at x_grid_ini and ends at 1.0 and contains the - provided number of points. The eko will be dumped in the provided path.""" - ), - ) - parser.add_argument( - "theoryID", type=int, help="ID of the theory used to produce the eko" - ) - parser.add_argument( - "dump", - type=pathlib.Path, - help="Path where the EKO is dumped", - ) - parser.add_argument( - "-i", - "--x-grid-ini", - default=None, - type=float, - help="Starting point of the x-grid", - ) - parser.add_argument( - "-p", - "--x-grid-points", - default=None, - type=int, - help="Number of points of the x-grid", - ) - return parser - - -def construct_evolven3fit_parser(subparsers): - parser = subparsers.add_parser( - "evolve", - help="Evolves the fitted PDFs. The q_grid starts at the Q0 given by the theory but the last point is q_fin and its number of points can be specified by q_points. If a path is given for the dump option, the eko will be dumped in that path after the computation. If a path is given for the load option, the eko to be used for the evolution will be loaded from that path. The two options are mutually exclusive.", - ) - parser.add_argument( - "configuration_folder", - help="Path to the folder containing the (pre-DGLAP) fit result", - ) - parser.add_argument( - "-l", - "--load", - type=pathlib.Path, - default=None, - help="Path of the EKO to be loaded", - ) - parser.add_argument( - "-d", - "--dump", - type=pathlib.Path, - default=None, - help="Path where the EKO is dumped", - ) - parser.add_argument( - "-f", - "--force", - action="store_true", - help="Force the evolution to be done even if it has already been done", - ) - return parser - - -def main(): - parser = ArgumentParser( - description="evolven3fit_new - a script with tools to evolve PDF fits" - ) - parser.add_argument( - "-q", - "--q-fin", - type=float, - default=None, - help="Final q-value of the evolution", - ) - parser.add_argument( - "-p", - "--q-points", - type=int, - default=None, - help="Number of q points for the evolution", - ) - parser.add_argument( - "-n", - "--n-cores", - type=int, - default=1, - help="Number of cores to be used", - ) - subparsers = parser.add_subparsers(title="actions", dest="actions") - eko_parser = construct_eko_parser(subparsers) - evolven3fit_parser = construct_evolven3fit_parser(subparsers) - - args = parser.parse_args() - op_card_info = { - "ev_op_max_order": 10, - "ev_op_iterations": 1, - "n_integration_cores": args.n_cores, - "backward_inversion": "expanded", - } - theory_card_info = {} - if args.actions == "evolve": - cli.cli_evolven3fit_new( - args.configuration_folder, - args.q_fin, - args.q_points, - op_card_info, - theory_card_info, - args.dump, - args.load, - args.force, - ) - elif args.actions == "produce_eko": - stdout_log = logging.StreamHandler(sys.stdout) - stdout_log.setLevel(evolve.LOGGING_SETTINGS["level"]) - stdout_log.setFormatter(evolve.LOGGING_SETTINGS["formatter"]) - for logger_ in (_logger, *[logging.getLogger("eko")]): - logger_.handlers = [] - logger_.setLevel(evolve.LOGGING_SETTINGS["level"]) - logger_.addHandler(stdout_log) - if args.x_grid_ini is None: - if args.x_grid_points is None: - x_grid = cli.XGRID - else: - raise ValueError( - "x_grid_ini and x_grid_points must be specified either both or none of them" - ) - elif args.x_grid_points is None: - raise ValueError( - "x_grid_ini and x_grid_points must be specified either both or none of them" - ) - else: - x_grid = np.geomspace(args.x_grid_ini, 1.0, args.x_grid_points) - tcard, opcard = eko_utils.construct_eko_cards( - args.theoryID, args.q_fin, args.q_points, x_grid, op_card_info, theory_card_info - ) - _ = eko_utils.construct_eko_for_fit(tcard, opcard, args.dump) - - -if __name__ == "__main__": - main() diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 7ec49e0e00..032f2d748c 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -133,9 +133,7 @@ def from_yaml(cls, o, *args, **kwargs): N3FIT_FIXED_CONFIG['actions_'].append(namespace + "performfit") - if fps := file_content["fitting"].get("savepseudodata", True): - if fps != True: - raise TypeError(f"fitting::savepseudodata is neither True nor False ({fps})") + if file_content["fitting"].get("savepseudodata"): if len(kwargs["environment"].replicas) != 1: raise ConfigError( "Cannot request that multiple replicas are fitted and that " diff --git a/n3fit/src/n3fit/tests/regressions/filter.yml b/n3fit/src/n3fit/tests/regressions/filter.yml deleted file mode 100644 index efb09d94e3..0000000000 --- a/n3fit/src/n3fit/tests/regressions/filter.yml +++ /dev/null @@ -1,88 +0,0 @@ -# -# Configuration file for n3fit regression tests -# This runcard includes two DIS datasets, one Hadronic dataset -# and two positivity datasets -# - -############################################################ -description: n3fit regression test - -############################################################ -# frac: training fraction -# ewk: apply ewk k-factors -# sys: systematics treatment (see systypes) -dataset_inputs: -- { dataset: NMC, frac: 0.5 } -- { dataset: SLACP, frac: 0.5} -- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 } -- { dataset: CMSTTBARTOT, frac: 1.0, cfac: ['QCD'] } - -############################################################ -datacuts: - t0pdfset: NNPDF40_nnlo_as_01180 # PDF set to generate t0 covmat - q2min : 3.49 # Q2 minimum - w2min : 12.5 # W2 minimum - combocuts : NNPDF31 # NNPDF3.0 final kin. cuts - jetptcut_tev : 0 # jet pt cut for tevatron - jetptcut_lhc : 0 # jet pt cut for lhc - wptcut_lhc : 30.0 # Minimum pT for W pT diff distributions - jetycut_tev : 1e30 # jet rap. cut for tevatron - jetycut_lhc : 1e30 # jet rap. cut for lhc - dymasscut_min: 0 # dy inv.mass. min cut - dymasscut_max: 1e30 # dy inv.mass. max cut - jetcfactcut : 1e30 # jet cfact. cut - -############################################################ -theory: - theoryid: 162 # database id - -############################################################ -genrep: True # True = generate MC replicas, False = use real data -trvlseed: 3 -nnseed: 2 -mcseed: 1 - -load: "weights.h5" - -parameters: # This defines the parameter dictionary that is passed to the Model Trainer - nodes_per_layer: [15, 10, 8] - activation_per_layer: ['sigmoid', 'sigmoid', 'linear'] - initializer: 'glorot_normal' - optimizer: - optimizer_name: 'RMSprop' - learning_rate: 0.00001 - clipnorm: 1.0 - epochs: 1000 - positivity: - multiplier: 1.05 - initial: 1.5 - stopping_patience: 0.30 # percentage of the number of epochs - layer_type: 'dense' - dropout: 0.0 - threshold_chi2: 10.0 - -fitting: - fitbasis: NN31IC # EVOL (7), EVOLQED (8), etc. - basis: - - { fl: sng, smallx: [1.05,1.19], largex: [1.47,2.70] } - - { fl: g, smallx: [0.94,1.25], largex: [0.11,5.87] } - - { fl: v, smallx: [0.54,0.75], largex: [1.15,2.76] } - - { fl: v3, smallx: [0.21,0.57], largex: [1.35,3.08] } - - { fl: v8, smallx: [0.52,0.76], largex: [0.77,3.56] } - - { fl: t3, smallx: [-0.37,1.52], largex: [1.74,3.39] } - - { fl: t8, smallx: [0.56,1.29], largex: [1.45,3.03] } - - { fl: cp, smallx: [0.12,1.19], largex: [1.83,6.70] } - -############################################################ -positivity: - posdatasets: - - { dataset: POSF2U, maxlambda: 1e6 } # Positivity Lagrange Multiplier - - { dataset: POSDYC, maxlambda: 1e5 } - -integrability: - integdatasets: - - {dataset: INTEGXT8, maxlambda: 1e2} - - {dataset: INTEGXT3, maxlambda: 1e2} - -############################################################ -debug: true diff --git a/n3fit/src/n3fit/tests/test_evolven3fit.py b/n3fit/src/n3fit/tests/test_evolven3fit.py deleted file mode 100644 index 278d7212e3..0000000000 --- a/n3fit/src/n3fit/tests/test_evolven3fit.py +++ /dev/null @@ -1,70 +0,0 @@ -import pathlib -import logging -from numpy.testing import assert_allclose -import numpy as np -from validphys.pdfbases import PIDS_DICT -from evolven3fit_new import utils, eko_utils - -REGRESSION_FOLDER = pathlib.Path(__file__).with_name("regressions") -log = logging.getLogger(__name__) - -def check_consecutive_members(grid, value): - """Check if the first occurrence of value in grid is followed by value again""" - return np.allclose(grid[list(grid).index(value)+1], value) - -def test_utils(): - #Testing the default grid - grid = utils.generate_q2grid(1.65, None, None, {}) - assert_allclose(1.65**2, grid[0]) - assert len(grid) == 50 - # We expect the bottom mass to be repeated twice because it is intended once in 4 flavors and once in 5 flavors. - assert check_consecutive_members(grid, 4.92**2) - #Testing if the points of the matching are correctly repeated twice - matched_grid = utils.generate_q2grid(1.65, 1.0e5, 100, {4.92:2.0, 100:1.0} ) - assert len(matched_grid) == 100 - assert_allclose((1.0e5)**2, matched_grid[-1]) - assert check_consecutive_members(matched_grid, (4.92*2.0)**2) - assert check_consecutive_members(matched_grid, (100.*1.0)**2) - #Testing the fake LHAPDF class - q20 = 1.65**2 - x_grid = np.geomspace(1.0e-7, 1.0, 30) - fake_grids = [[x*(1.-x) for x in x_grid] for pid in PIDS_DICT.keys()] - pdf_grid = dict([(pid,v) for pid,v in zip(range(len(PIDS_DICT)), fake_grids )]) - my_PDF = utils.LhapdfLike(pdf_grid, q20, x_grid) - assert my_PDF.hasFlavor(6) - assert not my_PDF.hasFlavor(0) - for pid in PIDS_DICT: - for x in x_grid: - assert_allclose(my_PDF.xfxQ2(pid, x, q20), x*(1.-x) ) - #Testing read_runcard - runcard = utils.read_runcard(REGRESSION_FOLDER) - assert runcard["description"] == "n3fit regression test" - assert runcard["datacuts"]["t0pdfset"] == "NNPDF40_nnlo_as_01180" - #Testing get_theoryID_from_runcard - ID = utils.get_theoryID_from_runcard(REGRESSION_FOLDER) - assert ID == 162 - -def test_eko_utils(): - #Testing construct eko cards - theoryID = 162 - q_fin = 100 - q_points = 5 - x_grid = [1.e-3, 0.1, 1.0] - pto = 2 - qref = 91.2 - comments = "Test" - n_cores = 6 - t_card, op_card = eko_utils.construct_eko_cards(theoryID, q_fin, q_points, x_grid, {'n_integration_cores' : n_cores, 'interpolation_polynomial_degree' : 2}, {'Comments' : comments}) - assert t_card['Qref'] == qref - assert t_card['PTO'] == pto - assert t_card['Comments'] == comments - assert op_card['n_integration_cores'] == n_cores - assert_allclose(op_card["interpolation_xgrid"], x_grid) - assert_allclose(op_card["Q2grid"][0], 1.65**2) - assert_allclose(op_card["Q2grid"][-1], q_fin**2) - #In this case there are not enough points to have twice the bottom matching scale - assert_allclose(op_card["Q2grid"][1], 4.92**2) - #Testing construct_eko_for_fit - eko_op = eko_utils.construct_eko_for_fit(t_card ,op_card) - assert_allclose(eko_op['interpolation_xgrid'], x_grid) - assert_allclose(list(eko_op['Q2grid']), op_card["Q2grid"]) diff --git a/n3fit/src/n3fit/tests/test_penalties.py b/n3fit/src/n3fit/tests/test_penalties.py deleted file mode 100644 index 3899f1a5a8..0000000000 --- a/n3fit/src/n3fit/tests/test_penalties.py +++ /dev/null @@ -1,39 +0,0 @@ -""" - Test the penalties for n3fit hyperopt -""" -from types import SimpleNamespace -from n3fit.hyper_optimization.penalties import integrability, patience, saturation -from n3fit.model_gen import pdfNN_layer_generator - - -def test_saturation(): - """Check that the saturation penalty runs and returns a float""" - fake_fl = [ - {"fl": i, "largex": [0, 1], "smallx": [1, 2]} - for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"] - ] - pdf_model = pdfNN_layer_generator( - nodes=[8], activations=["linear"], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR" - ) - assert isinstance(saturation(pdf_model, 5), float) - - -def test_patience(): - """Check that the patience penalty runs and returns a float""" - fake_stopping = SimpleNamespace( - e_best_chi2=1000, stopping_patience=500, total_epochs=5000, vl_chi2=2.42 - ) - res = patience(stopping_object=fake_stopping, alpha=1e-4) - assert isinstance(res, float) - - -def test_integrability_numbers(): - """Check that the integrability penalty runs and returns a float""" - fake_fl = [ - {"fl": i, "largex": [0, 1], "smallx": [1, 2]} - for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"] - ] - pdf_model = pdfNN_layer_generator( - nodes=[8], activations=["linear"], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR" - ) - assert isinstance(integrability(pdf_model), float) diff --git a/validphys2/src/validphys/commondataparser.py b/validphys2/src/validphys/commondataparser.py index 80d55c0390..ab2cbaf8c5 100644 --- a/validphys2/src/validphys/commondataparser.py +++ b/validphys2/src/validphys/commondataparser.py @@ -2,47 +2,17 @@ This module implements parsers for commondata and systype files into useful datastructures, contained in the :py:mod:`validphys.coredata` module, which are not backed by C++ managed memory, and so they can be easily pickled and -interfaced with common Python libraries. - -The validphys commondata structure is an instance of :py:class:`validphys.coredata.CommonData` +interfaces with common Python libraries. The integration of these objects into +the codebase is currently work in progress, and at the moment this module +serves as a proof of concept. """ -import dataclasses from operator import attrgetter -import logging import pandas as pd +from validphys.core import peek_commondata_metadata from validphys.coredata import CommonData -log = logging.getLogger(__name__) - -KINLABEL_LATEX = { - "DIJET": ("\\eta", "$\\m_{1,2} (GeV)", "$\\sqrt{s} (GeV)"), - "DIS": ("$x$", "$Q^2 (GeV^2)$", "$y$"), - "DYP": ("$y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWJ_JPT": ("$p_T (GeV)$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWJ_JRAP": ("$\\eta/y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWJ_MLL": ("$M_{ll} (GeV)$", "$M_{ll}^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWJ_PT": ("$p_T (GeV)$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWJ_PTRAP": ("$\\eta/y$", "$p_T^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWJ_RAP": ("$\\eta/y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWK_MLL": ("$M_{ll} (GeV)$", "$M_{ll}^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWK_PT": ("$p_T$ (GeV)", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWK_PTRAP": ("$\\eta/y$", "$p_T^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "EWK_RAP": ("$\\eta/y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "HIG_RAP": ("$y$", "$M_H^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "HQP_MQQ": ("$M^{QQ} (GeV)$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "HQP_PTQ": ("$p_T^Q (GeV)$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "HQP_PTQQ": ("$p_T^{QQ} (GeV)$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "HQP_YQ": ("$y^Q$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "HQP_YQQ": ("$y^{QQ}$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "INC": ("$0$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "JET": ("$\\eta$", "$p_T^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "PHT": ("$\\eta_\\gamma$", "$E_{T,\\gamma}^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), - "SIA": ("$z$", "$Q^2 (GeV^2)$", "$y$"), -} - - def load_commondata(spec): """ Load the data corresponding to a CommonDataSpec object. @@ -72,12 +42,12 @@ def parse_commondata(commondatafile, systypefile, setname): and systype files. """ # First parse commondata file - commondatatable = pd.read_csv(commondatafile, sep=r"\s+", skiprows=1, header=None) + commondatatable = pd.read_csv(commondatafile, sep=r'\s+', skiprows=1, header=None) # Remove NaNs # TODO: replace commondata files with bad formatting # Build header - commondataheader = ["entry", "process", "kin1", "kin2", "kin3", "data", "stat"] - nsys = (commondatatable.shape[1] - len(commondataheader)) // 2 + commondataheader = ['entry', 'process', 'kin1', 'kin2', 'kin3', 'data', 'stat'] + nsys = (commondatatable.shape[1] - len(commondataheader)) // 2 commondataheader += ["ADD", "MULT"] * nsys commondatatable.columns = commondataheader @@ -85,8 +55,8 @@ def parse_commondata(commondatafile, systypefile, setname): ndata = len(commondatatable) commondataproc = commondatatable["process"][1] # Check for consistency with commondata metadata - cdmetadata = peek_commondata_metadata(commondatafile) - if (setname, nsys, ndata) != attrgetter("name", "nsys", "ndata")(cdmetadata): + cdmetadata = peek_commondata_metadata(commondatafile) + if (setname, nsys, ndata) != attrgetter('name', 'nsys', 'ndata')(cdmetadata): raise ValueError("Commondata table information does not match metadata") # Now parse the systype file @@ -100,18 +70,18 @@ def parse_commondata(commondatafile, systypefile, setname): nkin=3, nsys=nsys, commondata_table=commondatatable, - systype_table=systypetable, + systype_table=systypetable ) - def parse_systypes(systypefile): - """Parses a systype file and returns a pandas dataframe.""" + """Parses a systype file and returns a pandas dataframe. + """ systypeheader = ["sys_index", "type", "name"] try: systypetable = pd.read_csv( systypefile, sep=r"\s+", names=systypeheader, skiprows=1, header=None ) - systypetable.dropna(axis="columns", inplace=True) + systypetable.dropna(axis='columns', inplace=True) # Some datasets e.g. CMSWCHARMRAT have no systematics except pd.errors.EmptyDataError: systypetable = pd.DataFrame(columns=systypeheader) @@ -119,54 +89,3 @@ def parse_systypes(systypefile): systypetable.set_index("sys_index", inplace=True) return systypetable - - -@dataclasses.dataclass(frozen=True) -class CommonDataMetadata: - """Contains metadata information about the data being read""" - - name: str - nsys: int - ndata: int - process_type: str - - -def peek_commondata_metadata(commondatafilename): - """Read some of the properties of the commondata object as a CommonData Metadata - """ - with open(commondatafilename) as f: - try: - l = f.readline() - name, nsys_str, ndata_str = l.split() - l = f.readline() - process_type_str = l.split()[1] - except Exception: - log.error(f"Error processing {commondatafilename}") - raise - - return CommonDataMetadata( - name, int(nsys_str), int(ndata_str), get_kinlabel_key(process_type_str) - ) - - -def get_plot_kinlabels(commondata): - """Return the LaTex kinematic labels for a given Commondata""" - key = commondata.process_type - - return KINLABEL_LATEX[key] - - -def get_kinlabel_key(process_label): - """ - Since there is no 1:1 correspondence between latex keys and GetProc, - we match the longest key such that the proc label starts with it. - """ - l = process_label - try: - return next(k for k in sorted(KINLABEL_LATEX, key=len, reverse=True) if l.startswith(k)) - except StopIteration as e: - raise ValueError( - "Could not find a set of kinematic " - "variables matching the process %s Check the " - "labels defined in commondata.cc. " % (l) - ) from e diff --git a/validphys2/src/validphys/commondatawriter.py b/validphys2/src/validphys/commondatawriter.py deleted file mode 100644 index 07b06b4585..0000000000 --- a/validphys2/src/validphys/commondatawriter.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -This module contains functions to write commondata and systypes -tables to files -""" - -def write_commondata_data(commondata, buffer): - """ - write commondata table to buffer, this can be a memory map, - compressed archive or strings (using for instance StringIO) - - - Parameters - ---------- - - commondata : validphys.coredata.CommonData - - buffer : memory map, compressed archive or strings - example: StringIO object - - - Example - ------- - >>> from validphys.loader import Loader - >>> from io import StringIO - - >>> l = Loader() - >>> cd = l.check_commondata("NMC").load_commondata_instance() - >>> sio = StringIO() - >>> write_commondata_data(cd,sio) - >>> print(sio.getvalue()) - - """ - header = f"{commondata.setname} {commondata.nsys} {commondata.ndata}\n" - buffer.write(header) - commondata.commondata_table.to_csv(buffer, sep="\t", header=None) - - -def write_commondata_to_file(commondata, path): - """ - write commondata table to file - """ - with open(path, "w") as file: - write_commondata_data(commondata, file) - - -def write_systype_data(commondata, buffer): - """ - write systype table to buffer, this can be a memory map, - compressed archive or strings (using for instance StringIO) - - - Parameters - ---------- - - commondata : validphys.coredata.CommonData - - buffer : memory map, compressed archive or strings - example: StringIO object - - - Example - ------- - >>> from validphys.loader import Loader - >>> from io import StringIO - - >>> l = Loader() - >>> cd = l.check_commondata("NMC").load_commondata_instance() - >>> sio = StringIO() - >>> write_systype_data(cd,sio) - >>> print(sio.getvalue()) - - """ - header = f"{commondata.nsys}\n" - buffer.write(header) - commondata.systype_table.to_csv(buffer, sep="\t", header=None) - - -def write_systype_to_file(commondata, path): - """ - write systype table to file - """ - with open(path, "w") as file: - write_systype_data(commondata, file) \ No newline at end of file diff --git a/validphys2/src/validphys/core.py b/validphys2/src/validphys/core.py index 944056a6d7..694135e3ae 100644 --- a/validphys2/src/validphys/core.py +++ b/validphys2/src/validphys/core.py @@ -8,6 +8,7 @@ """ from __future__ import generator_stop +from collections import namedtuple import re import enum import functools @@ -23,7 +24,7 @@ from reportengine.compat import yaml from NNPDF import (LHAPDFSet as libNNPDF_LHAPDFSet, - CommonData as LegacyCommonData, + CommonData, FKTable, FKSet, DataSet, @@ -40,9 +41,6 @@ from validphys.lhapdfset import LHAPDFSet from validphys.fkparser import load_fktable from validphys.pineparser import pineappl_reader -from validphys.commondataparser import (peek_commondata_metadata, - get_plot_kinlabels, - parse_commondata,) log = logging.getLogger(__name__) @@ -236,6 +234,46 @@ def get_members(self): return len(self) +kinlabels_latex = CommonData.kinLabel_latex.asdict() +_kinlabels_keys = sorted(kinlabels_latex, key=len, reverse=True) + + +def get_plot_kinlabels(commondata): + """Return the LaTex kinematic labels for a given Commondata""" + key = commondata.process_type + + return kinlabels_latex[key] + +def get_kinlabel_key(process_label): + #Since there is no 1:1 correspondence between latex keys and GetProc, + #we match the longest key such that the proc label starts with it. + l = process_label + try: + return next(k for k in _kinlabels_keys if l.startswith(k)) + except StopIteration as e: + raise ValueError("Could not find a set of kinematic " + "variables matching the process %s Check the " + "labels defined in commondata.cc. " % (l)) from e + +CommonDataMetadata = namedtuple('CommonDataMetadata', ('name', 'nsys', 'ndata', 'process_type')) + +def peek_commondata_metadata(commondatafilename): + """Check some basic properties commondata object without going though the + trouble of processing it on the C++ side""" + with open(commondatafilename) as f: + try: + l = f.readline() + name, nsys_str, ndata_str = l.split() + l = f.readline() + process_type_str = l.split()[1] + except Exception: + log.error(f"Error processing {commondatafilename}") + raise + + return CommonDataMetadata(name, int(nsys_str), int(ndata_str), + get_kinlabel_key(process_type_str)) + + class CommonDataSpec(TupleComp): def __init__(self, datafile, sysfile, plotfiles, name=None, metadata=None): self.datafile = datafile @@ -274,16 +312,9 @@ def __iter__(self): return iter((self.datafile, self.sysfile, self.plotfiles)) @functools.lru_cache() - def load(self): - return parse_commondata(self.datafile, self.sysfile, self.name) - - def load_commondata_instance(self): - """ - load a validphys.core.CommonDataSpec to validphys.core.CommonData - """ - from validphys.commondataparser import load_commondata - - return load_commondata(self) + def load(self)->CommonData: + #TODO: Use better path handling in python 3.6 + return CommonData.ReadFile(str(self.datafile), str(self.sysfile)) @property def plot_kinlabels(self): @@ -441,8 +472,7 @@ def __init__(self, *, name, commondata, fkspecs, thspec, cuts, @functools.lru_cache() def load(self): - """Load the libNNPDF version of the dataset""" - cd = LegacyCommonData.ReadFile(str(self.commondata.datafile), str(self.commondata.sysfile)) + cd = self.commondata.load() fktables = [] for p in self.fkspecs: @@ -478,7 +508,7 @@ def load_commondata(self): loaded_cuts = self.cuts.load() if not (hasattr(loaded_cuts, '_full') and loaded_cuts._full): intmask = [int(ele) for ele in loaded_cuts] - cd = cd.with_cuts(intmask) + cd = CommonData(cd, intmask) return cd def to_unweighted(self): @@ -626,21 +656,6 @@ def load(self): def load_commondata(self): return [d.load_commondata() for d in self.datasets] - - def load_commondata_instance(self): - """ - Given Experiment load list of validphys.coredata.CommonData - objects with cuts already applied - """ - commodata_list = [] - for dataset in self.datasets: - cd = dataset.commondata.load_commondata_instance() - if dataset.cuts is None: - commodata_list.append(cd) - else: - commodata_list.append(cd.with_cuts(dataset.cuts.load())) - return commodata_list - @property def thspec(self): #TODO: Is this good enough? Should we explicitly pass the theory diff --git a/validphys2/src/validphys/coredata.py b/validphys2/src/validphys/coredata.py index ce0a580cc4..5e73e3777f 100644 --- a/validphys2/src/validphys/coredata.py +++ b/validphys2/src/validphys/coredata.py @@ -7,8 +7,6 @@ import dataclasses import numpy as np import pandas as pd -from validphys.commondatawriter import write_systype_to_file, write_commondata_to_file -KIN_NAMES = ["kin1", "kin2", "kin3"] @dataclasses.dataclass(eq=False) @@ -177,7 +175,6 @@ def get_np_fktable(self): return fktable - @dataclasses.dataclass(eq=False) class CFactorData: """ @@ -221,9 +218,15 @@ class CommonData: nkin : int Number of kinematics specified + kinematics : list of str with length nkin + Kinematic variables kin1, kin2, kin3 ... + nsys : int Number of systematics + sysid : list of str with length nsys + ID for systematic + commondata_table : pd.DataFrame Pandas dataframe containing the commondata @@ -232,9 +235,6 @@ class CommonData: for each systematic alongside the uncertainty type (ADD/MULT/RAND) and name (CORR/UNCORR/THEORYCORR/SKIP) - - systematics_table: pd.DataFrame - Panda dataframe containing the table of systematics """ setname: str @@ -248,7 +248,7 @@ class CommonData: def __post_init__(self): self.systematics_table = self.commondata_table.drop( - columns=["process", "data", "stat"] + KIN_NAMES + columns=["process", "kin1", "kin2", "kin3", "data", "stat"] ) def with_cuts(self, cuts): @@ -284,25 +284,10 @@ def with_cuts(self, cuts): new_commondata_table = self.commondata_table.loc[cuts] return dataclasses.replace(self, ndata=newndata, commondata_table=new_commondata_table) - @property - def kinematics(self): - return self.commondata_table[KIN_NAMES] - - def get_kintable(self): - return self.kinematics.values - @property def central_values(self): return self.commondata_table["data"] - def with_central_value(self, cv): - tb = self.commondata_table.copy() - tb["data"] = cv - return dataclasses.replace(self, commondata_table=tb) - - def get_cv(self): - return self.central_values.values - @property def stat_errors(self): return self.commondata_table["stat"] @@ -368,19 +353,3 @@ def systematic_errors(self, central_values=None): central_values = self.central_values.to_numpy() converted_mult_errors = self.multiplicative_errors * central_values[:, np.newaxis] / 100 return pd.concat((self.additive_errors, converted_mult_errors), axis=1) - - - def export(self, path): - """Export the data, and error types - Use the same format as libNNPDF: - - - A DATA_.dat file with the dataframe of accepted points - - A systypes/STYPES_.dat file with the error types - """ - - dat_path = path / f"DATA_{self.setname}.dat" - sys_path = path / "systypes" / f"SYSTYPE_{self.setname}_DEFAULT.dat" - sys_path.parent.mkdir(exist_ok=True) - - write_systype_to_file(self, sys_path) - write_commondata_to_file(self, dat_path) diff --git a/validphys2/src/validphys/dataplots.py b/validphys2/src/validphys/dataplots.py index 7644756497..41248c842c 100644 --- a/validphys2/src/validphys/dataplots.py +++ b/validphys2/src/validphys/dataplots.py @@ -26,7 +26,6 @@ from validphys.plotoptions import get_info, kitable, transform_result from validphys import plotutils from validphys.utils import sane_groupby_iter, split_ranges, scale_from_grid -from validphys.coredata import KIN_NAMES log = logging.getLogger(__name__) @@ -267,7 +266,7 @@ def _plot_fancy_impl(results, commondata, cutlist, table[('err', i)] = err else: table[cvcol] = cv/norm_cv - table[('err', i)] = np.abs(err/norm_cv) + table[('err', i)] = err/norm_cv cvcols.append(cvcol) @@ -961,13 +960,12 @@ def plot_positivity(pdfs, positivity_predictions_for_pdfs, posdataset, pos_use_k ax.axhline(0, color='red') posset = posdataset.load_commondata() - ndata = posset.ndata + ndata = posset.GetNData() xvals = [] if pos_use_kin: - kin_name = KIN_NAMES[0] - ax.set_xlabel(kin_name) - xvals = posset.kinematics[kin_name].values + ax.set_xlabel('kin1') + xvals = [posset.GetKinematics(i, 0) for i in range(0, ndata)] else: ax.set_xlabel('idat') xvals = np.arange(ndata) diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index 2532ca2c00..913f883e8b 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -9,41 +9,12 @@ import numpy as np -from reportengine.checks import check, make_check +from NNPDF import CommonData +from reportengine.checks import make_argcheck, check, check_positive, make_check from reportengine.compat import yaml import validphys.cuts -from validphys.commondatawriter import ( - write_commondata_to_file, - write_systype_to_file, - ) -log = logging.getLogger(__name__) - -KIN_LABEL = { - "DIS": ("x", "Q2", "y"), - "DYP": ("y", "M2", "sqrts"), - "JET": ("eta", "p_T2", "sqrts"), - "DIJET": ("eta", "m_12", "sqrts"), - "PHT": ("eta_gamma", "E_{T,gamma)2", "sqrts"), - "INC": ("0", "mu2", "sqrts"), - "EWK_RAP": ("etay", "M2", "sqrts"), - "EWK_PT": ("p_T", "M2", "sqrts"), - "EWK_PTRAP": ("etay", "p_T2", "sqrts"), - "EWK_MLL": ("M_ll", "M_ll2", "sqrts"), - "EWJ_RAP": ("etay", "M2", "sqrts"), - "EWJ_PT": ("p_T", "M2", "sqrt(s)"), - "EWJ_PTRAP": ("etay", "p_T2", "sqrts"), - "EWJ_JRAP": ("etay", "M2", "sqrts"), - "EWJ_JPT": ("p_T", "M2", "sqrts"), - "EWJ_MLL": ("M_ll", "M_ll2", "sqrts"), - "HQP_YQQ": ("yQQ", "mu2", "sqrts"), - "HQP_MQQ": ("MQQ", "mu2", "sqrts"), - "HQP_PTQQ": ("p_TQQ", "mu2", "sqrts"), - "HQP_YQ": ("yQ", "mu2", "sqrts"), - "HQP_PTQ": ("p_TQ", "mu2", "sqrts"), - "HIG_RAP": ("y", "M_H2", "sqrts"), - "SIA": ("z", "Q2", "y"), -} +log = logging.getLogger(__name__) class RuleProcessingError(Exception): """Exception raised when we couldn't process a rule.""" @@ -75,6 +46,12 @@ def default_filter_rules_input(): return yaml.safe_load(read_text(validphys.cuts, "filters.yaml")) +@make_argcheck +def check_rngalgo(rngalgo: int): + """Check rngalgo content""" + check(0 <= rngalgo < 17, + "Invalid rngalgo. Must be int between [0, 16].") + def check_nonnegative(var: str): """Ensure that `var` is positive""" @@ -97,8 +74,27 @@ def export_mask(path, mask): """Dump mask to file""" np.savetxt(path, mask, fmt='%d') - -def filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed): +@check_rngalgo +@check_nonnegative('filterseed') +@check_nonnegative('seed') +def prepare_nnpdf_rng(filterseed:int, rngalgo:int, seed:int): + """Initialise the internal NNPDF RNG, specified by ``rngalgo`` which must + be an integer between 0 and 16, seeded with ``filterseed``. + The RNG can then be subsequently used to i.e generate pseudodata. + """ + try: + from NNPDF import RandomGenerator + except ImportError as e: + logging.error("Generating closure data needs a valid installation of libNNPDF") + raise e + + log.warning("Importing libNNPDF") + log.info("Initialising RNG") + RandomGenerator.InitRNG(rngalgo, seed) + RandomGenerator.GetRNG().SetSeed(filterseed) + +@check_positive('errorsize') +def filter_closure_data(filter_path, data, fakepdf, fakenoise, errorsize, prepare_nnpdf_rng): """Filter closure data. In addition to cutting data points, the data is generated from an underlying ``fakepdf``, applying a shift to the data if ``fakenoise`` is ``True``, which emulates the experimental central values @@ -106,11 +102,13 @@ def filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed): """ log.info('Filtering closure-test data.') - return _filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed) + return _filter_closure_data( + filter_path, data, fakepdf, fakenoise, errorsize) +@check_positive("errorsize") def filter_closure_data_by_experiment( - filter_path, experiments_data, fakepdf, fakenoise, filterseed, experiments_index + filter_path, experiments_data, fakepdf, fakenoise, errorsize, prepare_nnpdf_rng, ): """ Like :py:func:`filter_closure_data` except filters data by experiment. @@ -121,19 +119,10 @@ def filter_closure_data_by_experiment( not reproducible. """ - - res = [] - for exp in experiments_data: - experiment_index = experiments_index[ - experiments_index.isin([exp.name], level=0) - ] - res.append( - _filter_closure_data( - filter_path, exp, fakepdf, fakenoise, filterseed, experiment_index - ) - ) - - return res + return [ + _filter_closure_data(filter_path, exp, fakepdf, fakenoise, errorsize) + for exp in experiments_data + ] def filter_real_data(filter_path, data): @@ -168,7 +157,6 @@ def _write_ds_cut_data(path, dataset): def _filter_real_data(filter_path, data): """Filter real experimental data.""" - total_data_points = 0 total_cut_data_points = 0 for dataset in data.datasets: @@ -176,97 +164,28 @@ def _filter_real_data(filter_path, data): nfull, ncut = _write_ds_cut_data(path, dataset) total_data_points += nfull total_cut_data_points += ncut - dataset.load_commondata().export(path) + dataset.load_commondata().Export(str(path)) return total_data_points, total_cut_data_points -def _filter_closure_data( - filter_path, data, fakepdf, fakenoise, filterseed, experiments_index -): - """ - This function is accessed within a closure test only, that is, the fakedata - namespace has to be True (If fakedata = False, the _filter_real_data function - will be used to write the commondata files). - - The function writes commondata and systypes files within the - name_closure_test/filter folder. - If fakenoise is True, Level 1 type data is written to the filter folder, otherwise - Level 0 data is written. - - Level 1 data is generated from the Level 0 data by adding noise sampled from - the experimental covariance matrix using the validphys.pseudodata.make_replica - function. - - Parameters - ---------- - - filter_path : str - path to filter folder - - data : validphys.core.DataGroupSpec - - fakepdf : validphys.core.PDF - - fakenoise : bool - if fakenoise perform level1 shift of central data values - - filterseed : int - random seed used for the generation of - random noise added to Level 0 data - - - experiments_index : pandas.MultiIndex - - - Returns - ------- - tuple - total data points and points passing the cuts - - """ +def _filter_closure_data(filter_path, data, fakepdf, fakenoise, errorsize): + """Filter closure test data.""" total_data_points = 0 total_cut_data_points = 0 - - # circular import generated @ core.py - from validphys.pseudodata import level0_commondata_wc, make_level1_data - - closure_data = level0_commondata_wc(data, fakepdf) - - for dataset in data.datasets: - #== print number of points passing cuts, make dataset directory and write FKMASK ==# + fakeset = fakepdf.legacy_load() + # Load data, don't cache result + loaded_data = data.load.__wrapped__(data) + # generate level 1 shift if fakenoise + loaded_data.MakeClosure(fakeset, fakenoise) + for j, dataset in enumerate(data.datasets): path = filter_path / dataset.name nfull, ncut = _write_ds_cut_data(path, dataset) - make_dataset_dir(path / "systypes") total_data_points += nfull total_cut_data_points += ncut - - if fakenoise: - #======= Level 1 closure test =======# - - closure_data = make_level1_data( - data, - closure_data, - filterseed, - experiments_index, - ) - - #====== write commondata and systype files ======# - if fakenoise: - log.info("Writing Level1 data") - else: - log.info("Writing Level0 data") - - for cd in closure_data: - path_cd = filter_path / cd.setname / f"DATA_{cd.setname}.dat" - path_sys = ( - filter_path - / cd.setname - / "systypes" - / f"SYSTYPE_{cd.setname}_DEFAULT.dat" - ) - write_commondata_to_file(commondata=cd, path=path_cd) - write_systype_to_file(commondata=cd, path=path_sys) - + loaded_ds = loaded_data.GetSet(j) + if errorsize != 1.0: + loaded_ds.RescaleErrors(errorsize) + loaded_ds.Export(str(path)) return total_data_points, total_cut_data_points @@ -424,14 +343,14 @@ def __init__( f"Could not find dataset {self.dataset}" ) from e if cd.process_type[:3] == "DIS": - self.variables = KIN_LABEL["DIS"] + self.variables = CommonData.kinLabel["DIS"] else: - self.variables = KIN_LABEL[cd.process_type] + self.variables = CommonData.kinLabel[cd.process_type] else: if self.process_type[:3] == "DIS": - self.variables = KIN_LABEL["DIS"] + self.variables = CommonData.kinLabel["DIS"] else: - self.variables = KIN_LABEL[self.process_type] + self.variables = CommonData.kinLabel[self.process_type] if hasattr(self, "local_variables"): if not isinstance(self.local_variables, Mapping): @@ -503,21 +422,19 @@ def __hash__(self): return hash(self._properties) def __call__(self, dataset, idat): - central_value = dataset.get_cv()[idat] - process_name = dataset.commondataproc - + central_value = dataset.GetData(idat) # We return None if the rule doesn't apply. This # is different to the case where the rule does apply, # but the point was cut out by the rule. if ( - dataset.setname != self.dataset - and process_name != self.process_type + dataset.GetSetName() != self.dataset + and dataset.GetProc(idat) != self.process_type and self.process_type != "DIS_ALL" ): return None # Handle the generalised DIS cut - if self.process_type == "DIS_ALL" and not process_name.startswith("DIS"): + if self.process_type == "DIS_ALL" and dataset.GetProc(idat)[:3] != "DIS": return None ns = self._make_point_namespace(dataset, idat) @@ -551,7 +468,7 @@ def __repr__(self): # pragma: no cover def _make_kinematics_dict(self, dataset, idat) -> dict: """Fill in a dictionary with the kinematics for each point""" - kinematics = dataset.kinematics.values[idat] + kinematics = [dataset.GetKinematics(idat, j) for j in range(3)] return dict(zip(self.variables, kinematics)) def _make_point_namespace(self, dataset, idat) -> dict: @@ -571,7 +488,7 @@ def get_cuts_for_dataset(commondata, rules) -> list: Parameters ---------- - commondata: validphys.coredata.CommonData + commondata: NNPDF CommonData spec rules: List[Rule] A list of Rule objects specifying the filters. @@ -598,7 +515,7 @@ def get_cuts_for_dataset(commondata, rules) -> list: dataset = commondata.load() mask = [] - for idat in range(dataset.ndata): + for idat in range(dataset.GetNData()): broken = False for rule in rules: rule_result = rule(dataset, idat) diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index b2d07b7170..d518efcccd 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -323,7 +323,7 @@ def print_different_cuts(fits, test_for_same_cuts): res.write("The following datasets are both included but have different kinematical cuts:\n\n") for (first, second) in test_for_same_cuts: info = get_info(first.commondata) - total_points = first.commondata.ndata + total_points = len(first.commondata.load()) res.write(" - %s:\n" % info.dataset_label) first_len = len(first.cuts.load()) if first.cuts else total_points second_len = len(second.cuts.load()) if second.cuts else total_points @@ -440,10 +440,10 @@ def print_systype_overlap(groups_commondata, group_dataset_inputs_by_metadata): systype_groups = dict() for group_cd, group in zip(groups_commondata, group_dataset_inputs_by_metadata): systype_groups[group["group_name"]] = { - cd.load().systype_table.iloc[i]["name"] + cd.load().GetSys(0, i).name for cd in group_cd for i in range(cd.nsys) - if cd.load().systype_table.iloc[i]["name"] not in allow_list + if cd.load().GetSys(0, i).name not in allow_list } systype_overlap = set() diff --git a/validphys2/src/validphys/mc_gen.py b/validphys2/src/validphys/mc_gen.py index bf94b5b478..ff68d59c88 100644 --- a/validphys2/src/validphys/mc_gen.py +++ b/validphys2/src/validphys/mc_gen.py @@ -5,7 +5,6 @@ Tools to check the pseudo-data MC generation. """ # The functions in this module have been ported to not use libNNPDF -# but is still using it under the hood # it has been a direct port of the libnnpdf dependent structure # so they should not be used as an example import logging diff --git a/validphys2/src/validphys/n3fit_data.py b/validphys2/src/validphys/n3fit_data.py index e057a36a5b..7f50c96992 100644 --- a/validphys2/src/validphys/n3fit_data.py +++ b/validphys2/src/validphys/n3fit_data.py @@ -347,9 +347,8 @@ def pseudodata_table(groups_replicas_indexed_make_replica, replicas): Notes ----- Whilst running ``n3fit``, this action will only be called if - `fitting::savepseudodata` is `true` (as per the default setting) and - replicas are fitted one at a time. The table can be found in the replica - folder i.e. /nnfit/replica_*/ + `fitting::savepseudodata` is `true` and replicas are fitted one at a time. + The table can be found in the replica folder i.e. /nnfit/replica_*/ """ # Concatenate over replicas @@ -361,7 +360,7 @@ def pseudodata_table(groups_replicas_indexed_make_replica, replicas): @table def training_pseudodata(pseudodata_table, training_mask): """Save the training data for the given replica. - Deactivate by setting ``fitting::savepseudodata: False`` + Activate by setting ``fitting::savepseudodata: True`` from within the fit runcard. See Also @@ -374,7 +373,7 @@ def training_pseudodata(pseudodata_table, training_mask): @table def validation_pseudodata(pseudodata_table, training_mask): """Save the training data for the given replica. - Deactivate by setting ``fitting::savepseudodata: False`` + Activate by setting ``fitting::savepseudodata: True`` from within the fit runcard. See Also diff --git a/validphys2/src/validphys/paramfits/dataops.py b/validphys2/src/validphys/paramfits/dataops.py index 81c0992fc4..fed61f8b2f 100644 --- a/validphys2/src/validphys/paramfits/dataops.py +++ b/validphys2/src/validphys/paramfits/dataops.py @@ -57,7 +57,7 @@ def location(self): @property def scale(self): - return (np.diff(np.percentile(self.data, [15.87, 84.13]))).item()/2 + return np.asscalar(np.diff(np.percentile(self.data, [15.87, 84.13])))/2 def get_parabola(asvals, chi2vals): diff --git a/validphys2/src/validphys/pdfbases.py b/validphys2/src/validphys/pdfbases.py index aac0f01ef1..ffa6af5bee 100644 --- a/validphys2/src/validphys/pdfbases.py +++ b/validphys2/src/validphys/pdfbases.py @@ -35,23 +35,6 @@ (21 , r"g"), )) -PIDS_DICT = { - -6: "TBAR", - -5: "BBAR", - -4: "CBAR", - -3: "SBAR", - -2: "UBAR", - -1: "DBAR", - 21: "GLUON", - 1: "D", - 2: "U", - 3: "S", - 4: "C", - 5: "B", - 6: "T", - 22: "PHT", - } - # Canonical ordering of PDG codes (so flavour basis) ALL_FLAVOURS = (-6, -5, -4, -3, -2, -1, 21, 1, 2, 3, 4, 5, 6, 22) DEFAULT_FLARR = (-3,-2,-1,0,1,2,3,4) @@ -493,7 +476,8 @@ def f_(transform_func): default_elements = DEFAULT_FLARR, element_representations=PDG_PARTONS ) -# dicts are ordered in python 3.6+... code shouldn't break if they aren't though +#dicts are oredered in python 3.6+... code shouldn't vreak if they aren't +#though #see Eqs.(56),(57) https://arxiv.org/pdf/0808.1231.pdf for evolution basis definition evolution = LinearBasis.from_mapping({ r'\Sigma' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1 ,'b':1, 'bbar': 1, 't': 1, 'tbar': 1}, @@ -607,14 +591,10 @@ def f_(transform_func): 's': {'s': 1}, 'sbar': {'sbar': 1}, 'c': {'c': 1}, - 'cbar': {'cbar': 1}, 'g': {'g': 1}, }, default_elements=('u', 'ubar', 'd', 'dbar', 's', 'sbar', 'c', 'g', )) -CCBAR_ASYMM_FLAVOUR = copy.deepcopy(FLAVOUR) -CCBAR_ASYMM_FLAVOUR.default_elements=('u', 'ubar', 'd', 'dbar', 's', 'sbar', 'c', 'cbar', 'g') - pdg = LinearBasis.from_mapping({ 'g/10': {'g':0.1}, 'u_{v}': {'u':1, 'ubar':-1}, @@ -773,17 +753,6 @@ def fitbasis_to_NN31IC(flav_info, fitbasis): g = {'sng': 0, 'v': 0, 'v3': 0, 'v8': 0, 't3': 0, 't8': 0, 't15': 0, 'g': 1, 'v15': 0 } v15 = {'sng': 0, 'v': 0, 'v3': 0, 'v8': 0, 't3': 0, 't8': 0, 't15': 0, 'g': 0, 'v15': 1 } - elif fitbasis == 'CCBAR_ASYMM_FLAVOUR': - sng = {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1, 'g': 0 } - v = {'u': 1, 'ubar': -1, 'd': 1, 'dbar': -1, 's': 1, 'sbar': -1, 'c': 1, 'cbar': -1, 'g': 0 } - v3 = {'u': 1, 'ubar': -1, 'd': -1, 'dbar': 1, 's': 0, 'sbar': 0, 'c': 0, 'cbar': 0, 'g': 0 } - v8 = {'u': 1, 'ubar': -1, 'd': 1, 'dbar': -1, 's': -2, 'sbar': 2, 'c': 0, 'cbar': 0, 'g': 0 } - t3 = {'u': 1, 'ubar': 1, 'd': -1, 'dbar': -1, 's': 0, 'sbar': 0, 'c': 0, 'cbar': 0, 'g': 0 } - t8 = {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': -2, 'sbar': -2, 'c': 0, 'cbar': 0, 'g': 0 } - cp = {'u': 0, 'ubar': 0, 'd': 0, 'dbar': 0, 's': 0, 'sbar': 0, 'c': 1, 'cbar': 1, 'g': 0 } - g = {'u': 0, 'ubar': 0, 'd': 0, 'dbar': 0, 's': 0, 'sbar': 0, 'c': 0, 'cbar': 0, 'g': 1 } - v15 = {'u': 1, 'ubar': -1, 'd': 1, 'dbar': -1, 's': 1, 'sbar': -1, 'c': -3, 'cbar': 3, 'g': 0 } - flist = [sng, g, v, v3, v8, t3, t8, cp, v15] mat = [] diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index eda5bc2b36..6ff2ff5773 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -8,10 +8,6 @@ from scipy.integrate import trapezoid from pathlib import Path -from eko.io import EKO -from eko.io.manipulate import xgrid_reshape -from eko.interpolation import XGrid - import yaml from os import remove @@ -179,9 +175,12 @@ def compute_photon_array(self, id): photon_100GeV /= self.xgrid # Load eko and reshape it + from eko.io import EKO with EKO.edit(Path(self.fiatlux_runcard['path_to_eko'])) as eko: # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape + from eko.io.manipulate import xgrid_reshape + from eko.interpolation import XGrid xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) # construct PDFs diff --git a/validphys2/src/validphys/plotoptions/core.py b/validphys2/src/validphys/plotoptions/core.py index f97fee6fac..d96aef7290 100644 --- a/validphys2/src/validphys/plotoptions/core.py +++ b/validphys2/src/validphys/plotoptions/core.py @@ -19,9 +19,8 @@ from reportengine.compat import yaml from reportengine.utils import get_functions, ChainMap -from NNPDF import DataSet +from NNPDF import CommonData, DataSet from validphys.core import CommonDataSpec, DataSetSpec, Cuts, InternalCutsWrapper -from validphys.coredata import CommonData from validphys.plotoptions.utils import apply_to_all_columns, get_subclasses from validphys.plotoptions import labelers, kintransforms, resulttransforms from validphys.utils import parse_yaml_inp @@ -58,11 +57,10 @@ def get_info(data, *, normalize=False, cuts=None, use_plotfiles=True): if cuts is None: if isinstance(data, DataSetSpec): cuts = data.cuts.load() if data.cuts else None - elif hasattr(cuts, 'load'): + elif isinstance(cuts, (Cuts, InternalCutsWrapper)): cuts = cuts.load() - - if cuts is not None and not len(cuts): - raise NotImplementedError("No point passes the cuts. Cannot retieve info") + elif not cuts: + cuts = None if isinstance(data, DataSetSpec): data = data.commondata @@ -177,11 +175,6 @@ def from_commondata(cls, commondata, cuts=None, normalize=False): kinlabels = commondata.plot_kinlabels kinlabels = plot_params['kinematics_override'].new_labels(*kinlabels) - if "extra_labels" in plot_params and cuts is not None: - cut_extra_labels ={ - k: [v[i] for i in cuts] for k, v in plot_params["extra_labels"].items() - } - plot_params["extra_labels"] = cut_extra_labels return cls(kinlabels=kinlabels, **plot_params) diff --git a/validphys2/src/validphys/pseudodata.py b/validphys2/src/validphys/pseudodata.py index 67d0144a7b..142072e0d6 100644 --- a/validphys2/src/validphys/pseudodata.py +++ b/validphys2/src/validphys/pseudodata.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd -from validphys.covmats import INTRA_DATASET_SYS_NAME, sqrt_covmat, dataset_inputs_covmat_from_systematics +from validphys.covmats import INTRA_DATASET_SYS_NAME, sqrt_covmat from reportengine import collect @@ -236,139 +236,10 @@ def indexed_make_replica(groups_index, make_replica): return pd.DataFrame(make_replica, index=groups_index, columns=["data"]) -def level0_commondata_wc(data, fakepdf): - """ - Given a validphys.core.DataGroupSpec object, load commondata and - generate a new commondata instance with central values replaced - by fakepdf prediction - - Parameters - ---------- - - data : validphys.core.DataGroupSpec - - fakepdf: validphys.core.PDF - - Returns - ------- - list - list of validphys.coredata.CommonData instances corresponding to - all datasets within one experiment. The central value is replaced - by Level 0 fake data. - - Example - ------- - >>> from validphys.api import API - >>> API.level0_commondata_wc(dataset_inputs = [{"dataset":"NMC"}], use_cuts="internal", theoryid=200,fakepdf = "NNPDF40_nnlo_as_01180") - - [CommonData(setname='NMC', ndata=204, commondataproc='DIS_NCE', nkin=3, nsys=16)] - """ - from validphys.covmats import dataset_t0_predictions - level0_commondata_instances_wc = [] - - # ==== Load validphys.coredata.CommonData instance with cuts ====# - - for dataset in data.datasets: - commondata_wc = dataset.commondata.load_commondata_instance() - if dataset.cuts is not None: - cuts = dataset.cuts.load() - commondata_wc = commondata_wc.with_cuts(cuts) - - # == Generate a new CommonData instance with central value given by Level 0 data generated with fakepdf ==# - - t0_prediction = dataset_t0_predictions( - dataset=dataset, t0set=fakepdf - ) # N.B. cuts already applied to th. pred. - level0_commondata_instances_wc.append( - commondata_wc.with_central_value(t0_prediction) - ) - - return level0_commondata_instances_wc - -def make_level1_data( - data, level0_commondata_wc, filterseed, experiments_index -): - """ - Given a list of level0 commondata instances, return the same list - with central values replaced by level1 data - - - Parameters - ---------- - - data : validphys.core.DataGroupSpec - - level0_commondata_wc : list - list of validphys.coredata.CommonData instances corresponding to - all datasets within one experiment. The central value is replaced - by Level 0 fake data. Cuts already applied. - - filterseed: int - random seed used for the generation of Level 1 data - - - Returns - ------- - list - list of validphys.coredata.CommonData instances corresponding to - all datasets within one experiment. The central value is replaced - by Level 1 fake data. - - Example - ------- - - >>> from validphys.api import API - >>> dataset='NMC' - >>> l1_cd = API.make_level1_data(dataset_inputs = [{"dataset":dataset}],use_cuts="internal", theoryid=200, - fakepdf = "NNPDF40_nnlo_as_01180",filterseed=1) - >>> l1_cd - [CommonData(setname='NMC', ndata=204, commondataproc='DIS_NCE', nkin=3, nsys=16)] - """ - # =============== generate experimental covariance matrix ===============# - - dataset_input_list = list(data.dsinputs) - - commondata_wc = data.load_commondata_instance() - - covmat = dataset_inputs_covmat_from_systematics( - commondata_wc, - dataset_input_list, - use_weights_in_covmat=False, - norm_threshold=None, - _list_of_central_values=None, - _only_additive=False, - ) - - # ================== generation of pseudo data ======================# - # = generate pseudo data starting from theory predictions - level1_data = make_replica( - level0_commondata_wc, filterseed, covmat, sep_mult=False, genrep=True - ) - - indexed_level1_data = indexed_make_replica(experiments_index, level1_data) - - # ===== create commondata instances with central values given by pseudo_data =====# - level1_commondata_dict = {c.setname: c for c in level0_commondata_wc} - level1_commondata_instances_wc = [] - - for xx, grp in indexed_level1_data.groupby('dataset'): - level1_commondata_instances_wc.append( - level1_commondata_dict[xx].with_central_value(grp.values) - ) - - return level1_commondata_instances_wc - - -_group_recreate_pseudodata = collect( - 'indexed_make_replica', ('group_dataset_inputs_by_experiment',) -) -_recreate_fit_pseudodata = collect( - '_group_recreate_pseudodata', ('fitreplicas', 'fitenvironment') -) -_recreate_pdf_pseudodata = collect( - '_group_recreate_pseudodata', ('pdfreplicas', 'fitenvironment') -) +_group_recreate_pseudodata = collect('indexed_make_replica', ('group_dataset_inputs_by_experiment',)) +_recreate_fit_pseudodata = collect('_group_recreate_pseudodata', ('fitreplicas', 'fitenvironment')) +_recreate_pdf_pseudodata = collect('_group_recreate_pseudodata', ('pdfreplicas', 'fitenvironment')) fit_tr_masks = collect('replica_training_mask_table', ('fitreplicas', 'fitenvironment')) pdf_tr_masks = collect('replica_training_mask_table', ('pdfreplicas', 'fitenvironment')) @@ -433,4 +304,4 @@ def recreate_pdf_pseudodata(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks) pdf_tr_masks_no_table = collect('replica_training_mask', ('pdfreplicas', 'fitenvironment')) def recreate_pdf_pseudodata_no_table(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks_no_table): - return recreate_pdf_pseudodata(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks_no_table) + return recreate_pdf_pseudodata(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks_no_table) \ No newline at end of file diff --git a/validphys2/src/validphys/tests/conftest.py b/validphys2/src/validphys/tests/conftest.py index b41e7258ad..74eb4315c3 100644 --- a/validphys2/src/validphys/tests/conftest.py +++ b/validphys2/src/validphys/tests/conftest.py @@ -23,8 +23,6 @@ def tmp(tmpdir): # Here define the default config items like the PDF, theory and experiment specs SINGLE_DATAPOINT = {'dataset': 'ATLASTTBARTOT8TEV', 'cfac': ['QCD']} -SINGLE_DATASET = {'dataset': 'NMC'} - DATA = [ {'dataset': 'NMC'}, {'dataset': 'ATLASTTBARTOT', 'cfac':['QCD']}, @@ -53,7 +51,6 @@ def tmp(tmpdir): THEORYID = 162 FIT = "NNPDF40_nnlo_lowprecision" FIT_3REPLICAS = "Basic_runcard_3replicas_lowprec_221130" -FIT_3REPLICAS_DCUTS = "Basic_runcard_3replicas_diffcuts_230221" FIT_ITERATED = "NNPDF40_nnlo_low_precision_iterated" PSEUDODATA_FIT = "pseudodata_test_fit_n3fit_221130" diff --git a/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv b/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv index a46ad353a5..f7ce521f40 100644 --- a/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv +++ b/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv @@ -1,236 +1,236 @@ group dataset id data_central -NMC NMC 16 0.28019164359912174 -NMC NMC 21 0.35499349354805787 -NMC NMC 22 0.36699960383943325 -NMC NMC 27 0.36514731639520354 -NMC NMC 28 0.3796019817168817 -NMC NMC 29 0.3557283684527493 -NMC NMC 34 0.358213498117281 -NMC NMC 35 0.38344229108403 -NMC NMC 36 0.3906176180402464 -NMC NMC 40 0.3565478540570389 -NMC NMC 41 0.35449556242092534 -NMC NMC 42 0.3477428857498088 -NMC NMC 46 0.34058587588303324 -NMC NMC 47 0.37528607055424945 -NMC NMC 48 0.35810702495585167 -NMC NMC 51 0.35948420833368 -NMC NMC 52 0.3252481995431795 -NMC NMC 53 0.34839699344491915 -NMC NMC 54 0.3305599715630647 -NMC NMC 57 0.33854839805110815 -NMC NMC 58 0.3475248192617279 -NMC NMC 59 0.3224033407693148 -NMC NMC 60 0.3557230755468301 -NMC NMC 63 0.3076864562936749 -NMC NMC 64 0.3249652996062988 -NMC NMC 65 0.3225373982130234 -NMC NMC 68 0.2698593749956943 -NMC NMC 69 0.28697589129899725 -NMC NMC 83 0.3314942368553224 -NMC NMC 84 0.3497035726336489 -NMC NMC 87 0.3561247503065496 -NMC NMC 88 0.36048612554048426 -NMC NMC 89 0.342867775009473 -NMC NMC 91 0.3462149531948183 -NMC NMC 92 0.33322645114254057 -NMC NMC 93 0.3496968402988778 -NMC NMC 94 0.37224312344590277 -NMC NMC 95 0.3501229381653187 -NMC NMC 97 0.34862218791470695 -NMC NMC 98 0.3513864997869911 -NMC NMC 99 0.3463189635741315 -NMC NMC 100 0.34999374714067055 -NMC NMC 101 0.3219866280166407 -NMC NMC 104 0.3136568526784361 -NMC NMC 105 0.3238255813447048 -NMC NMC 106 0.34905991139783304 -NMC NMC 107 0.34365171884104645 -NMC NMC 108 0.2700951102701827 -NMC NMC 110 0.3516749264080651 -NMC NMC 111 0.3420426414588959 -NMC NMC 112 0.3368571659898279 -NMC NMC 113 0.32766489680568944 -NMC NMC 114 0.3309229352262125 -NMC NMC 115 0.3087559452541232 -NMC NMC 116 0.30434091887231657 -NMC NMC 117 0.31985278442485376 -NMC NMC 118 0.3081369779237656 -NMC NMC 119 0.32428991893790204 -NMC NMC 120 0.29177949202546344 -NMC NMC 121 0.2986166917278112 -NMC NMC 122 0.28530172048925173 -NMC NMC 123 0.30960790490633666 -NMC NMC 124 0.2838914377920225 -NMC NMC 125 0.327108232453106 -NMC NMC 126 0.30800213354866085 -NMC NMC 127 0.27505741362840286 -NMC NMC 128 0.3057396895330445 -NMC NMC 129 0.2948720021522578 -NMC NMC 130 0.28654594665723143 -NMC NMC 131 0.24998019481392156 -NMC NMC 132 0.23305710852110711 -NMC NMC 133 0.2285884324760879 -NMC NMC 134 0.21022049472555798 -NMC NMC 136 0.14289441509543208 -NMC NMC 137 0.1467309459465361 -NMC NMC 147 0.3783814687867924 -NMC NMC 148 0.3660143252628356 -NMC NMC 152 0.37316750851562697 -NMC NMC 153 0.38611716465278784 -NMC NMC 154 0.3698410284359639 -NMC NMC 157 0.36961892116181694 -NMC NMC 158 0.40918582365959344 -NMC NMC 159 0.3772237079249307 -NMC NMC 160 0.36695342000619174 -NMC NMC 161 0.37837880076113106 -NMC NMC 162 0.3574940348440908 -NMC NMC 163 0.38209182150349047 -NMC NMC 164 0.38555197565482335 -NMC NMC 165 0.38435365356895423 -NMC NMC 166 0.39523350385236933 -NMC NMC 167 0.352962329300064 -NMC NMC 168 0.33498853883029145 -NMC NMC 169 0.3799963472642203 -NMC NMC 170 0.3626711192860542 -NMC NMC 171 0.3726539470995713 -NMC NMC 172 0.38205670828934024 -NMC NMC 173 0.3624890112249862 -NMC NMC 174 0.351766590223616 -NMC NMC 175 0.38120620112890435 -NMC NMC 176 0.376704707091722 -NMC NMC 177 0.35767966843982835 -NMC NMC 178 0.3739917821379859 -NMC NMC 179 0.3779713226850912 -NMC NMC 180 0.3540178478852946 -NMC NMC 181 0.3563337922341492 -NMC NMC 182 0.3495895799725437 -NMC NMC 183 0.40513857686448185 -NMC NMC 184 0.3737246199455833 -NMC NMC 185 0.34224373934105096 -NMC NMC 186 0.3572968130894208 -NMC NMC 187 0.3513259873499916 -NMC NMC 188 0.36191625620248563 -NMC NMC 189 0.3757225472452201 -NMC NMC 190 0.34496599482267015 -NMC NMC 191 0.3671454519682669 -NMC NMC 192 0.3355832294500817 -NMC NMC 193 0.32611756260750746 -NMC NMC 194 0.3264723091842311 -NMC NMC 195 0.31358299893261243 -NMC NMC 196 0.3063679612675748 -NMC NMC 197 0.3243783766742867 -NMC NMC 198 0.2951522224003636 -NMC NMC 199 0.30478606823022336 -NMC NMC 200 0.3162026195490957 -NMC NMC 201 0.24410226609924124 -NMC NMC 202 0.29437714007416727 -NMC NMC 203 0.29086153651126373 -NMC NMC 204 0.2618537028601226 -NMC NMC 205 0.24746460684350655 -NMC NMC 206 0.22851397088173026 -NMC NMC 207 0.23143418017058778 -NMC NMC 208 0.2083333904728336 -NMC NMC 209 0.1871220049907259 -NMC NMC 210 0.13694037697890943 -NMC NMC 211 0.1410254122612783 -NMC NMC 212 0.10913012127955796 -NMC NMC 221 0.38635187132002613 -NMC NMC 222 0.4142522089627604 -NMC NMC 225 0.40345790673282933 -NMC NMC 226 0.40425695161223724 -NMC NMC 227 0.39659246253718794 -NMC NMC 229 0.4111692386727388 -NMC NMC 230 0.40555697596449714 -NMC NMC 231 0.4035342194456604 -NMC NMC 232 0.4151050115450948 -NMC NMC 233 0.3925537185116469 -NMC NMC 234 0.3991798063407916 -NMC NMC 235 0.42128296044596675 -NMC NMC 236 0.39594826617785617 -NMC NMC 237 0.4132863843007187 -NMC NMC 238 0.4063071052878472 -NMC NMC 239 0.39412536731089337 -NMC NMC 240 0.38212850421302924 -NMC NMC 241 0.3914294452161547 -NMC NMC 242 0.40047857374987483 -NMC NMC 243 0.4008070275555427 -NMC NMC 244 0.40262694743529254 -NMC NMC 245 0.39621958826618797 -NMC NMC 246 0.39102221801161896 -NMC NMC 247 0.3886398830939979 -NMC NMC 248 0.38188378549251667 -NMC NMC 249 0.3988887017211007 -NMC NMC 250 0.39585021194541986 -NMC NMC 251 0.40126477628252327 -NMC NMC 252 0.3964150971271786 -NMC NMC 253 0.3902202435316766 -NMC NMC 254 0.38160232623513723 -NMC NMC 255 0.3839102218492622 -NMC NMC 256 0.39033449259461933 -NMC NMC 257 0.37931861168091624 -NMC NMC 258 0.3709117293446901 -NMC NMC 259 0.3732315660927265 -NMC NMC 260 0.3720221661101343 -NMC NMC 261 0.3517503669287229 -NMC NMC 262 0.36807144164962735 -NMC NMC 263 0.3693458573629922 -NMC NMC 264 0.3592451130180051 -NMC NMC 265 0.35632376909940633 -NMC NMC 266 0.35150220374085334 -NMC NMC 267 0.34496329443474294 -NMC NMC 268 0.33956152225234476 -NMC NMC 269 0.3485667202754675 -NMC NMC 270 0.3391216940975037 -NMC NMC 271 0.33736702477837127 -NMC NMC 272 0.3133373105891819 -NMC NMC 273 0.32982788223049275 -NMC NMC 274 0.3211288416924518 -NMC NMC 275 0.3060547284123289 -NMC NMC 276 0.335436416428492 -NMC NMC 277 0.3073804246714358 -NMC NMC 278 0.294093691234672 -NMC NMC 279 0.2884616638265187 -NMC NMC 280 0.26397391314307916 -NMC NMC 281 0.2444528193940454 -NMC NMC 282 0.27832660956025357 -NMC NMC 283 0.25803828744513574 -NMC NMC 284 0.23389825136222092 -NMC NMC 285 0.2146733551898643 -NMC NMC 286 0.21813290674472682 -NMC NMC 287 0.212151341826686 -NMC NMC 288 0.16141376202106558 -NMC NMC 289 0.13433721471473095 -NMC NMC 290 0.11551212545868192 -NMC NMC 291 0.10901466846254028 -ATLAS ATLASTTBARTOT 0 165.5652425198355 -ATLAS ATLASTTBARTOT 1 245.16706727829992 -ATLAS ATLASTTBARTOT 2 801.771432400817 -CMS CMSZDIFF12 1 2973.4725103674823 -CMS CMSZDIFF12 2 1056.3849990121557 -CMS CMSZDIFF12 3 456.57202617345774 -CMS CMSZDIFF12 4 221.42094319673168 -CMS CMSZDIFF12 5 107.23839580843014 -CMS CMSZDIFF12 6 58.24831168720775 -CMS CMSZDIFF12 7 29.594897586398673 -CMS CMSZDIFF12 11 2828.9994410871873 -CMS CMSZDIFF12 12 1042.0799324021702 -CMS CMSZDIFF12 13 443.3584194436015 -CMS CMSZDIFF12 14 204.46185503728015 -CMS CMSZDIFF12 15 105.84479937812591 -CMS CMSZDIFF12 16 59.79512008357787 -CMS CMSZDIFF12 17 30.620091935371985 -CMS CMSZDIFF12 21 2590.081496607627 -CMS CMSZDIFF12 22 912.4546874257419 -CMS CMSZDIFF12 23 410.2638687475507 -CMS CMSZDIFF12 24 203.38813364945474 -CMS CMSZDIFF12 25 103.36390836336159 -CMS CMSZDIFF12 26 54.7135715358862 -CMS CMSZDIFF12 27 28.40706067537687 -CMS CMSZDIFF12 31 2029.9154133864206 -CMS CMSZDIFF12 32 737.238837202754 -CMS CMSZDIFF12 33 331.45213196343286 -CMS CMSZDIFF12 34 172.74641496097794 -CMS CMSZDIFF12 35 88.99325953676879 -CMS CMSZDIFF12 36 49.525868163359064 -CMS CMSZDIFF12 37 24.187338718035164 +NMC NMC 16 0.4007353489505 +NMC NMC 21 0.3596156369461 +NMC NMC 22 0.3696772738631 +NMC NMC 27 0.3662960538326 +NMC NMC 28 0.3671016068397 +NMC NMC 29 0.3911462081599 +NMC NMC 34 0.3873596614146 +NMC NMC 35 0.3685951249355 +NMC NMC 36 0.3652752448135 +NMC NMC 40 0.3782916051482 +NMC NMC 41 0.3910438628795 +NMC NMC 42 0.3733418699219 +NMC NMC 46 0.3810537704863 +NMC NMC 47 0.3708532044501 +NMC NMC 48 0.3593684720153 +NMC NMC 51 0.3693367172184 +NMC NMC 52 0.3737791523646 +NMC NMC 53 0.3386288622738 +NMC NMC 54 0.3500948571991 +NMC NMC 57 0.3418514086764 +NMC NMC 58 0.3489404958264 +NMC NMC 59 0.3488612312165 +NMC NMC 60 0.3505041034015 +NMC NMC 63 0.3333040875332 +NMC NMC 64 0.2944010260022 +NMC NMC 65 0.3099040341106 +NMC NMC 68 0.2811103711856 +NMC NMC 69 0.2595731013581 +NMC NMC 83 0.332503113838 +NMC NMC 84 0.3062223521694 +NMC NMC 87 0.3477451853918 +NMC NMC 88 0.352597827366 +NMC NMC 89 0.3689837642521 +NMC NMC 91 0.3410075219517 +NMC NMC 92 0.3380026268081 +NMC NMC 93 0.3630165854001 +NMC NMC 94 0.3692037594232 +NMC NMC 95 0.3379900856468 +NMC NMC 97 0.3498975977998 +NMC NMC 98 0.3455821206785 +NMC NMC 99 0.3561806744993 +NMC NMC 100 0.3732490901176 +NMC NMC 101 0.3500032818012 +NMC NMC 104 0.3619815034828 +NMC NMC 105 0.3577747301693 +NMC NMC 106 0.3448546937843 +NMC NMC 107 0.3970272458214 +NMC NMC 108 0.3194277040841 +NMC NMC 110 0.3205419458354 +NMC NMC 111 0.3367281886952 +NMC NMC 112 0.3386950891887 +NMC NMC 113 0.3225662456658 +NMC NMC 114 0.29975210255 +NMC NMC 115 0.3258513025439 +NMC NMC 116 0.3369058963301 +NMC NMC 117 0.3302714626432 +NMC NMC 118 0.3149870988695 +NMC NMC 119 0.2993100089626 +NMC NMC 120 0.3368228206002 +NMC NMC 121 0.3102225912073 +NMC NMC 122 0.3180821969045 +NMC NMC 123 0.2933430928775 +NMC NMC 124 0.3098840948832 +NMC NMC 125 0.3231255065332 +NMC NMC 126 0.2902405904389 +NMC NMC 127 0.2669059748255 +NMC NMC 128 0.2689476065289 +NMC NMC 129 0.2956116242333 +NMC NMC 130 0.288082548032 +NMC NMC 131 0.2226505072116 +NMC NMC 132 0.2198391302795 +NMC NMC 133 0.204048963916 +NMC NMC 134 0.2127696389248 +NMC NMC 136 0.1375618648213 +NMC NMC 137 0.1339894685257 +NMC NMC 147 0.3489969251822 +NMC NMC 148 0.3552399410999 +NMC NMC 152 0.3483599212619 +NMC NMC 153 0.3560033903034 +NMC NMC 154 0.3796442730621 +NMC NMC 157 0.3429642181623 +NMC NMC 158 0.3706142444086 +NMC NMC 159 0.355255409241 +NMC NMC 160 0.352842258266 +NMC NMC 161 0.3539020551091 +NMC NMC 162 0.3521469993182 +NMC NMC 163 0.3632537732513 +NMC NMC 164 0.3725308840612 +NMC NMC 165 0.3576604586946 +NMC NMC 166 0.3816526263355 +NMC NMC 167 0.3688809286074 +NMC NMC 168 0.3385882966462 +NMC NMC 169 0.3387628659638 +NMC NMC 170 0.3760634504695 +NMC NMC 171 0.3785323104752 +NMC NMC 172 0.3632421316272 +NMC NMC 173 0.3623753029918 +NMC NMC 174 0.3362183838135 +NMC NMC 175 0.3432266150754 +NMC NMC 176 0.3640134644566 +NMC NMC 177 0.3576634295374 +NMC NMC 178 0.3597463154209 +NMC NMC 179 0.3603372275518 +NMC NMC 180 0.3177940693752 +NMC NMC 181 0.3620568994759 +NMC NMC 182 0.3701314089822 +NMC NMC 183 0.2978652392955 +NMC NMC 184 0.3541449769575 +NMC NMC 185 0.3448722906877 +NMC NMC 186 0.3412588395999 +NMC NMC 187 0.3570684620173 +NMC NMC 188 0.3286891216355 +NMC NMC 189 0.3477526408617 +NMC NMC 190 0.335703925785 +NMC NMC 191 0.3444529842929 +NMC NMC 192 0.3403578691808 +NMC NMC 193 0.3347353981548 +NMC NMC 194 0.3203688496486 +NMC NMC 195 0.3159487761597 +NMC NMC 196 0.3443688712277 +NMC NMC 197 0.3124097261611 +NMC NMC 198 0.3154190766316 +NMC NMC 199 0.2942857811848 +NMC NMC 200 0.2799806891526 +NMC NMC 201 0.2775840928341 +NMC NMC 202 0.2532591445827 +NMC NMC 203 0.2708438289781 +NMC NMC 204 0.254383734349 +NMC NMC 205 0.2584578349333 +NMC NMC 206 0.2288369641399 +NMC NMC 207 0.2168810713002 +NMC NMC 208 0.2203718114199 +NMC NMC 209 0.1847450793115 +NMC NMC 210 0.1459140382753 +NMC NMC 211 0.1143166792701 +NMC NMC 212 0.1267472998114 +NMC NMC 221 0.377311655587 +NMC NMC 222 0.3786167921399 +NMC NMC 225 0.3880648558735 +NMC NMC 226 0.3835579098506 +NMC NMC 227 0.3700815811285 +NMC NMC 229 0.3784519070427 +NMC NMC 230 0.3982216265502 +NMC NMC 231 0.4017000200127 +NMC NMC 232 0.3922919247205 +NMC NMC 233 0.4124193564822 +NMC NMC 234 0.3590905662548 +NMC NMC 235 0.3967950366239 +NMC NMC 236 0.3971726043601 +NMC NMC 237 0.4020683891529 +NMC NMC 238 0.3814902989618 +NMC NMC 239 0.386741867313 +NMC NMC 240 0.369385519281 +NMC NMC 241 0.3814990929537 +NMC NMC 242 0.381564927955 +NMC NMC 243 0.3948804610387 +NMC NMC 244 0.3839173773074 +NMC NMC 245 0.3802441689135 +NMC NMC 246 0.3752499091384 +NMC NMC 247 0.3974188959257 +NMC NMC 248 0.3836237312771 +NMC NMC 249 0.3911270447289 +NMC NMC 250 0.3877800272259 +NMC NMC 251 0.3629868559682 +NMC NMC 252 0.3754088188714 +NMC NMC 253 0.380648396326 +NMC NMC 254 0.379643330629 +NMC NMC 255 0.3717431381859 +NMC NMC 256 0.3850277979529 +NMC NMC 257 0.357666057149 +NMC NMC 258 0.3843712169454 +NMC NMC 259 0.3677026936208 +NMC NMC 260 0.3655736280598 +NMC NMC 261 0.3441364575008 +NMC NMC 262 0.3972401996537 +NMC NMC 263 0.3645143376461 +NMC NMC 264 0.3510977851517 +NMC NMC 265 0.3539138992202 +NMC NMC 266 0.3493708856316 +NMC NMC 267 0.3320087427879 +NMC NMC 268 0.3284299620477 +NMC NMC 269 0.3321367638445 +NMC NMC 270 0.3209261313501 +NMC NMC 271 0.3152880392914 +NMC NMC 272 0.3067196941747 +NMC NMC 273 0.3051998380667 +NMC NMC 274 0.3111118466168 +NMC NMC 275 0.2900155628937 +NMC NMC 276 0.2883840542816 +NMC NMC 277 0.2756559342559 +NMC NMC 278 0.2855005116247 +NMC NMC 279 0.2662589115441 +NMC NMC 280 0.2559570780724 +NMC NMC 281 0.2504174862552 +NMC NMC 282 0.2307957738123 +NMC NMC 283 0.241680108918 +NMC NMC 284 0.2146215617884 +NMC NMC 285 0.211746450436 +NMC NMC 286 0.2128987035927 +NMC NMC 287 0.1935284098196 +NMC NMC 288 0.131946290253 +NMC NMC 289 0.1180789438391 +NMC NMC 290 0.1049547912561 +NMC NMC 291 0.1106025920512 +ATLAS ATLASTTBARTOT 0 162.5686864865 +ATLAS ATLASTTBARTOT 1 236.1859490199 +ATLAS ATLASTTBARTOT 2 830.4482576268 +CMS CMSZDIFF12 1 2914.124612961 +CMS CMSZDIFF12 2 1080.410756147 +CMS CMSZDIFF12 3 466.257652438 +CMS CMSZDIFF12 4 227.8725970988 +CMS CMSZDIFF12 5 112.9922431625 +CMS CMSZDIFF12 6 60.11148107363 +CMS CMSZDIFF12 7 31.01641512591 +CMS CMSZDIFF12 11 2841.234006315 +CMS CMSZDIFF12 12 1045.712255313 +CMS CMSZDIFF12 13 455.7116015988 +CMS CMSZDIFF12 14 208.6895742617 +CMS CMSZDIFF12 15 111.8661789107 +CMS CMSZDIFF12 16 61.01526352725 +CMS CMSZDIFF12 17 30.58163323228 +CMS CMSZDIFF12 21 2534.025669336 +CMS CMSZDIFF12 22 911.9673411774 +CMS CMSZDIFF12 23 415.3681972017 +CMS CMSZDIFF12 24 198.7630470755 +CMS CMSZDIFF12 25 100.2899391664 +CMS CMSZDIFF12 26 56.45171623313 +CMS CMSZDIFF12 27 28.6103128109 +CMS CMSZDIFF12 31 1990.217512849 +CMS CMSZDIFF12 32 728.4681533061 +CMS CMSZDIFF12 33 332.4189366359 +CMS CMSZDIFF12 34 165.4302576598 +CMS CMSZDIFF12 35 88.19728756934 +CMS CMSZDIFF12 36 47.85196748986 +CMS CMSZDIFF12 37 24.06201644101 diff --git a/validphys2/src/validphys/tests/test_fitdata.py b/validphys2/src/validphys/tests/test_fitdata.py index ccabfd2e69..c4069ba798 100644 --- a/validphys2/src/validphys/tests/test_fitdata.py +++ b/validphys2/src/validphys/tests/test_fitdata.py @@ -1,26 +1,5 @@ from validphys.api import API -from validphys.fitdata import print_systype_overlap, print_different_cuts -from validphys.tests.conftest import FIT_3REPLICAS, FIT_3REPLICAS_DCUTS - - -def test_print_different_cuts(): - """Checks the print_different_cuts functions - using two fits with a different choice of q2min and w2min in the runcard - One of the datasets (SLACP) gets 0 points in in the most restrictive case - The different cuts are: - q2min: 3.49 - 13.49 - w2min: 12.5 - 22.5 - """ - fit_1 = API.fit(fit=FIT_3REPLICAS) - fit_2 = API.fit(fit=FIT_3REPLICAS_DCUTS) - fits = [fit_1, fit_2] - testi = API.test_for_same_cuts(fits=[FIT_3REPLICAS, FIT_3REPLICAS_DCUTS], use_cuts="fromfit") - res = print_different_cuts(fits, testi) - assert "121 out of 260" in res - assert "59 out of 260" in res - assert "33 out of 211" in res - assert "0 out of 211" in res - +from validphys.fitdata import print_systype_overlap def test_print_systype_overlap(): """Test that print_systype_overlap does expected thing diff --git a/validphys2/src/validphys/tests/test_loader.py b/validphys2/src/validphys/tests/test_loader.py index c0385ab109..decf0a020c 100644 --- a/validphys2/src/validphys/tests/test_loader.py +++ b/validphys2/src/validphys/tests/test_loader.py @@ -17,24 +17,18 @@ #The sorted is to appease hypothesis dss = sorted(l.available_datasets - {"PDFEVOLTEST"}) -class MockCuts(): - def __init__(self, arr): - self.arr = arr - def load(self): - return self.arr - @composite -def commondata_and_cuts(draw): +def commodata_and_cuts(draw): cd = l.check_commondata(draw(sampled_from(dss))) ndata = cd.metadata.ndata - # Get a cut mask with at least one selected datapoint - masks = sets(sampled_from(range(ndata)), min_size=1) - mask = sorted(draw(masks)) + #TODO: Maybe upgrade to this + #https://github.com/HypothesisWorks/hypothesis/issues/1115 + mask = sorted(draw(sets(sampled_from(range(ndata))))) return cd, mask -@given(arg=commondata_and_cuts()) +@given(arg=commodata_and_cuts()) @settings(deadline=None) def test_rebuild_commondata_without_cuts(tmp_path_factory, arg): # We need to create a new directory for each call of the test @@ -48,8 +42,8 @@ def test_rebuild_commondata_without_cuts(tmp_path_factory, arg): cutpath = tmp / "cuts.txt" np.savetxt(cutpath, np.asarray(cuts, dtype=int), fmt="%u") cutspec = Cuts(cd, cutpath) - lcd = lcd.with_cuts(cuts) - lcd.export(tmp) + lcd = type(lcd)(lcd, cuts) + lcd.Export(str(tmp)) # We have to reconstruct the name here... with_cuts = tmp / f"DATA_{cd.name}.dat" newpath = tmp / "commondata.dat" @@ -66,13 +60,6 @@ def test_rebuild_commondata_without_cuts(tmp_path_factory, arg): nocuts[cuts] = False assert (lncd.get_cv()[nocuts] == 0).all() -@given(inp=commondata_and_cuts()) -def test_kitable_with_cuts(inp): - cd, cuts = inp - info = get_info(cd, cuts=cuts) - tb = kitable(cd, info, cuts=MockCuts(cuts)) - assert len(tb) == len(cuts) - def test_load_fit(): assert l.check_fit(FIT) with pytest.raises(FitNotFound): diff --git a/validphys2/src/validphys/tests/test_pseudodata.py b/validphys2/src/validphys/tests/test_pseudodata.py index b6149006ff..413975af7c 100644 --- a/validphys2/src/validphys/tests/test_pseudodata.py +++ b/validphys2/src/validphys/tests/test_pseudodata.py @@ -8,14 +8,11 @@ recreation. """ import pandas as pd -import numpy as np -from numpy.testing import assert_allclose import pytest from validphys.api import API -from validphys.tests.conftest import FIT, PSEUDODATA_FIT, THEORYID, SINGLE_DATASET, PDF -from validphys.loader import Loader -from validphys.covmats import dataset_t0_predictions +from validphys.tests.conftest import FIT, PSEUDODATA_FIT + def test_read_fit_pseudodata(): fit_pseudodata = API.read_fit_pseudodata(fit=PSEUDODATA_FIT) @@ -83,29 +80,3 @@ def test_read_matches_recreate(): ) pd.testing.assert_index_equal(read.tr_idx, recreate.tr_idx, check_order=False) pd.testing.assert_index_equal(read.val_idx, recreate.val_idx, check_order=False) - - -def test_level0_commondata_wc(): - """ - check whether level0_commondata_wc and dataset_t0_predictions - coincide - """ - dataset = SINGLE_DATASET - pdfname = PDF - l = Loader() - datasetspec = l.check_dataset(list(dataset.values())[0], theoryid=THEORYID) - t0set = l.check_pdf(pdfname) - - l0_cd = API.level0_commondata_wc( - dataset_inputs=[dataset], - use_cuts="internal", - theoryid=THEORYID, - fakepdf=pdfname, - ) - l0_vals = l0_cd[0].central_values - assert_allclose( - dataset_t0_predictions(dataset=datasetspec, t0set=t0set), - l0_vals, - rtol=1e-07, - atol=0, - ) diff --git a/validphys2/src/validphys/tests/test_weights.py b/validphys2/src/validphys/tests/test_weights.py index 134cf47e2f..0c1043fa44 100644 --- a/validphys2/src/validphys/tests/test_weights.py +++ b/validphys2/src/validphys/tests/test_weights.py @@ -8,15 +8,9 @@ def test_weights_have_same_commondata(weighted_data_witht0_config): data = API.data(**weighted_data_witht0_config) normal, weighted = data.datasets - normalds, weightedds = normal.load_commondata(), weighted.load_commondata() - assert ( - normalds.systematics_table["MULT"].iloc[0][0] - == weightedds.systematics_table["MULT"].iloc[0][0] - ) - assert ( - normalds.systematics_table["ADD"].iloc[0][0] - == weightedds.systematics_table["ADD"].iloc[0][0] - ) + normalds, weightedds = normal.load(), weighted.load() + assert normalds.GetSys(0, 0).mult == weightedds.GetSys(0, 0).mult + assert normalds.GetSys(0, 0).add == weightedds.GetSys(0, 0).add def test_chi2_arithmetic(weighted_data_witht0_internal_cuts_config): diff --git a/validphys2/src/validphys/theorycovariance/tests.py b/validphys2/src/validphys/theorycovariance/tests.py index 6dd14d85c2..1a90302cac 100644 --- a/validphys2/src/validphys/theorycovariance/tests.py +++ b/validphys2/src/validphys/theorycovariance/tests.py @@ -219,7 +219,7 @@ def all_matched_data_lengths(all_matched_datasets): """Returns a list of the data sets lengths.""" lens = [] for rlist in all_matched_datasets: - lens.append(rlist[0].load_commondata().ndata) + lens.append(rlist[0].load_commondata().GetNData()) return lens From cfe7325c3049d3d3302165b801b73cbf880e07fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 27 Feb 2023 15:11:05 +0100 Subject: [PATCH 076/204] Add tests --- validphys2/src/validphys/photon/compute.py | 4 ++++ .../validphys/tests/photon/test_compute.py | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 6ff2ff5773..ac02992faf 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -10,6 +10,7 @@ import yaml from os import remove +import time class Photon: def __init__(self, theoryid, fiatlux_runcard, replicas_id): @@ -170,9 +171,12 @@ def compute_photon_array(self, id): photon PDF at the scale 1 GeV """ # Compute photon PDF + start_time = time.perf_counter() photon_100GeV = np.array([self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid]) photon_100GeV += self.generate_errors() photon_100GeV /= self.xgrid + print("Time to compute photon:", time.perf_counter() - start_time) + # TODO : the different x points could be even computed in parallel # Load eko and reshape it from eko.io import EKO diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 2af723dc2d..750e2b759a 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -75,14 +75,24 @@ def FxQ(self): def test_init(monkeypatch): monkeypatch.setattr(structure_functions, "StructureFunction", fakeStructureFunction) monkeypatch.setattr(structure_functions, "F2LO", fakeF2LO) - monkeypatch.setattr(lhapdf, "mkPDF", lambda *args: 1) + monkeypatch.setattr(lhapdf, "mkPDF", lambda *args: fiatlux_runcard["pdf_name"]) monkeypatch.setattr(fiatlux, "FiatLux", fakeFiatlux) monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) + photon = Photon(faketheory(), fiatlux_runcard, [1,2,3]) + + # test the easy parameters np.testing.assert_equal(photon.replicas_id, [1,2,3]) + np.testing.assert_equal(photon.fiatlux_runcard, fiatlux_runcard) + np.testing.assert_almost_equal(photon.q_in2, 1e4) + np.testing.assert_almost_equal(photon.alpha_em_ref, faketheory().get_description()["alphaqed"]) + + # test masses np.testing.assert_equal(photon.Qmt, np.inf) np.testing.assert_almost_equal(photon.Qmb, 4.92) np.testing.assert_almost_equal(photon.Qmc, 1.3) + + # test set_thresholds_alpha_em np.testing.assert_almost_equal(photon.thresh[5], 91.2) np.testing.assert_almost_equal(photon.thresh[4], 4.92) np.testing.assert_almost_equal(photon.thresh[3], 1.3) @@ -92,4 +102,11 @@ def test_init(monkeypatch): np.testing.assert_equal(len(photon.alpha_thresh), 3) np.testing.assert_equal(len(photon.thresh), 3) + # test betas + vec_beta0 = [-0.5305164769729844, -0.6719875374991137, -0.7073553026306458, -0.8488263631567751] + vec_b1 = [0.17507043740108488, 0.1605510390839295, 0.1538497783221655, 0.1458920311675707] + for nf in range(3, 6+1): + np.testing.assert_allclose(photon.beta0[nf], vec_beta0[nf - 3], rtol=1e-7) + np.testing.assert_allclose(photon.b1[nf], vec_b1[nf - 3], rtol=1e-7) + From fd0daf224e71c1df3d42c8c07fe5d56eb24227c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 27 Feb 2023 15:51:29 +0100 Subject: [PATCH 077/204] Add test on structurefunctions --- .../tests/photon/test_structurefunctions.py | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index 1fb4904318..c2204c18e5 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -1,6 +1,8 @@ import pytest -from validphys.photon.structure_functions import F2LO +import validphys.photon.structure_functions as sf import numpy as np +from pathlib import Path +import pineappl def test_zero_pdfs(): class fake_pdfs: @@ -19,7 +21,7 @@ def xfxQ(self, x, Q): "MaxNfPdf": 5, } - f2lo = F2LO(pdfs, fake_theory) + f2lo = sf.F2LO(pdfs, fake_theory) np.testing.assert_equal(f2lo.Qmt, np.inf) @@ -27,3 +29,46 @@ def xfxQ(self, x, Q): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(f2lo.FxQ(x, Q), 0.) +class FakeFKTable(): + def __init__(self, path): + self.path = path + self.xgrid = np.geomspace(1e-4, 1., 10) + self.qgrid = np.geomspace(1.65, 1000, 10) + + def bin_left(self, i): + if i == 1: + return self.xgrid + if i == 0 : + return self.qgrid + else: + return 0 + + def convolute_with_one(self, pdgid, xfxQ2): + return np.zeros((10, 10)) + +class fakeset(): + def get_entry(self, string): + return 0 + +class fakepdf(): + def __init__(self): + self.ao = 1 + + def xfxQ(self, x, Q): + return 1. + + def xfxQ2(self, x, Q): + return 1.**2 + + def set(self): + return fakeset() + + +def test_F2(monkeypatch): + monkeypatch.setattr(pineappl.fk_table.FkTable, "read", FakeFKTable) + structurefunc = sf.StructureFunction("", fakepdf()) + for x in np.geomspace(1e-4, 1., 10): + for Q in np.geomspace(10, 1000000, 10): + np.testing.assert_allclose(structurefunc.FxQ(x, Q), 0., rtol=1e-5) + + \ No newline at end of file From 7d9293143bbc4402f11546e6864f81654f96fad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 27 Feb 2023 17:25:13 +0100 Subject: [PATCH 078/204] Move import fiatlux --- validphys2/src/validphys/tests/photon/test_compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 750e2b759a..a58cf3daff 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -2,7 +2,6 @@ from validphys.photon.compute import Photon from validphys.photon import structure_functions import lhapdf -import fiatlux import numpy as np from collections import namedtuple @@ -76,6 +75,7 @@ def test_init(monkeypatch): monkeypatch.setattr(structure_functions, "StructureFunction", fakeStructureFunction) monkeypatch.setattr(structure_functions, "F2LO", fakeF2LO) monkeypatch.setattr(lhapdf, "mkPDF", lambda *args: fiatlux_runcard["pdf_name"]) + import fiatlux monkeypatch.setattr(fiatlux, "FiatLux", fakeFiatlux) monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) From 05392d1a3d204bd9d0a6f8bd4fc7d83cd82ec959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 28 Feb 2023 12:00:52 +0100 Subject: [PATCH 079/204] Restore RectBivariateSpline and remove interp2d --- .../src/validphys/photon/structure_functions.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index ac335b2521..b73aade21a 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -1,7 +1,7 @@ from pathlib import Path import pineappl import numpy as np -from scipy.interpolate import interp2d +from scipy.interpolate import RectBivariateSpline class StructureFunction : def __init__(self, path_to_fktable, pdfs): @@ -15,15 +15,22 @@ def __init__(self, path_to_fktable, pdfs): def produce_interpolator(self): x = np.unique(self.fktable.bin_left(1)) q2 = np.unique(self.fktable.bin_left(0)) + self.xmin = min(x) + self.xmax = max(x) + self.qmin = min(np.sqrt(q2)) + self.qmax = max(np.sqrt(q2)) + predictions = self.fktable.convolute_with_one(self.pdgid, self.pdfs.xfxQ2) # here we require that the (x,Q2) couples that we passed # to pinefarm is a rectangular matrix - grid2D = predictions.reshape(len(x),len(q2)).T - self.interpolator = interp2d(x, q2, grid2D, kind='cubic', fill_value=0) + grid2D = predictions.reshape(len(x), len(q2)) + self.interpolator = RectBivariateSpline(x, q2, grid2D) def FxQ(self, x, Q): - return self.interpolator(x, Q**2)[0] + if x < self.xmin or x > self.xmax or Q < self.qmin or Q > self.qmax : + return 0. + return self.interpolator(x, Q**2)[0, 0] class F2LO : def __init__(self, pdfs, theory): From 6d0bd2304ee50a65a453ff45a6614ae22d74bb67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 7 Mar 2023 12:48:07 +0100 Subject: [PATCH 080/204] Fix thresholds for photon --- validphys2/src/validphys/photon/compute.py | 28 +++++++++---------- .../validphys/photon/structure_functions.py | 18 ++++++------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index ac02992faf..f4e3f61f16 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -22,15 +22,15 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # parameters for the alphaem running self.alpha_em_ref = self.theory["alphaqed"] self.qref = self.theory["Qref"] - self.Qmc = self.theory["Qmc"] - self.Qmb = self.theory["Qmb"] - self.Qmt = self.theory["Qmt"] + self.kcThr = self.theory["kcThr"] * self.theory["mc"] + self.kbThr = self.theory["kbThr"] * self.theory["mb"] + self.ktThr = self.theory["ktThr"] * self.theory["mt"] if self.theory["MaxNfAs"] <= 5 : - self.Qmt = np.inf + self.ktThr = np.inf if self.theory["MaxNfAs"] <= 4 : - self.Qmb = np.inf + self.kbThr = np.inf if self.theory["MaxNfAs"] <= 3 : - self.Qmc = np.inf + self.kcThr = np.inf self.set_betas() self.set_thresholds_alpha_em() @@ -52,7 +52,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # TODO : remove this dirty trick for i in range(len(replicas_id)): self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) - self.lux[i].InsertInelasticSplitQ([4.18, 1e100]) + self.lux[i].InsertInelasticSplitQ([self.kbThr, self.ktThr if self.theory["MaxNfPdf"]==6 else 1e100]) self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) # TODO : once that #1537 is merged do: @@ -78,11 +78,11 @@ def alpha_em(self, q): alpha_em: float electromagnetic coupling """ - if q < self.Qmc : + if q < self.kcThr : nf = 3 - elif q < self.Qmb : + elif q < self.kbThr : nf = 4 - elif q < self.Qmt : + elif q < self.ktThr : nf = 5 else : nf = 6 @@ -121,12 +121,12 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" - thresh_list = [self.Qmc, self.Qmb, self.Qmt] - if self.qref < self.Qmc : + thresh_list = [self.kcThr, self.kbThr, self.ktThr] + if self.qref < self.kcThr : nfref = 3 - elif self.qref < self.Qmb : + elif self.qref < self.kbThr : nfref = 4 - elif self.qref < self.Qmt : + elif self.qref < self.ktThr : nfref = 5 else : nfref = 6 diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index b73aade21a..1a4b13cb82 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -35,15 +35,15 @@ def FxQ(self, x, Q): class F2LO : def __init__(self, pdfs, theory): self.pdfs = pdfs - self.Qmc = theory["Qmc"] - self.Qmb = theory["Qmb"] - self.Qmt = theory["Qmt"] + self.kcThr = theory["kcThr"] * theory["mc"] + self.kbThr = theory["kbThr"] * theory["mb"] + self.ktThr = theory["ktThr"] * theory["mt"] if theory["MaxNfPdf"] <= 5 : - self.Qmt = np.inf + self.ktThr = np.inf if theory["MaxNfPdf"] <= 4 : - self.Qmb = np.inf + self.kbThr = np.inf if theory["MaxNfPdf"] <= 3 : - self.Qmc = np.inf + self.kcThr = np.inf eu2 = 4. / 9 ed2 = 1. / 9 self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t @@ -65,11 +65,11 @@ def FxQ(self, x, Q): Structure function F2 at LO """ # at LO we use ZM-VFS - if Q < self.Qmc : + if Q < self.kcThr : nf = 3 - elif Q < self.Qmb : + elif Q < self.kbThr : nf = 4 - elif Q < self.Qmt : + elif Q < self.ktThr : nf = 5 else : nf = 6 From 9b83aed19959fa48db41ecf9dffb6c9b22df454b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 7 Mar 2023 14:24:49 +0100 Subject: [PATCH 081/204] Fix test --- validphys2/src/validphys/photon/compute.py | 1 + .../src/validphys/photon/structure_functions.py | 1 + .../validphys/tests/photon/test_structurefunctions.py | 11 +++++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index f4e3f61f16..0a345d51c4 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -22,6 +22,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # parameters for the alphaem running self.alpha_em_ref = self.theory["alphaqed"] self.qref = self.theory["Qref"] + # TODO : maybe they shoud be kDIS instead of k, but usually they are the same self.kcThr = self.theory["kcThr"] * self.theory["mc"] self.kbThr = self.theory["kbThr"] * self.theory["mb"] self.ktThr = self.theory["ktThr"] * self.theory["mt"] diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 1a4b13cb82..050e0f6a7a 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -35,6 +35,7 @@ def FxQ(self, x, Q): class F2LO : def __init__(self, pdfs, theory): self.pdfs = pdfs + # TODO : maybe they shoud be kDIS instead of k, but usually they are the same self.kcThr = theory["kcThr"] * theory["mc"] self.kbThr = theory["kbThr"] * theory["mb"] self.ktThr = theory["ktThr"] * theory["mt"] diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index c2204c18e5..97b711e503 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -15,15 +15,18 @@ def xfxQ(self, x, Q): pdfs = fake_pdfs() fake_theory = { - "Qmc": 1.3, - "Qmb": 5. , - "Qmt": 172., + "mc": 1.3, + "mb": 5. , + "mt": 172., + "kcThr": 1., + "kbThr": 1., + "ktThr": 1., "MaxNfPdf": 5, } f2lo = sf.F2LO(pdfs, fake_theory) - np.testing.assert_equal(f2lo.Qmt, np.inf) + np.testing.assert_equal(f2lo.ktThr, np.inf) for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): From 5a229480b75751b95b1fa0778dd09e37f5bf8883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 7 Mar 2023 14:28:47 +0100 Subject: [PATCH 082/204] Fix test_compute --- .../src/validphys/tests/photon/test_compute.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index a58cf3daff..f2642bb8b5 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -10,10 +10,14 @@ def get_description(self): return { "alphaqed": 0.01, "Qref": 91.2, - "Qmc": 1.3, - "Qmb": 4.92, - "Qmt": 173., + "mc": 1.3, + "mb": 4.92 , + "mt": 172., + "kcThr": 1., + "kbThr": 1., + "ktThr": 1., "MaxNfAs": 5, + "MaxNfPdf": 5, } @@ -88,9 +92,9 @@ def test_init(monkeypatch): np.testing.assert_almost_equal(photon.alpha_em_ref, faketheory().get_description()["alphaqed"]) # test masses - np.testing.assert_equal(photon.Qmt, np.inf) - np.testing.assert_almost_equal(photon.Qmb, 4.92) - np.testing.assert_almost_equal(photon.Qmc, 1.3) + np.testing.assert_equal(photon.ktThr, np.inf) + np.testing.assert_almost_equal(photon.kbThr, 4.92) + np.testing.assert_almost_equal(photon.kcThr, 1.3) # test set_thresholds_alpha_em np.testing.assert_almost_equal(photon.thresh[5], 91.2) From 76a746c0d5c287d7505738ad16d13841d8dbffa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 15 Mar 2023 11:48:35 +0100 Subject: [PATCH 083/204] Added theory 522 to db --- nnpdfcpp/data/theory.db | Bin 102400 -> 102400 bytes validphys2/src/validphys/photon/compute.py | 28 +++++++++--------- .../validphys/photon/structure_functions.py | 22 +++++++------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/nnpdfcpp/data/theory.db b/nnpdfcpp/data/theory.db index 8aba30e1b7b04620870a97e5ba6f6af61029daf5..15ef2e27da09dbfaa563f1a32bca2bab0d21ba12 100644 GIT binary patch delta 133 zcmV;00DAv`pay`T29O&8Tag??5nBK*V*w2h_y7V2@Bjz!55cz)fB}F9v+xHL7y}gu zShEKpC;|gm2w<^sCI*vWC5n2E)KiLfr_y7V2@Bjz!55cz~fB}F9v+xHL7y}dt zV6z7xC<3!FA}0osPzr-!D7RoJ0i;(71Pc2B3i}Sg4S%-~fC0}5v49={xBH3#1qE{r B9ghG2 diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 0a345d51c4..fccec6a5e4 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -23,15 +23,15 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.alpha_em_ref = self.theory["alphaqed"] self.qref = self.theory["Qref"] # TODO : maybe they shoud be kDIS instead of k, but usually they are the same - self.kcThr = self.theory["kcThr"] * self.theory["mc"] - self.kbThr = self.theory["kbThr"] * self.theory["mb"] - self.ktThr = self.theory["ktThr"] * self.theory["mt"] + self.thresh_c = self.theory["kcThr"] * self.theory["mc"] + self.thresh_b = self.theory["kbThr"] * self.theory["mb"] + self.thresh_t = self.theory["ktThr"] * self.theory["mt"] if self.theory["MaxNfAs"] <= 5 : - self.ktThr = np.inf + self.thresh_t = np.inf if self.theory["MaxNfAs"] <= 4 : - self.kbThr = np.inf + self.thresh_b = np.inf if self.theory["MaxNfAs"] <= 3 : - self.kcThr = np.inf + self.thresh_c = np.inf self.set_betas() self.set_thresholds_alpha_em() @@ -53,7 +53,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # TODO : remove this dirty trick for i in range(len(replicas_id)): self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) - self.lux[i].InsertInelasticSplitQ([self.kbThr, self.ktThr if self.theory["MaxNfPdf"]==6 else 1e100]) + self.lux[i].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) # TODO : once that #1537 is merged do: @@ -79,11 +79,11 @@ def alpha_em(self, q): alpha_em: float electromagnetic coupling """ - if q < self.kcThr : + if q < self.thresh_c : nf = 3 - elif q < self.kbThr : + elif q < self.thresh_b : nf = 4 - elif q < self.ktThr : + elif q < self.thresh_t : nf = 5 else : nf = 6 @@ -122,12 +122,12 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" - thresh_list = [self.kcThr, self.kbThr, self.ktThr] - if self.qref < self.kcThr : + thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] + if self.qref < self.thresh_c : nfref = 3 - elif self.qref < self.kbThr : + elif self.qref < self.thresh_b : nfref = 4 - elif self.qref < self.ktThr : + elif self.qref < self.thresh_t : nfref = 5 else : nfref = 6 diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 050e0f6a7a..9eb11634ef 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -28,7 +28,9 @@ def produce_interpolator(self): self.interpolator = RectBivariateSpline(x, q2, grid2D) def FxQ(self, x, Q): - if x < self.xmin or x > self.xmax or Q < self.qmin or Q > self.qmax : + # here we are requiring that the grid that we pass to fiatlux + # has Qmin = 1 (fiatlux doesn't go below Q=1) + if x < self.xmin or Q > self.qmax : return 0. return self.interpolator(x, Q**2)[0, 0] @@ -36,15 +38,15 @@ class F2LO : def __init__(self, pdfs, theory): self.pdfs = pdfs # TODO : maybe they shoud be kDIS instead of k, but usually they are the same - self.kcThr = theory["kcThr"] * theory["mc"] - self.kbThr = theory["kbThr"] * theory["mb"] - self.ktThr = theory["ktThr"] * theory["mt"] + self.thresh_c = theory["kcThr"] * theory["mc"] + self.thresh_b = theory["kbThr"] * theory["mb"] + self.thresh_t = theory["ktThr"] * theory["mt"] if theory["MaxNfPdf"] <= 5 : - self.ktThr = np.inf + self.thresh_t = np.inf if theory["MaxNfPdf"] <= 4 : - self.kbThr = np.inf + self.thresh_b = np.inf if theory["MaxNfPdf"] <= 3 : - self.kcThr = np.inf + self.thresh_c = np.inf eu2 = 4. / 9 ed2 = 1. / 9 self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t @@ -66,11 +68,11 @@ def FxQ(self, x, Q): Structure function F2 at LO """ # at LO we use ZM-VFS - if Q < self.kcThr : + if Q < self.thresh_c : nf = 3 - elif Q < self.kbThr : + elif Q < self.thresh_b : nf = 4 - elif Q < self.ktThr : + elif Q < self.thresh_t : nf = 5 else : nf = 6 From 0321635469bda9dda86c9a6cba237682ec350c89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 16 Mar 2023 12:57:17 +0100 Subject: [PATCH 084/204] Remove path_to_F2, L, eko from runcard --- validphys2/src/validphys/photon/compute.py | 12 ++++++++---- .../src/validphys/photon/structure_functions.py | 3 --- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index fccec6a5e4..c788277355 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -37,12 +37,16 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # structure functions self.qcd_pdfs = [lhapdf.mkPDF(fiatlux_runcard["pdf_name"], id) for id in replicas_id] - path_to_F2 = fiatlux_runcard["path_to_F2"] - path_to_FL = fiatlux_runcard["path_to_FL"] + + # TODO : maybe find a different name for fiatlux_dis_F2 + path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" + path_to_FL = theoryid.path / "fastkernel/fiatlux_dis_FL.pineappl.lz4" f2 = [sf.StructureFunction(path_to_F2, pdfs) for pdfs in self.qcd_pdfs] fl = [sf.StructureFunction(path_to_FL, pdfs) for pdfs in self.qcd_pdfs] f2lo = [sf.F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] + self.path_to_eko_photon = theoryid.path / "eko_photon.tar" + # set fiatlux import fiatlux ff = open('fiatlux_runcard.yml', 'w+') @@ -60,7 +64,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # from evolven3fit_new.cli import XGRID # self.xgrid = XGRID self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) - + self.error_matrix = self.generate_error_matrix() self.produce_interpolators() @@ -181,7 +185,7 @@ def compute_photon_array(self, id): # Load eko and reshape it from eko.io import EKO - with EKO.edit(Path(self.fiatlux_runcard['path_to_eko'])) as eko: + with EKO.edit(self.path_to_eko_photon) as eko: # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape from eko.io.manipulate import xgrid_reshape diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 9eb11634ef..412acde066 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -1,11 +1,9 @@ -from pathlib import Path import pineappl import numpy as np from scipy.interpolate import RectBivariateSpline class StructureFunction : def __init__(self, path_to_fktable, pdfs): - self.path_to_fktable = Path(path_to_fktable) self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs self.pdgid = int(pdfs.set().get_entry("Particle")) @@ -16,7 +14,6 @@ def produce_interpolator(self): x = np.unique(self.fktable.bin_left(1)) q2 = np.unique(self.fktable.bin_left(0)) self.xmin = min(x) - self.xmax = max(x) self.qmin = min(np.sqrt(q2)) self.qmax = max(np.sqrt(q2)) From 8b5b1011b8683d69fc97aa4dc83f0e0996074fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 20 Mar 2023 18:27:35 +0100 Subject: [PATCH 085/204] Revert "Revert "Merge branch 'master' into add-photon"" This reverts commit 42bc035e74afdfb2e78b49b4ef26704513bf8661. --- .github/workflows/fitbot.yml | 4 +- .github/workflows/tests.yml | 5 +- .gitignore | 3 + conda-recipe/conda_build_config.yaml | 2 +- conda-recipe/meta.yaml | 7 +- doc/sphinx/source/releases.rst | 3 + doc/sphinx/source/tutorials/pseudodata.rst | 94 ++-- n3fit/runcards/examples/developing.yml | 1 + n3fit/setup.py | 1 + n3fit/src/evolven3fit_new/__init__.py | 0 n3fit/src/evolven3fit_new/cli.py | 244 +++++++++ n3fit/src/evolven3fit_new/eko_utils.py | 83 ++++ n3fit/src/evolven3fit_new/evolve.py | 220 ++++++++ n3fit/src/evolven3fit_new/utils.py | 201 ++++++++ .../src/n3fit/hyper_optimization/penalties.py | 16 +- n3fit/src/n3fit/io/writer.py | 6 +- n3fit/src/n3fit/scripts/evolven3fit_new.py | 159 ++++++ n3fit/src/n3fit/scripts/n3fit_exec.py | 4 +- n3fit/src/n3fit/tests/regressions/filter.yml | 88 ++++ n3fit/src/n3fit/tests/test_evolven3fit.py | 70 +++ n3fit/src/n3fit/tests/test_penalties.py | 39 ++ validphys2/src/validphys/commondataparser.py | 107 +++- validphys2/src/validphys/commondatawriter.py | 83 ++++ validphys2/src/validphys/core.py | 79 ++- validphys2/src/validphys/coredata.py | 45 +- validphys2/src/validphys/dataplots.py | 10 +- validphys2/src/validphys/filters.py | 207 +++++--- validphys2/src/validphys/fitdata.py | 6 +- validphys2/src/validphys/mc_gen.py | 1 + validphys2/src/validphys/n3fit_data.py | 9 +- validphys2/src/validphys/paramfits/dataops.py | 2 +- validphys2/src/validphys/pdfbases.py | 35 +- validphys2/src/validphys/photon/compute.py | 7 +- validphys2/src/validphys/plotoptions/core.py | 15 +- validphys2/src/validphys/pseudodata.py | 139 +++++- validphys2/src/validphys/tests/conftest.py | 3 + .../test_filter_rebuild_closure_data.csv | 470 +++++++++--------- .../src/validphys/tests/test_fitdata.py | 23 +- validphys2/src/validphys/tests/test_loader.py | 27 +- .../src/validphys/tests/test_pseudodata.py | 33 +- .../src/validphys/tests/test_weights.py | 12 +- .../src/validphys/theorycovariance/tests.py | 2 +- 42 files changed, 2100 insertions(+), 465 deletions(-) create mode 100644 n3fit/src/evolven3fit_new/__init__.py create mode 100644 n3fit/src/evolven3fit_new/cli.py create mode 100644 n3fit/src/evolven3fit_new/eko_utils.py create mode 100644 n3fit/src/evolven3fit_new/evolve.py create mode 100644 n3fit/src/evolven3fit_new/utils.py create mode 100644 n3fit/src/n3fit/scripts/evolven3fit_new.py create mode 100644 n3fit/src/n3fit/tests/regressions/filter.yml create mode 100644 n3fit/src/n3fit/tests/test_evolven3fit.py create mode 100644 n3fit/src/n3fit/tests/test_penalties.py create mode 100644 validphys2/src/validphys/commondatawriter.py diff --git a/.github/workflows/fitbot.yml b/.github/workflows/fitbot.yml index 17f8162877..e859b520be 100644 --- a/.github/workflows/fitbot.yml +++ b/.github/workflows/fitbot.yml @@ -11,7 +11,7 @@ env: N3FIT_MAXNREP: 20 # total number of replicas to fit POSTFIT_NREP: 16 # requested replicas for postfit REFERENCE_SET: NNBOT-90875c07e-2022-11-30 # reference set for vp - CONDA_PY: 39 + CONDA_PY: 310 jobs: build: @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: [3.9] + python-version: ["3.10"] runs-on: ${{ matrix.os }} env: NETRC_FILE: ${{ secrets.NETRC_FILE }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e12b15d9d2..ca417bb12c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,12 +12,13 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.8, 3.9] + python-version: ["3.9", "3.10"] include: - os: ubuntu-latest CONDA_OS: linux-64 - os: macos-latest CONDA_OS: osx-64 + fail-fast: false runs-on: ${{ matrix.os }} env: NETRC_FILE: ${{ secrets.NETRC_FILE }} @@ -67,7 +68,7 @@ jobs: $CONDA_PREFIX/conda-bld/${{matrix.CONDA_OS}}/*.tar.bz2 \ dummy@packages.nnpdf.science:~/packages/conda/${{matrix.CONDA_OS}} - name: Build and upload sphinx documentation to NNPDF server - if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master' + if: startsWith(matrix.os, 'ubuntu') && github.ref == 'refs/heads/master' && matrix.python-version == '3.9' shell: bash -l {0} run: | KEY=$( mktemp ) diff --git a/.gitignore b/.gitignore index 14960be3b1..0fbe0bcd81 100644 --- a/.gitignore +++ b/.gitignore @@ -378,6 +378,9 @@ ENV/ env.bak/ venv.bak/ +conda-build +conda_bld/ + # Spyder project settings .spyderproject .spyproject diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index 327e710892..a15ac5fdef 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,7 +1,7 @@ #For some reason the resolver decides to use an old version of numpy #without this numpy: - - 1.19 + - 1.21 pin_run_as_build: lhapdf: x.x.x diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index e966cabd28..d31a3ba3d5 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -50,11 +50,14 @@ requirements: - recommonmark - sphinx_rtd_theme >0.5 - sphinxcontrib-bibtex - - docutils =0.16 # This dependency is not explicity needed but https://github.com/NNPDF/nnpdf/issues/1220 - curio >=1.0 - - pineappl >=0.5.2 + - pineappl >=0.5.8 + - eko ==0.10.2 + - banana-hep ==0.6.6 + - lz4 >=3.1.10 # see https://github.com/conda-forge/eko-feedstock/issues/10 - fiatlux # [linux] + test: requires: - hypothesis diff --git a/doc/sphinx/source/releases.rst b/doc/sphinx/source/releases.rst index d4b6b7f957..f451f9b843 100644 --- a/doc/sphinx/source/releases.rst +++ b/doc/sphinx/source/releases.rst @@ -21,6 +21,9 @@ The code is tagged to contextualize the versioning, mark significant developments and to mark versions used to produce main results. The significant releases since the code was made public are: +`Version 4.0.6 `_ + The last version that uses C++ objects in n3fit and validphys, and apfel for + PDF evolution. `Version 4.0.5 `_ The last version to support legacy genetic algorithms fits based on C++. `Version 4.0.4 `_ diff --git a/doc/sphinx/source/tutorials/pseudodata.rst b/doc/sphinx/source/tutorials/pseudodata.rst index da35664a69..9bfe25eab8 100644 --- a/doc/sphinx/source/tutorials/pseudodata.rst +++ b/doc/sphinx/source/tutorials/pseudodata.rst @@ -2,9 +2,58 @@ Obtaining the pseudodata used by an ``n3fit`` fit ================================================= +Since version 4.0.7 the Monte Carlo data replicas are saved to disk by default as the fit is performed. +This can be deactivated by setting the ``savepseudodata`` flag to ``False`` under the ``fitting`` namespace in the fit runcard: + +.. code-block :: yaml + + fitting: + savepseudodata: False + +If the ``savepseudodata`` flag is not set to ``False``, the training and validation splits to disk +under files named ``datacuts_theory_fitting_training_pseudodata.csv`` and similarly for the validation +split. These can then be loaded within validphys by leveraging the +:py:func:`validphys.pseudodata.read_fit_pseudodata` action: + +.. code-block :: python + + >>> from validphys.api import API + >>> pseudodata = API.read_fit_pseudodata(fit="pseudodata_test_fit_n3fit") + >>> replica1_info = pseudodata[0] + >>> replica1_info.pseudodata.loc[replica1_info.tr_idx] + replica 1 + group dataset id + ATLAS ATLASZPT8TEVMDIST 1 29.856281 + 3 14.686290 + 4 8.568288 + 5 2.848544 + 6 0.704977 + ... ... + NMC NMCPD 247 0.688019 + 249 0.713272 + 255 0.673997 + 256 0.751973 + 259 0.750572 + + [223 rows x 1 columns] + +With the postfit reshuffling handled instead by :py:func:`validphys.pseudodata.read_pdf_pseudodata`. + + +Reconstructing pseudodata +-------------------------- + +.. warning:: + + The functionality described here is not guaranteed to work between different versions of the code + or its dependencies. Specifically, if anything breaks the pseudodata generation between commits, + e.g. changes to the theory predictions or settings or the random number generator, it is **not + possible to reconstruct previously generated pseudodata** for the code state at such different + commits. + Suppose one has obtained a fit using the ``n3fit`` framework and wants to do some analysis that requires -knowing exactly the data that the neural networks saw during the fitting procedure. The -information is reproducible given the various seeds in the fit runcard. +knowing exactly the data that the neural networks saw during the fitting procedure while this has not been stored. +The information is reproducible given the various seeds in the fit runcard. The 3 random seeds used in the fit are ``trvlseed`` which determines the training/validation splitting, ``nnseed`` which concerns the initialization of the neural netowrks themselves, and finally ``mcseed`` which is the @@ -62,43 +111,4 @@ is a ``namedtuple`` containing the entire dataset, alongside the training and va When running this action from a runcard, it may be worthwhile to use the ``--parallel`` flag when calling validphys. This flag parallelizes dependencies which will compute the pseudodata replicas in an asynchronous manner. This is - advantageous since the MC replica generation is computationally intensive. - -Saving and loading pseudodata ------------------------------ - -It is possible to instead save the Monte Carlo replicas to disk as the fit is performed. This can be achieved by setting -the ``savepseudodata`` flag to ``true`` under the ``fitting`` namespace in the fit runcard: - -.. code-block :: yaml - - fitting: - savepseudodata: true - -This will save the training and validation splits to disk under files named ``datacuts_theory_fitting_training_pseudodata.csv`` -and similarly for the validation split. These can then be loaded within validphys be leveraging the -:py:func:`validphys.pseudodata.read_fit_pseudodata` action: - -.. code-block :: python - - >>> from validphys.api import API - >>> pseudodata = API.read_fit_pseudodata(fit="pseudodata_test_fit_n3fit") - >>> replica1_info = pseudodata[0] - >>> replica1_info.pseudodata.loc[replica1_info.tr_idx] - replica 1 - group dataset id - ATLAS ATLASZPT8TEVMDIST 1 29.856281 - 3 14.686290 - 4 8.568288 - 5 2.848544 - 6 0.704977 - ... ... - NMC NMCPD 247 0.688019 - 249 0.713272 - 255 0.673997 - 256 0.751973 - 259 0.750572 - - [223 rows x 1 columns] - -With the postfit reshuffling be handled instead by the :py:func:`validphys.pseudodata.read_pdf_pseudodata`. + advantageous since the MC replica generation is computationally intensive. \ No newline at end of file diff --git a/n3fit/runcards/examples/developing.yml b/n3fit/runcards/examples/developing.yml index 57c10236bc..2db98eacd5 100644 --- a/n3fit/runcards/examples/developing.yml +++ b/n3fit/runcards/examples/developing.yml @@ -92,6 +92,7 @@ parameters: # This defines the parameter dictionary that is passed to the Model fitting: fitbasis: EVOL # EVOL (7), EVOLQED (8), etc. + savepseudodata: False basis: - {fl: sng, trainable: false, smallx: [1.093, 1.121], largex: [1.486, 3.287]} - {fl: g, trainable: false, smallx: [0.8329, 1.071], largex: [3.084, 6.767]} diff --git a/n3fit/setup.py b/n3fit/setup.py index e68a67f4fe..e81c54809b 100644 --- a/n3fit/setup.py +++ b/n3fit/setup.py @@ -13,6 +13,7 @@ entry_points = {'console_scripts': ['n3fit = n3fit.scripts.n3fit_exec:main', + 'evolven3fit_new = n3fit.scripts.evolven3fit_new:main', 'vp-setupfit = n3fit.scripts.vp_setupfit:main', 'varflavors = n3fit.scripts.varflavors:main', ] diff --git a/n3fit/src/evolven3fit_new/__init__.py b/n3fit/src/evolven3fit_new/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/n3fit/src/evolven3fit_new/cli.py b/n3fit/src/evolven3fit_new/cli.py new file mode 100644 index 0000000000..92ecbdf231 --- /dev/null +++ b/n3fit/src/evolven3fit_new/cli.py @@ -0,0 +1,244 @@ +from . import evolve, utils + +import numpy as np + +XGRID = np.array( + [ + 1.00000000000000e-09, + 1.29708482343957e-09, + 1.68242903474257e-09, + 2.18225315420583e-09, + 2.83056741739819e-09, + 3.67148597892941e-09, + 4.76222862935315e-09, + 6.17701427376180e-09, + 8.01211109898438e-09, + 1.03923870607245e-08, + 1.34798064073805e-08, + 1.74844503691778e-08, + 2.26788118881103e-08, + 2.94163370300835e-08, + 3.81554746595878e-08, + 4.94908707232129e-08, + 6.41938295708371e-08, + 8.32647951986859e-08, + 1.08001422993829e-07, + 1.40086873081130e-07, + 1.81704331793772e-07, + 2.35685551545377e-07, + 3.05703512595323e-07, + 3.96522309841747e-07, + 5.14321257236570e-07, + 6.67115245136676e-07, + 8.65299922973143e-07, + 1.12235875241487e-06, + 1.45577995547683e-06, + 1.88824560514613e-06, + 2.44917352454946e-06, + 3.17671650028717e-06, + 4.12035415232797e-06, + 5.34425265752090e-06, + 6.93161897806315e-06, + 8.99034258238145e-06, + 1.16603030112258e-05, + 1.51228312288769e-05, + 1.96129529349212e-05, + 2.54352207134502e-05, + 3.29841683435992e-05, + 4.27707053972016e-05, + 5.54561248105849e-05, + 7.18958313632514e-05, + 9.31954227979614e-05, + 1.20782367731330e-04, + 1.56497209466554e-04, + 2.02708936328495e-04, + 2.62459799331951e-04, + 3.39645244168985e-04, + 4.39234443000422e-04, + 5.67535660104533e-04, + 7.32507615725537e-04, + 9.44112105452451e-04, + 1.21469317686978e-03, + 1.55935306118224e-03, + 1.99627451141338e-03, + 2.54691493736552e-03, + 3.23597510213126e-03, + 4.09103436509565e-03, + 5.14175977083962e-03, + 6.41865096062317e-03, + 7.95137940306351e-03, + 9.76689999624100e-03, + 1.18876139251364e-02, + 1.43298947643919e-02, + 1.71032279460271e-02, + 2.02100733925079e-02, + 2.36463971369542e-02, + 2.74026915728357e-02, + 3.14652506132444e-02, + 3.58174829282429e-02, + 4.04411060163317e-02, + 4.53171343973807e-02, + 5.04266347950069e-02, + 5.57512610084339e-02, + 6.12736019390519e-02, + 6.69773829498255e-02, + 7.28475589986517e-02, + 7.88703322292727e-02, + 8.50331197801452e-02, + 9.13244910278679e-02, + 9.77340879783772e-02, + 1.04252538208639e-01, + 1.10871366547237e-01, + 1.17582909372878e-01, + 1.24380233801599e-01, + 1.31257062945031e-01, + 1.38207707707289e-01, + 1.45227005135651e-01, + 1.52310263065985e-01, + 1.59453210652156e-01, + 1.66651954293987e-01, + 1.73902938455578e-01, + 1.81202910873333e-01, + 1.88548891679097e-01, + 1.95938145999193e-01, + 2.03368159629765e-01, + 2.10836617429103e-01, + 2.18341384106561e-01, + 2.25880487124065e-01, + 2.33452101459503e-01, + 2.41054536011681e-01, + 2.48686221452762e-01, + 2.56345699358723e-01, + 2.64031612468684e-01, + 2.71742695942783e-01, + 2.79477769504149e-01, + 2.87235730364833e-01, + 2.95015546847664e-01, + 3.02816252626866e-01, + 3.10636941519503e-01, + 3.18476762768082e-01, + 3.26334916761672e-01, + 3.34210651149156e-01, + 3.42103257303627e-01, + 3.50012067101685e-01, + 3.57936449985571e-01, + 3.65875810279643e-01, + 3.73829584735962e-01, + 3.81797240286494e-01, + 3.89778271981947e-01, + 3.97772201099286e-01, + 4.05778573402340e-01, + 4.13796957540671e-01, + 4.21826943574548e-01, + 4.29868141614175e-01, + 4.37920180563205e-01, + 4.45982706956990e-01, + 4.54055383887562e-01, + 4.62137890007651e-01, + 4.70229918607142e-01, + 4.78331176755675e-01, + 4.86441384506059e-01, + 4.94560274153348e-01, + 5.02687589545177e-01, + 5.10823085439086e-01, + 5.18966526903235e-01, + 5.27117688756998e-01, + 5.35276355048428e-01, + 5.43442318565661e-01, + 5.51615380379768e-01, + 5.59795349416641e-01, + 5.67982042055800e-01, + 5.76175281754088e-01, + 5.84374898692498e-01, + 5.92580729444440e-01, + 6.00792616663950e-01, + 6.09010408792398e-01, + 6.17233959782450e-01, + 6.25463128838069e-01, + 6.33697780169485e-01, + 6.41937782762089e-01, + 6.50183010158361e-01, + 6.58433340251944e-01, + 6.66688655093089e-01, + 6.74948840704708e-01, + 6.83213786908386e-01, + 6.91483387159697e-01, + 6.99757538392251e-01, + 7.08036140869916e-01, + 7.16319098046733e-01, + 7.24606316434025e-01, + 7.32897705474271e-01, + 7.41193177421404e-01, + 7.49492647227008e-01, + 7.57796032432224e-01, + 7.66103253064927e-01, + 7.74414231541921e-01, + 7.82728892575836e-01, + 7.91047163086478e-01, + 7.99368972116378e-01, + 8.07694250750291e-01, + 8.16022932038457e-01, + 8.24354950923382e-01, + 8.32690244169987e-01, + 8.41028750298844e-01, + 8.49370409522600e-01, + 8.57715163684985e-01, + 8.66062956202683e-01, + 8.74413732009721e-01, + 8.82767437504206e-01, + 8.91124020497459e-01, + 8.99483430165226e-01, + 9.07845617001021e-01, + 9.16210532771399e-01, + 9.24578130473112e-01, + 9.32948364292029e-01, + 9.41321189563734e-01, + 9.49696562735755e-01, + 9.58074441331298e-01, + 9.66454783914439e-01, + 9.74837550056705e-01, + 9.83222700304978e-01, + 9.91610196150662e-01, + 1.00000000000000e00, + ] +) + + +def cli_evolven3fit_new( + configuration_folder, + q_fin, + q_points, + op_card_info, + theory_card_info, + dump, + load, + force, +): + """Evolves the fitted PDFs. + + The q_grid starts at the Q0 given by the theory but + the last point is q_fin and its number of + points can be specified by q_points. If just one of the + two is not specified by the user, the default grid + will be used. + + If a path is given for the dump option, the eko + will be dumped in that path after the computation. + + If a path is given for the load option, the eko + to be used for the evolution will be loaded from that + path. + + The two options are mutually exclusive. + """ + utils.check_is_a_fit(configuration_folder) + return evolve.evolve_fit( + configuration_folder, + q_fin, + q_points, + op_card_info, + theory_card_info, + force, + load, + dump, + ) diff --git a/n3fit/src/evolven3fit_new/eko_utils.py b/n3fit/src/evolven3fit_new/eko_utils.py new file mode 100644 index 0000000000..076e38144d --- /dev/null +++ b/n3fit/src/evolven3fit_new/eko_utils.py @@ -0,0 +1,83 @@ +from ekobox import gen_theory, gen_op +from eko import run_dglap + +from validphys.loader import Loader +from . import utils + +from typing import Any, Dict, Optional +import logging + +_logger = logging.getLogger(__name__) + + +def construct_eko_cards( + theoryID, + q_fin, + q_points, + x_grid, + op_card_dict: Optional[Dict[str, Any]] = None, + theory_card_dict: Optional[Dict[str, Any]] = None, +): + """ + Return the theory and operator cards used to construct the eko. + theoryID is the ID of the theory for which we are computing the theory and operator card. + q_fin is the final point of the q grid while q_points is the number of points of the grid. + x_grid is the x grid to be used. + op_card_dict and theory_card_dict are optional updates that can be provided respectively to the + operator card and to the theory card. + """ + if theory_card_dict is None: + theory_card_dict = {} + if op_card_dict is None: + op_card_dict = {} + # theory_card construction + theory = Loader().check_theoryID(theoryID).get_description() + theory.pop("FNS") + theory.update(theory_card_dict) + theory_card = gen_theory.gen_theory_card(theory["PTO"], theory["Q0"], update=theory) + # construct operator card + q2_grid = utils.generate_q2grid( + theory["Q0"], + q_fin, + q_points, + {theory["mb"]: theory["kbThr"], theory["mt"]: theory["ktThr"]}, + ) + op_card = gen_op.gen_op_card(q2_grid, update={"interpolation_xgrid": x_grid}) + op_card.update(op_card_dict) + return theory_card, op_card + + +def construct_eko_for_fit(theory_card, op_card, save_path=None): + """ + Construct the eko operator needed for evolution of fitted pdfs + + Parameters + ---------- + theory_card: dict + theory card to use for the eko + op_card: dict + operator card to use for the eko + save_path: pathlib.Path + path where the eko will be saved (the eko + won't be saved if save_path is None) + Returns + ------- + : eko.output.Output + eko operator + """ + # generate eko operator + if save_path is not None: + if not save_path.parent.exists(): + raise FileNotFoundError( + f"Path where eko should be dumped does not exist: {save_path}" + ) + eko_op = run_dglap(theory_card, op_card) + if save_path is not None: + # Here we want to catch all possible exceptions in order to avoid losing the computed eko + try: + _logger.info(f"Saving computed eko to : {save_path}") + eko_op.dump_tar(save_path) + except: + _logger.error(f"Error saving the eko to : {save_path}") + pass + return eko_op diff --git a/n3fit/src/evolven3fit_new/evolve.py b/n3fit/src/evolven3fit_new/evolve.py new file mode 100644 index 0000000000..2cf0180040 --- /dev/null +++ b/n3fit/src/evolven3fit_new/evolve.py @@ -0,0 +1,220 @@ +import pathlib +import logging +import sys + +import numpy as np +from reportengine.compat import yaml + +from ekobox import genpdf, gen_info +from ekomark import apply +from eko import basis_rotation as br +from eko import output +from validphys.loader import Loader + +from . import utils, eko_utils + + +_logger = logging.getLogger(__name__) + +LOG_FILE = "evolven3fit_new.log" + +LOGGING_SETTINGS = {"formatter" : "%(asctime)s %(name)s/%(levelname)s: %(message)s", + "level" : logging.INFO +} + +def evolve_fit( + fit_folder, + q_fin, + q_points, + op_card_dict, + theory_card_dict, + force, + eko_path=None, + dump_eko=None, +): + """ + Evolves all the fitted replica in fit_folder/nnfit + + Parameters + ---------- + + fit_folder: str or pathlib.Path + path to the folder containing the fit + q_fin: float + final point of the q_grid + q_points: int + number of points in the q_grid + op_card_dict: dict + user settings for the op_card + theory_card_dict: dict + user settings for the t_card + force: bool + whether to force the evolution to be done again + eko_path: str or pathlib.Path + path where the eko is stored (if None the eko will be + recomputed) + dump_eko: str or pathlib.Path + path where the eko is dumped (if None the eko won't be + stored) + """ + log_file = pathlib.Path(fit_folder) / LOG_FILE + if log_file.exists(): + if force: + log_file.unlink() + else: + raise FileExistsError( + f"Log file already exists: {log_file} evolven3fit_new has already been run?" + ) + log_file = logging.FileHandler(log_file) + stdout_log = logging.StreamHandler(sys.stdout) + for log in [log_file, stdout_log]: + log.setLevel(LOGGING_SETTINGS["level"]) + log.setFormatter( + logging.Formatter(LOGGING_SETTINGS["formatter"]) + ) + for logger in (_logger, *[logging.getLogger("eko")]): + logger.handlers = [] + logger.setLevel(LOGGING_SETTINGS["level"]) + logger.addHandler(log_file) + logger.addHandler(stdout_log) + + usr_path = pathlib.Path(fit_folder) + initial_PDFs_dict = load_fit(usr_path) + x_grid = np.array( + initial_PDFs_dict[list(initial_PDFs_dict.keys())[0]]["xgrid"] + ).astype(np.float) + theoryID = utils.get_theoryID_from_runcard(usr_path) + theory, op = eko_utils.construct_eko_cards( + theoryID, q_fin, q_points, x_grid, op_card_dict, theory_card_dict + ) + if eko_path is not None: + eko_path = pathlib.Path(eko_path) + _logger.info(f"Loading eko from : {eko_path}") + eko_op = output.Output.load_tar(eko_path) + else: + try: + _logger.info(f"Loading eko from theory {theoryID}") + theory_eko_path = (Loader().check_theoryID(theoryID).path)/'eko.tar' + eko_op = output.Output.load_tar(theory_eko_path) + except FileNotFoundError: + _logger.info(f"eko not found in theory {theoryID}, we will construct it") + eko_op = eko_utils.construct_eko_for_fit(theory, op, dump_eko) + eko_op.xgrid_reshape(targetgrid=x_grid, inputgrid=x_grid) + info = gen_info.create_info_file(theory, op, 1, info_update={}) + info["NumMembers"] = "REPLACE_NREP" + info["ErrorType"] = "replicas" + info["AlphaS_Qs"] = list(eko_op["Q2grid"]) + info["XMin"] = float(x_grid[0]) + info["XMax"] = float(x_grid[-1]) + dump_info_file(usr_path, info) + for replica in initial_PDFs_dict.keys(): + evolved_block = evolve_exportgrid(initial_PDFs_dict[replica], eko_op, x_grid) + dump_evolved_replica( + evolved_block, usr_path, int(replica.removeprefix("replica_")) + ) + # remove folder: + # The function dump_evolved_replica dumps the replica files in a temporary folder + # We need then to remove it after fixing the position of those replica files + (usr_path / "nnfit" / usr_path.stem).rmdir() + + +def load_fit(usr_path): + """ + Loads all the replica pdfs at fitting scale in usr_path and return the exportgrids + + Parameters + ---------- + + usr_path: pathlib.Path + path to the folder containing the fit + + Returns + ------- + + : dict + exportgrids info + """ + nnfitpath = usr_path / "nnfit" + pdf_dict = {} + for yaml_file in nnfitpath.glob("replica_*/*.exportgrid"): + data = yaml.safe_load(yaml_file.read_text(encoding="UTF-8")) + pdf_dict[yaml_file.parent.stem] = data + return pdf_dict + + +def evolve_exportgrid(exportgrid, eko, x_grid): + """ + Evolves the provided exportgrid for the desired replica with the eko and returns the evolved block + + Parameters + ---------- + exportgrid: dict + exportgrid of pdf at fitting scale + eko: eko object + eko operator for evolution + xgrid: list + xgrid to be used as the targetgrid + Returns + ------- + : np.array + evolved block + """ + # construct LhapdfLike object + pdf_grid = np.array(exportgrid["pdfgrid"]).transpose() + pdf_to_evolve = utils.LhapdfLike(pdf_grid, exportgrid["q20"], x_grid) + # evolve pdf + evolved_pdf = apply.apply_pdf(eko, pdf_to_evolve) + # generate block to dump + targetgrid = list(eko["targetgrid"]) + + def ev_pdf(pid, x, Q2): + return x * evolved_pdf[Q2]["pdfs"][pid][targetgrid.index(x)] + + block = genpdf.generate_block( + ev_pdf, + xgrid=targetgrid, + Q2grid=list(eko["Q2grid"]), + pids=br.flavor_basis_pids, + ) + return block + + +def dump_evolved_replica(evolved_block, usr_path, replica_num): + """ + Dump the evolved replica given by evolved_block as the replica num "replica_num" in + the folder usr_path/nnfit/replica_/usr_path.stem.dat + + Parameters + ---------- + evolved_block: numpy.array + block of an evolved PDF + usr_path: pathlib.Path + path of the fit folder + replica_num: int + replica number + """ + path_where_dump = usr_path / "nnfit" / usr_path.stem + # create folder to dump the evolved replica if it does not exist + path_where_dump.mkdir(exist_ok=True) + to_write_in_head = f"PdfType: replica\nFromMCReplica: {replica_num}\n" + genpdf.export.dump_blocks( + path_where_dump, replica_num, [evolved_block], pdf_type=to_write_in_head + ) + # fixing_replica_path + utils.fix_replica_path(usr_path, replica_num) + + +def dump_info_file(usr_path, info): + """ + Dump the info file given by info in the folder usr_path/nnfit/usr_path.stem.info. + + Parameters + ---------- + usr_path: pathlib.Path + path of the fit folder + info: dict + info of the fit + """ + path_where_dump = usr_path / "nnfit" / usr_path.stem + genpdf.export.dump_info(path_where_dump, info) + utils.fix_info_path(usr_path) diff --git a/n3fit/src/evolven3fit_new/utils.py b/n3fit/src/evolven3fit_new/utils.py new file mode 100644 index 0000000000..afbfa6f445 --- /dev/null +++ b/n3fit/src/evolven3fit_new/utils.py @@ -0,0 +1,201 @@ +import shutil +import pathlib +from scipy.interpolate import interp1d +import numpy as np +from reportengine.compat import yaml +from validphys.pdfbases import PIDS_DICT + +DEFAULT_Q2GRID = np.array( + [ + 1.6500000e00, + 1.7874388e00, + 1.9429053e00, + 2.1193749e00, + 2.3204100e00, + 2.5502944e00, + 2.8142025e00, + 3.1184122e00, + 3.4705775e00, + 3.8800751e00, + 4.3584516e00, + 4.9200000e00, + 4.9200000e00, + 5.5493622e00, + 6.2897452e00, + 7.1650687e00, + 8.2052867e00, + 9.4481248e00, + 1.0941378e01, + 1.2745972e01, + 1.4940062e01, + 1.7624572e01, + 2.0930715e01, + 2.5030298e01, + 3.0149928e01, + 3.6590777e01, + 4.4756282e01, + 5.5191298e01, + 6.8637940e01, + 8.6115921e01, + 1.0903923e02, + 1.3938725e02, + 1.7995815e02, + 2.3474820e02, + 3.0952544e02, + 4.1270732e02, + 5.5671861e02, + 7.6011795e02, + 1.0509694e03, + 1.4722574e03, + 2.0906996e03, + 3.0112909e03, + 4.4016501e03, + 6.5333918e03, + 9.8535186e03, + 1.5109614e04, + 2.3573066e04, + 3.7444017e04, + 6.0599320e04, + 1.0000000e05, + ] + ) **2 + +class LhapdfLike: + """ + Class which emulates lhapdf but only for an initial condition PDF (i.e. with only one q2 value). + + Q20 is the fitting scale fo the pdf and it is the only available scale for the objects of this class. + + X_GRID is the grid of x values on top of which the pdf is interpolated. + + PDF_GRID is a dictionary containing the pdf grids at fitting scale for each pid. + """ + + def __init__(self, pdf_grid, q20, x_grid): + self.pdf_grid = pdf_grid + self.q20 = q20 + self.x_grid = x_grid + self.funcs = [ + interp1d(self.x_grid, self.pdf_grid[pid], kind="cubic") + for pid in range(len(PIDS_DICT)) + ] + + def xfxQ2(self, pid, x, q2): + """Return the value of the PDF for the requested pid, x value and, whatever the requested + q2 value, for the fitting q2. + + Parameters + ---------- + + pid: int + pid index of particle + x: float + x-value + q2: float + Q square value + + Returns + ------- + : float + x * PDF value + """ + return self.funcs[list(PIDS_DICT.values()).index(PIDS_DICT[pid])](x) + + def hasFlavor(self, pid): + """Check if the requested pid is in the PDF.""" + return pid in PIDS_DICT + + +def read_runcard(usr_path): + """Read the runcard and return the relevant information for evolven3fit_new""" + return yaml.safe_load((usr_path / "filter.yml").read_text(encoding="UTF-8")) + + +def get_theoryID_from_runcard(usr_path): + """Return the theoryID from the runcard""" + # read the runcard + my_runcard = read_runcard(usr_path) + return my_runcard["theory"]["theoryid"] + + +def generate_q2grid(Q0, Qfin, Q_points, match_dict): + """Generate the q2grid used in the final evolved pdfs or use the default grid if Qfin or Q_points is + not provided. + + match_dict contains the couples (mass : factor) where factor is the number to be multiplied to mass + in order to obtain the relative matching scale. + """ + if Qfin is None and Q_points is None: + return DEFAULT_Q2GRID + elif Qfin is None or Q_points is None: + raise ValueError( + "q_fin and q_points must be specified either both or none of them" + ) + else: + grids = [] + Q_ini = Q0 + num_points_list = [] + for masses in match_dict: + match_scale = masses * match_dict[masses] + # Fraction of the total points to be included in this batch is proportional + # to the log of the ratio between the initial scale and final scale of the + # batch itself (normalized to the same log of the global initial and final + # scales) + if match_scale < Qfin: + frac_of_point = np.log(match_scale / Q_ini) / np.log(Qfin / Q0) + num_points = int( + Q_points * frac_of_point + ) + num_points_list.append(num_points) + grids.append(np.geomspace(Q_ini**2, match_scale**2, num=num_points)) + Q_ini = match_scale + num_points = Q_points - sum(num_points_list) + grids.append(np.geomspace(Q_ini**2, Qfin**2, num=num_points)) + return np.concatenate(grids).tolist() + + +def fix_info_path(usr_path): + """Fix the location of the info file from the folder nnfit/usr_path to + just nnfit + + Examples + -------- + Starting from the info path + initial_info_file_path = "/myfolder/myfit/nnfit/myfit/myfit.info" + and using this function with usr_path = "/myfolder/myfit", one gets + final_info_file_path = "/myfolder/myfit/nnfit/myfit.info" + """ + nnfit = usr_path / "nnfit" + info_file = usr_path.stem + ".info" + info_file_path = nnfit / usr_path.stem / info_file + dest_path_info = nnfit / info_file + shutil.move(info_file_path, dest_path_info) + + +def fix_replica_path(usr_path, replica_num): + """Fix the location of the dat file of the replica from the folder nnfit/usr_path to + just nnfit/replica_ + + Examples + -------- + Starting from the replica 5 path + initial_replica_file_path = "/myfolder/myfit/nnfit/myfit/myfit_5.dat" + and using this function with usr_path = "/myfolder/myfit", one gets + final_replica_file_path = "/myfolder/myfit/nnfit/replica_5/myfit.dat" + """ + nnfit = usr_path / "nnfit" + replica_file_path = nnfit / usr_path.stem / f"{usr_path.stem}_{replica_num:04d}.dat" + dest_path_replica = nnfit / f"replica_{replica_num}" / f"{usr_path.stem}.dat" + shutil.move(replica_file_path, dest_path_replica) + + +def check_is_a_fit(config_folder): + """Check if config_folder is a fit folder, i.e. if it contains the filter.yml file + and the nnfit folder.""" + usr_path = pathlib.Path(config_folder) + filter_path = usr_path / "filter.yml" + if not filter_path.is_file(): + raise ValueError("filter.yaml file not found: the path" + str(filter_path.absolute()) + " is not valid") + nnfitpath = usr_path / "nnfit" + if not nnfitpath.is_dir(): + raise ValueError("nnfit folder not found: provided path is not valid") diff --git a/n3fit/src/n3fit/hyper_optimization/penalties.py b/n3fit/src/n3fit/hyper_optimization/penalties.py index a17067edaf..6e2e2ab12f 100644 --- a/n3fit/src/n3fit/hyper_optimization/penalties.py +++ b/n3fit/src/n3fit/hyper_optimization/penalties.py @@ -20,7 +20,7 @@ """ import numpy as np from validphys import fitveto -from n3fit.vpinterface import N3PDF +from n3fit.vpinterface import N3PDF, integrability_numbers def saturation(pdf_models=None, n=100, min_x=1e-6, max_x=1e-4, flavors=None, **_kwargs): @@ -43,8 +43,8 @@ def saturation(pdf_models=None, n=100, min_x=1e-6, max_x=1e-4, flavors=None, **_ ------- >>> from n3fit.hyper_optimization.penalties import saturation >>> from n3fit.model_gen import pdfNN_layer_generator - >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'cbar', 's', 'sbar']] - >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl) + >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'g', 's', 'sbar']] + >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR") >>> isinstance(saturation(pdf_model, 5), float) True @@ -82,7 +82,7 @@ def patience(stopping_object=None, alpha=1e-4, **_kwargs): >>> from n3fit.hyper_optimization.penalties import patience >>> from types import SimpleNamespace >>> fake_stopping = SimpleNamespace(e_best_chi2=1000, stopping_patience=500, total_epochs=5000, vl_loss=2.42) - >>> patience(None, fake_stopping, alpha=1e-4) + >>> patience(fake_stopping, alpha=1e-4) 3.434143467595683 """ @@ -105,14 +105,14 @@ def integrability(pdf_models=None, **_kwargs): ------- >>> from n3fit.hyper_optimization.penalties import integrability >>> from n3fit.model_gen import pdfNN_layer_generator - >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'cbar', 's', 'sbar']] - >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl) - >>> isinstance(integrability([pdf_model], None), float) + >>> fake_fl = [{'fl' : i, 'largex' : [0,1], 'smallx': [1,2]} for i in ['u', 'ubar', 'd', 'dbar', 'c', 'g', 's', 'sbar']] + >>> pdf_model = pdfNN_layer_generator(nodes=[8], activations=['linear'], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR") + >>> isinstance(integrability(pdf_model), float) True """ pdf_instance = N3PDF(pdf_models) - integ_values = pdf_instance.integrability_numbers() + integ_values = integrability_numbers(pdf_instance) integ_overflow = np.sum(integ_values[integ_values > fitveto.INTEG_THRESHOLD]) if integ_overflow > 50.0: # before reaching an overflow, just give a stupidly big number diff --git a/n3fit/src/n3fit/io/writer.py b/n3fit/src/n3fit/io/writer.py index 9eb81068ce..a670a9e365 100644 --- a/n3fit/src/n3fit/io/writer.py +++ b/n3fit/src/n3fit/io/writer.py @@ -11,7 +11,7 @@ import validphys import n3fit from n3fit import vpinterface - +from evolven3fit_new.cli import XGRID class WriterWrapper: def __init__(self, replica_number, pdf_object, stopping_object, q2, timings): @@ -249,8 +249,8 @@ def storefit( q_0^2 """ # build exportgrid - xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]).reshape(-1, 1) - + xgrid = XGRID.reshape(-1, 1) + result = pdf_object(xgrid, flavours="n3fit").squeeze() lha = evln2lha(result.T).T diff --git a/n3fit/src/n3fit/scripts/evolven3fit_new.py b/n3fit/src/n3fit/scripts/evolven3fit_new.py new file mode 100644 index 0000000000..a1bfe29370 --- /dev/null +++ b/n3fit/src/n3fit/scripts/evolven3fit_new.py @@ -0,0 +1,159 @@ +""" +This module contains the CLI for evolven3fit_new +""" +import logging +import pathlib +from argparse import ArgumentParser +import sys +import numpy as np + +from evolven3fit_new import eko_utils, evolve, cli + +_logger = logging.getLogger(__name__) + + +def construct_eko_parser(subparsers): + parser = subparsers.add_parser( + "produce_eko", + help=( + """Produce the eko for the specified theory. + The q_grid starts at the Q0 given by the theory but the last point is q_fin + and its number of points can be specified by q_points. + The x_grid starts at x_grid_ini and ends at 1.0 and contains the + provided number of points. The eko will be dumped in the provided path.""" + ), + ) + parser.add_argument( + "theoryID", type=int, help="ID of the theory used to produce the eko" + ) + parser.add_argument( + "dump", + type=pathlib.Path, + help="Path where the EKO is dumped", + ) + parser.add_argument( + "-i", + "--x-grid-ini", + default=None, + type=float, + help="Starting point of the x-grid", + ) + parser.add_argument( + "-p", + "--x-grid-points", + default=None, + type=int, + help="Number of points of the x-grid", + ) + return parser + + +def construct_evolven3fit_parser(subparsers): + parser = subparsers.add_parser( + "evolve", + help="Evolves the fitted PDFs. The q_grid starts at the Q0 given by the theory but the last point is q_fin and its number of points can be specified by q_points. If a path is given for the dump option, the eko will be dumped in that path after the computation. If a path is given for the load option, the eko to be used for the evolution will be loaded from that path. The two options are mutually exclusive.", + ) + parser.add_argument( + "configuration_folder", + help="Path to the folder containing the (pre-DGLAP) fit result", + ) + parser.add_argument( + "-l", + "--load", + type=pathlib.Path, + default=None, + help="Path of the EKO to be loaded", + ) + parser.add_argument( + "-d", + "--dump", + type=pathlib.Path, + default=None, + help="Path where the EKO is dumped", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + help="Force the evolution to be done even if it has already been done", + ) + return parser + + +def main(): + parser = ArgumentParser( + description="evolven3fit_new - a script with tools to evolve PDF fits" + ) + parser.add_argument( + "-q", + "--q-fin", + type=float, + default=None, + help="Final q-value of the evolution", + ) + parser.add_argument( + "-p", + "--q-points", + type=int, + default=None, + help="Number of q points for the evolution", + ) + parser.add_argument( + "-n", + "--n-cores", + type=int, + default=1, + help="Number of cores to be used", + ) + subparsers = parser.add_subparsers(title="actions", dest="actions") + eko_parser = construct_eko_parser(subparsers) + evolven3fit_parser = construct_evolven3fit_parser(subparsers) + + args = parser.parse_args() + op_card_info = { + "ev_op_max_order": 10, + "ev_op_iterations": 1, + "n_integration_cores": args.n_cores, + "backward_inversion": "expanded", + } + theory_card_info = {} + if args.actions == "evolve": + cli.cli_evolven3fit_new( + args.configuration_folder, + args.q_fin, + args.q_points, + op_card_info, + theory_card_info, + args.dump, + args.load, + args.force, + ) + elif args.actions == "produce_eko": + stdout_log = logging.StreamHandler(sys.stdout) + stdout_log.setLevel(evolve.LOGGING_SETTINGS["level"]) + stdout_log.setFormatter(evolve.LOGGING_SETTINGS["formatter"]) + for logger_ in (_logger, *[logging.getLogger("eko")]): + logger_.handlers = [] + logger_.setLevel(evolve.LOGGING_SETTINGS["level"]) + logger_.addHandler(stdout_log) + if args.x_grid_ini is None: + if args.x_grid_points is None: + x_grid = cli.XGRID + else: + raise ValueError( + "x_grid_ini and x_grid_points must be specified either both or none of them" + ) + elif args.x_grid_points is None: + raise ValueError( + "x_grid_ini and x_grid_points must be specified either both or none of them" + ) + else: + x_grid = np.geomspace(args.x_grid_ini, 1.0, args.x_grid_points) + tcard, opcard = eko_utils.construct_eko_cards( + args.theoryID, args.q_fin, args.q_points, x_grid, op_card_info, theory_card_info + ) + _ = eko_utils.construct_eko_for_fit(tcard, opcard, args.dump) + + +if __name__ == "__main__": + main() diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 032f2d748c..7ec49e0e00 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -133,7 +133,9 @@ def from_yaml(cls, o, *args, **kwargs): N3FIT_FIXED_CONFIG['actions_'].append(namespace + "performfit") - if file_content["fitting"].get("savepseudodata"): + if fps := file_content["fitting"].get("savepseudodata", True): + if fps != True: + raise TypeError(f"fitting::savepseudodata is neither True nor False ({fps})") if len(kwargs["environment"].replicas) != 1: raise ConfigError( "Cannot request that multiple replicas are fitted and that " diff --git a/n3fit/src/n3fit/tests/regressions/filter.yml b/n3fit/src/n3fit/tests/regressions/filter.yml new file mode 100644 index 0000000000..efb09d94e3 --- /dev/null +++ b/n3fit/src/n3fit/tests/regressions/filter.yml @@ -0,0 +1,88 @@ +# +# Configuration file for n3fit regression tests +# This runcard includes two DIS datasets, one Hadronic dataset +# and two positivity datasets +# + +############################################################ +description: n3fit regression test + +############################################################ +# frac: training fraction +# ewk: apply ewk k-factors +# sys: systematics treatment (see systypes) +dataset_inputs: +- { dataset: NMC, frac: 0.5 } +- { dataset: SLACP, frac: 0.5} +- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 } +- { dataset: CMSTTBARTOT, frac: 1.0, cfac: ['QCD'] } + +############################################################ +datacuts: + t0pdfset: NNPDF40_nnlo_as_01180 # PDF set to generate t0 covmat + q2min : 3.49 # Q2 minimum + w2min : 12.5 # W2 minimum + combocuts : NNPDF31 # NNPDF3.0 final kin. cuts + jetptcut_tev : 0 # jet pt cut for tevatron + jetptcut_lhc : 0 # jet pt cut for lhc + wptcut_lhc : 30.0 # Minimum pT for W pT diff distributions + jetycut_tev : 1e30 # jet rap. cut for tevatron + jetycut_lhc : 1e30 # jet rap. cut for lhc + dymasscut_min: 0 # dy inv.mass. min cut + dymasscut_max: 1e30 # dy inv.mass. max cut + jetcfactcut : 1e30 # jet cfact. cut + +############################################################ +theory: + theoryid: 162 # database id + +############################################################ +genrep: True # True = generate MC replicas, False = use real data +trvlseed: 3 +nnseed: 2 +mcseed: 1 + +load: "weights.h5" + +parameters: # This defines the parameter dictionary that is passed to the Model Trainer + nodes_per_layer: [15, 10, 8] + activation_per_layer: ['sigmoid', 'sigmoid', 'linear'] + initializer: 'glorot_normal' + optimizer: + optimizer_name: 'RMSprop' + learning_rate: 0.00001 + clipnorm: 1.0 + epochs: 1000 + positivity: + multiplier: 1.05 + initial: 1.5 + stopping_patience: 0.30 # percentage of the number of epochs + layer_type: 'dense' + dropout: 0.0 + threshold_chi2: 10.0 + +fitting: + fitbasis: NN31IC # EVOL (7), EVOLQED (8), etc. + basis: + - { fl: sng, smallx: [1.05,1.19], largex: [1.47,2.70] } + - { fl: g, smallx: [0.94,1.25], largex: [0.11,5.87] } + - { fl: v, smallx: [0.54,0.75], largex: [1.15,2.76] } + - { fl: v3, smallx: [0.21,0.57], largex: [1.35,3.08] } + - { fl: v8, smallx: [0.52,0.76], largex: [0.77,3.56] } + - { fl: t3, smallx: [-0.37,1.52], largex: [1.74,3.39] } + - { fl: t8, smallx: [0.56,1.29], largex: [1.45,3.03] } + - { fl: cp, smallx: [0.12,1.19], largex: [1.83,6.70] } + +############################################################ +positivity: + posdatasets: + - { dataset: POSF2U, maxlambda: 1e6 } # Positivity Lagrange Multiplier + - { dataset: POSDYC, maxlambda: 1e5 } + +integrability: + integdatasets: + - {dataset: INTEGXT8, maxlambda: 1e2} + - {dataset: INTEGXT3, maxlambda: 1e2} + +############################################################ +debug: true diff --git a/n3fit/src/n3fit/tests/test_evolven3fit.py b/n3fit/src/n3fit/tests/test_evolven3fit.py new file mode 100644 index 0000000000..278d7212e3 --- /dev/null +++ b/n3fit/src/n3fit/tests/test_evolven3fit.py @@ -0,0 +1,70 @@ +import pathlib +import logging +from numpy.testing import assert_allclose +import numpy as np +from validphys.pdfbases import PIDS_DICT +from evolven3fit_new import utils, eko_utils + +REGRESSION_FOLDER = pathlib.Path(__file__).with_name("regressions") +log = logging.getLogger(__name__) + +def check_consecutive_members(grid, value): + """Check if the first occurrence of value in grid is followed by value again""" + return np.allclose(grid[list(grid).index(value)+1], value) + +def test_utils(): + #Testing the default grid + grid = utils.generate_q2grid(1.65, None, None, {}) + assert_allclose(1.65**2, grid[0]) + assert len(grid) == 50 + # We expect the bottom mass to be repeated twice because it is intended once in 4 flavors and once in 5 flavors. + assert check_consecutive_members(grid, 4.92**2) + #Testing if the points of the matching are correctly repeated twice + matched_grid = utils.generate_q2grid(1.65, 1.0e5, 100, {4.92:2.0, 100:1.0} ) + assert len(matched_grid) == 100 + assert_allclose((1.0e5)**2, matched_grid[-1]) + assert check_consecutive_members(matched_grid, (4.92*2.0)**2) + assert check_consecutive_members(matched_grid, (100.*1.0)**2) + #Testing the fake LHAPDF class + q20 = 1.65**2 + x_grid = np.geomspace(1.0e-7, 1.0, 30) + fake_grids = [[x*(1.-x) for x in x_grid] for pid in PIDS_DICT.keys()] + pdf_grid = dict([(pid,v) for pid,v in zip(range(len(PIDS_DICT)), fake_grids )]) + my_PDF = utils.LhapdfLike(pdf_grid, q20, x_grid) + assert my_PDF.hasFlavor(6) + assert not my_PDF.hasFlavor(0) + for pid in PIDS_DICT: + for x in x_grid: + assert_allclose(my_PDF.xfxQ2(pid, x, q20), x*(1.-x) ) + #Testing read_runcard + runcard = utils.read_runcard(REGRESSION_FOLDER) + assert runcard["description"] == "n3fit regression test" + assert runcard["datacuts"]["t0pdfset"] == "NNPDF40_nnlo_as_01180" + #Testing get_theoryID_from_runcard + ID = utils.get_theoryID_from_runcard(REGRESSION_FOLDER) + assert ID == 162 + +def test_eko_utils(): + #Testing construct eko cards + theoryID = 162 + q_fin = 100 + q_points = 5 + x_grid = [1.e-3, 0.1, 1.0] + pto = 2 + qref = 91.2 + comments = "Test" + n_cores = 6 + t_card, op_card = eko_utils.construct_eko_cards(theoryID, q_fin, q_points, x_grid, {'n_integration_cores' : n_cores, 'interpolation_polynomial_degree' : 2}, {'Comments' : comments}) + assert t_card['Qref'] == qref + assert t_card['PTO'] == pto + assert t_card['Comments'] == comments + assert op_card['n_integration_cores'] == n_cores + assert_allclose(op_card["interpolation_xgrid"], x_grid) + assert_allclose(op_card["Q2grid"][0], 1.65**2) + assert_allclose(op_card["Q2grid"][-1], q_fin**2) + #In this case there are not enough points to have twice the bottom matching scale + assert_allclose(op_card["Q2grid"][1], 4.92**2) + #Testing construct_eko_for_fit + eko_op = eko_utils.construct_eko_for_fit(t_card ,op_card) + assert_allclose(eko_op['interpolation_xgrid'], x_grid) + assert_allclose(list(eko_op['Q2grid']), op_card["Q2grid"]) diff --git a/n3fit/src/n3fit/tests/test_penalties.py b/n3fit/src/n3fit/tests/test_penalties.py new file mode 100644 index 0000000000..3899f1a5a8 --- /dev/null +++ b/n3fit/src/n3fit/tests/test_penalties.py @@ -0,0 +1,39 @@ +""" + Test the penalties for n3fit hyperopt +""" +from types import SimpleNamespace +from n3fit.hyper_optimization.penalties import integrability, patience, saturation +from n3fit.model_gen import pdfNN_layer_generator + + +def test_saturation(): + """Check that the saturation penalty runs and returns a float""" + fake_fl = [ + {"fl": i, "largex": [0, 1], "smallx": [1, 2]} + for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"] + ] + pdf_model = pdfNN_layer_generator( + nodes=[8], activations=["linear"], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR" + ) + assert isinstance(saturation(pdf_model, 5), float) + + +def test_patience(): + """Check that the patience penalty runs and returns a float""" + fake_stopping = SimpleNamespace( + e_best_chi2=1000, stopping_patience=500, total_epochs=5000, vl_chi2=2.42 + ) + res = patience(stopping_object=fake_stopping, alpha=1e-4) + assert isinstance(res, float) + + +def test_integrability_numbers(): + """Check that the integrability penalty runs and returns a float""" + fake_fl = [ + {"fl": i, "largex": [0, 1], "smallx": [1, 2]} + for i in ["u", "ubar", "d", "dbar", "c", "g", "s", "sbar"] + ] + pdf_model = pdfNN_layer_generator( + nodes=[8], activations=["linear"], seed=0, flav_info=fake_fl, fitbasis="FLAVOUR" + ) + assert isinstance(integrability(pdf_model), float) diff --git a/validphys2/src/validphys/commondataparser.py b/validphys2/src/validphys/commondataparser.py index ab2cbaf8c5..80d55c0390 100644 --- a/validphys2/src/validphys/commondataparser.py +++ b/validphys2/src/validphys/commondataparser.py @@ -2,17 +2,47 @@ This module implements parsers for commondata and systype files into useful datastructures, contained in the :py:mod:`validphys.coredata` module, which are not backed by C++ managed memory, and so they can be easily pickled and -interfaces with common Python libraries. The integration of these objects into -the codebase is currently work in progress, and at the moment this module -serves as a proof of concept. +interfaced with common Python libraries. + +The validphys commondata structure is an instance of :py:class:`validphys.coredata.CommonData` """ +import dataclasses from operator import attrgetter +import logging import pandas as pd -from validphys.core import peek_commondata_metadata from validphys.coredata import CommonData +log = logging.getLogger(__name__) + +KINLABEL_LATEX = { + "DIJET": ("\\eta", "$\\m_{1,2} (GeV)", "$\\sqrt{s} (GeV)"), + "DIS": ("$x$", "$Q^2 (GeV^2)$", "$y$"), + "DYP": ("$y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWJ_JPT": ("$p_T (GeV)$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWJ_JRAP": ("$\\eta/y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWJ_MLL": ("$M_{ll} (GeV)$", "$M_{ll}^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWJ_PT": ("$p_T (GeV)$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWJ_PTRAP": ("$\\eta/y$", "$p_T^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWJ_RAP": ("$\\eta/y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWK_MLL": ("$M_{ll} (GeV)$", "$M_{ll}^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWK_PT": ("$p_T$ (GeV)", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWK_PTRAP": ("$\\eta/y$", "$p_T^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "EWK_RAP": ("$\\eta/y$", "$M^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "HIG_RAP": ("$y$", "$M_H^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "HQP_MQQ": ("$M^{QQ} (GeV)$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "HQP_PTQ": ("$p_T^Q (GeV)$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "HQP_PTQQ": ("$p_T^{QQ} (GeV)$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "HQP_YQ": ("$y^Q$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "HQP_YQQ": ("$y^{QQ}$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "INC": ("$0$", "$\\mu^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "JET": ("$\\eta$", "$p_T^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "PHT": ("$\\eta_\\gamma$", "$E_{T,\\gamma}^2 (GeV^2)$", "$\\sqrt{s} (GeV)$"), + "SIA": ("$z$", "$Q^2 (GeV^2)$", "$y$"), +} + + def load_commondata(spec): """ Load the data corresponding to a CommonDataSpec object. @@ -42,12 +72,12 @@ def parse_commondata(commondatafile, systypefile, setname): and systype files. """ # First parse commondata file - commondatatable = pd.read_csv(commondatafile, sep=r'\s+', skiprows=1, header=None) + commondatatable = pd.read_csv(commondatafile, sep=r"\s+", skiprows=1, header=None) # Remove NaNs # TODO: replace commondata files with bad formatting # Build header - commondataheader = ['entry', 'process', 'kin1', 'kin2', 'kin3', 'data', 'stat'] - nsys = (commondatatable.shape[1] - len(commondataheader)) // 2 + commondataheader = ["entry", "process", "kin1", "kin2", "kin3", "data", "stat"] + nsys = (commondatatable.shape[1] - len(commondataheader)) // 2 commondataheader += ["ADD", "MULT"] * nsys commondatatable.columns = commondataheader @@ -55,8 +85,8 @@ def parse_commondata(commondatafile, systypefile, setname): ndata = len(commondatatable) commondataproc = commondatatable["process"][1] # Check for consistency with commondata metadata - cdmetadata = peek_commondata_metadata(commondatafile) - if (setname, nsys, ndata) != attrgetter('name', 'nsys', 'ndata')(cdmetadata): + cdmetadata = peek_commondata_metadata(commondatafile) + if (setname, nsys, ndata) != attrgetter("name", "nsys", "ndata")(cdmetadata): raise ValueError("Commondata table information does not match metadata") # Now parse the systype file @@ -70,18 +100,18 @@ def parse_commondata(commondatafile, systypefile, setname): nkin=3, nsys=nsys, commondata_table=commondatatable, - systype_table=systypetable + systype_table=systypetable, ) + def parse_systypes(systypefile): - """Parses a systype file and returns a pandas dataframe. - """ + """Parses a systype file and returns a pandas dataframe.""" systypeheader = ["sys_index", "type", "name"] try: systypetable = pd.read_csv( systypefile, sep=r"\s+", names=systypeheader, skiprows=1, header=None ) - systypetable.dropna(axis='columns', inplace=True) + systypetable.dropna(axis="columns", inplace=True) # Some datasets e.g. CMSWCHARMRAT have no systematics except pd.errors.EmptyDataError: systypetable = pd.DataFrame(columns=systypeheader) @@ -89,3 +119,54 @@ def parse_systypes(systypefile): systypetable.set_index("sys_index", inplace=True) return systypetable + + +@dataclasses.dataclass(frozen=True) +class CommonDataMetadata: + """Contains metadata information about the data being read""" + + name: str + nsys: int + ndata: int + process_type: str + + +def peek_commondata_metadata(commondatafilename): + """Read some of the properties of the commondata object as a CommonData Metadata + """ + with open(commondatafilename) as f: + try: + l = f.readline() + name, nsys_str, ndata_str = l.split() + l = f.readline() + process_type_str = l.split()[1] + except Exception: + log.error(f"Error processing {commondatafilename}") + raise + + return CommonDataMetadata( + name, int(nsys_str), int(ndata_str), get_kinlabel_key(process_type_str) + ) + + +def get_plot_kinlabels(commondata): + """Return the LaTex kinematic labels for a given Commondata""" + key = commondata.process_type + + return KINLABEL_LATEX[key] + + +def get_kinlabel_key(process_label): + """ + Since there is no 1:1 correspondence between latex keys and GetProc, + we match the longest key such that the proc label starts with it. + """ + l = process_label + try: + return next(k for k in sorted(KINLABEL_LATEX, key=len, reverse=True) if l.startswith(k)) + except StopIteration as e: + raise ValueError( + "Could not find a set of kinematic " + "variables matching the process %s Check the " + "labels defined in commondata.cc. " % (l) + ) from e diff --git a/validphys2/src/validphys/commondatawriter.py b/validphys2/src/validphys/commondatawriter.py new file mode 100644 index 0000000000..07b06b4585 --- /dev/null +++ b/validphys2/src/validphys/commondatawriter.py @@ -0,0 +1,83 @@ +""" +This module contains functions to write commondata and systypes +tables to files +""" + +def write_commondata_data(commondata, buffer): + """ + write commondata table to buffer, this can be a memory map, + compressed archive or strings (using for instance StringIO) + + + Parameters + ---------- + + commondata : validphys.coredata.CommonData + + buffer : memory map, compressed archive or strings + example: StringIO object + + + Example + ------- + >>> from validphys.loader import Loader + >>> from io import StringIO + + >>> l = Loader() + >>> cd = l.check_commondata("NMC").load_commondata_instance() + >>> sio = StringIO() + >>> write_commondata_data(cd,sio) + >>> print(sio.getvalue()) + + """ + header = f"{commondata.setname} {commondata.nsys} {commondata.ndata}\n" + buffer.write(header) + commondata.commondata_table.to_csv(buffer, sep="\t", header=None) + + +def write_commondata_to_file(commondata, path): + """ + write commondata table to file + """ + with open(path, "w") as file: + write_commondata_data(commondata, file) + + +def write_systype_data(commondata, buffer): + """ + write systype table to buffer, this can be a memory map, + compressed archive or strings (using for instance StringIO) + + + Parameters + ---------- + + commondata : validphys.coredata.CommonData + + buffer : memory map, compressed archive or strings + example: StringIO object + + + Example + ------- + >>> from validphys.loader import Loader + >>> from io import StringIO + + >>> l = Loader() + >>> cd = l.check_commondata("NMC").load_commondata_instance() + >>> sio = StringIO() + >>> write_systype_data(cd,sio) + >>> print(sio.getvalue()) + + """ + header = f"{commondata.nsys}\n" + buffer.write(header) + commondata.systype_table.to_csv(buffer, sep="\t", header=None) + + +def write_systype_to_file(commondata, path): + """ + write systype table to file + """ + with open(path, "w") as file: + write_systype_data(commondata, file) \ No newline at end of file diff --git a/validphys2/src/validphys/core.py b/validphys2/src/validphys/core.py index 694135e3ae..944056a6d7 100644 --- a/validphys2/src/validphys/core.py +++ b/validphys2/src/validphys/core.py @@ -8,7 +8,6 @@ """ from __future__ import generator_stop -from collections import namedtuple import re import enum import functools @@ -24,7 +23,7 @@ from reportengine.compat import yaml from NNPDF import (LHAPDFSet as libNNPDF_LHAPDFSet, - CommonData, + CommonData as LegacyCommonData, FKTable, FKSet, DataSet, @@ -41,6 +40,9 @@ from validphys.lhapdfset import LHAPDFSet from validphys.fkparser import load_fktable from validphys.pineparser import pineappl_reader +from validphys.commondataparser import (peek_commondata_metadata, + get_plot_kinlabels, + parse_commondata,) log = logging.getLogger(__name__) @@ -234,46 +236,6 @@ def get_members(self): return len(self) -kinlabels_latex = CommonData.kinLabel_latex.asdict() -_kinlabels_keys = sorted(kinlabels_latex, key=len, reverse=True) - - -def get_plot_kinlabels(commondata): - """Return the LaTex kinematic labels for a given Commondata""" - key = commondata.process_type - - return kinlabels_latex[key] - -def get_kinlabel_key(process_label): - #Since there is no 1:1 correspondence between latex keys and GetProc, - #we match the longest key such that the proc label starts with it. - l = process_label - try: - return next(k for k in _kinlabels_keys if l.startswith(k)) - except StopIteration as e: - raise ValueError("Could not find a set of kinematic " - "variables matching the process %s Check the " - "labels defined in commondata.cc. " % (l)) from e - -CommonDataMetadata = namedtuple('CommonDataMetadata', ('name', 'nsys', 'ndata', 'process_type')) - -def peek_commondata_metadata(commondatafilename): - """Check some basic properties commondata object without going though the - trouble of processing it on the C++ side""" - with open(commondatafilename) as f: - try: - l = f.readline() - name, nsys_str, ndata_str = l.split() - l = f.readline() - process_type_str = l.split()[1] - except Exception: - log.error(f"Error processing {commondatafilename}") - raise - - return CommonDataMetadata(name, int(nsys_str), int(ndata_str), - get_kinlabel_key(process_type_str)) - - class CommonDataSpec(TupleComp): def __init__(self, datafile, sysfile, plotfiles, name=None, metadata=None): self.datafile = datafile @@ -312,9 +274,16 @@ def __iter__(self): return iter((self.datafile, self.sysfile, self.plotfiles)) @functools.lru_cache() - def load(self)->CommonData: - #TODO: Use better path handling in python 3.6 - return CommonData.ReadFile(str(self.datafile), str(self.sysfile)) + def load(self): + return parse_commondata(self.datafile, self.sysfile, self.name) + + def load_commondata_instance(self): + """ + load a validphys.core.CommonDataSpec to validphys.core.CommonData + """ + from validphys.commondataparser import load_commondata + + return load_commondata(self) @property def plot_kinlabels(self): @@ -472,7 +441,8 @@ def __init__(self, *, name, commondata, fkspecs, thspec, cuts, @functools.lru_cache() def load(self): - cd = self.commondata.load() + """Load the libNNPDF version of the dataset""" + cd = LegacyCommonData.ReadFile(str(self.commondata.datafile), str(self.commondata.sysfile)) fktables = [] for p in self.fkspecs: @@ -508,7 +478,7 @@ def load_commondata(self): loaded_cuts = self.cuts.load() if not (hasattr(loaded_cuts, '_full') and loaded_cuts._full): intmask = [int(ele) for ele in loaded_cuts] - cd = CommonData(cd, intmask) + cd = cd.with_cuts(intmask) return cd def to_unweighted(self): @@ -656,6 +626,21 @@ def load(self): def load_commondata(self): return [d.load_commondata() for d in self.datasets] + + def load_commondata_instance(self): + """ + Given Experiment load list of validphys.coredata.CommonData + objects with cuts already applied + """ + commodata_list = [] + for dataset in self.datasets: + cd = dataset.commondata.load_commondata_instance() + if dataset.cuts is None: + commodata_list.append(cd) + else: + commodata_list.append(cd.with_cuts(dataset.cuts.load())) + return commodata_list + @property def thspec(self): #TODO: Is this good enough? Should we explicitly pass the theory diff --git a/validphys2/src/validphys/coredata.py b/validphys2/src/validphys/coredata.py index 5e73e3777f..ce0a580cc4 100644 --- a/validphys2/src/validphys/coredata.py +++ b/validphys2/src/validphys/coredata.py @@ -7,6 +7,8 @@ import dataclasses import numpy as np import pandas as pd +from validphys.commondatawriter import write_systype_to_file, write_commondata_to_file +KIN_NAMES = ["kin1", "kin2", "kin3"] @dataclasses.dataclass(eq=False) @@ -175,6 +177,7 @@ def get_np_fktable(self): return fktable + @dataclasses.dataclass(eq=False) class CFactorData: """ @@ -218,15 +221,9 @@ class CommonData: nkin : int Number of kinematics specified - kinematics : list of str with length nkin - Kinematic variables kin1, kin2, kin3 ... - nsys : int Number of systematics - sysid : list of str with length nsys - ID for systematic - commondata_table : pd.DataFrame Pandas dataframe containing the commondata @@ -235,6 +232,9 @@ class CommonData: for each systematic alongside the uncertainty type (ADD/MULT/RAND) and name (CORR/UNCORR/THEORYCORR/SKIP) + + systematics_table: pd.DataFrame + Panda dataframe containing the table of systematics """ setname: str @@ -248,7 +248,7 @@ class CommonData: def __post_init__(self): self.systematics_table = self.commondata_table.drop( - columns=["process", "kin1", "kin2", "kin3", "data", "stat"] + columns=["process", "data", "stat"] + KIN_NAMES ) def with_cuts(self, cuts): @@ -284,10 +284,25 @@ def with_cuts(self, cuts): new_commondata_table = self.commondata_table.loc[cuts] return dataclasses.replace(self, ndata=newndata, commondata_table=new_commondata_table) + @property + def kinematics(self): + return self.commondata_table[KIN_NAMES] + + def get_kintable(self): + return self.kinematics.values + @property def central_values(self): return self.commondata_table["data"] + def with_central_value(self, cv): + tb = self.commondata_table.copy() + tb["data"] = cv + return dataclasses.replace(self, commondata_table=tb) + + def get_cv(self): + return self.central_values.values + @property def stat_errors(self): return self.commondata_table["stat"] @@ -353,3 +368,19 @@ def systematic_errors(self, central_values=None): central_values = self.central_values.to_numpy() converted_mult_errors = self.multiplicative_errors * central_values[:, np.newaxis] / 100 return pd.concat((self.additive_errors, converted_mult_errors), axis=1) + + + def export(self, path): + """Export the data, and error types + Use the same format as libNNPDF: + + - A DATA_.dat file with the dataframe of accepted points + - A systypes/STYPES_.dat file with the error types + """ + + dat_path = path / f"DATA_{self.setname}.dat" + sys_path = path / "systypes" / f"SYSTYPE_{self.setname}_DEFAULT.dat" + sys_path.parent.mkdir(exist_ok=True) + + write_systype_to_file(self, sys_path) + write_commondata_to_file(self, dat_path) diff --git a/validphys2/src/validphys/dataplots.py b/validphys2/src/validphys/dataplots.py index 41248c842c..7644756497 100644 --- a/validphys2/src/validphys/dataplots.py +++ b/validphys2/src/validphys/dataplots.py @@ -26,6 +26,7 @@ from validphys.plotoptions import get_info, kitable, transform_result from validphys import plotutils from validphys.utils import sane_groupby_iter, split_ranges, scale_from_grid +from validphys.coredata import KIN_NAMES log = logging.getLogger(__name__) @@ -266,7 +267,7 @@ def _plot_fancy_impl(results, commondata, cutlist, table[('err', i)] = err else: table[cvcol] = cv/norm_cv - table[('err', i)] = err/norm_cv + table[('err', i)] = np.abs(err/norm_cv) cvcols.append(cvcol) @@ -960,12 +961,13 @@ def plot_positivity(pdfs, positivity_predictions_for_pdfs, posdataset, pos_use_k ax.axhline(0, color='red') posset = posdataset.load_commondata() - ndata = posset.GetNData() + ndata = posset.ndata xvals = [] if pos_use_kin: - ax.set_xlabel('kin1') - xvals = [posset.GetKinematics(i, 0) for i in range(0, ndata)] + kin_name = KIN_NAMES[0] + ax.set_xlabel(kin_name) + xvals = posset.kinematics[kin_name].values else: ax.set_xlabel('idat') xvals = np.arange(ndata) diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index 913f883e8b..2532ca2c00 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -9,13 +9,42 @@ import numpy as np -from NNPDF import CommonData -from reportengine.checks import make_argcheck, check, check_positive, make_check +from reportengine.checks import check, make_check from reportengine.compat import yaml import validphys.cuts - +from validphys.commondatawriter import ( + write_commondata_to_file, + write_systype_to_file, + ) log = logging.getLogger(__name__) +KIN_LABEL = { + "DIS": ("x", "Q2", "y"), + "DYP": ("y", "M2", "sqrts"), + "JET": ("eta", "p_T2", "sqrts"), + "DIJET": ("eta", "m_12", "sqrts"), + "PHT": ("eta_gamma", "E_{T,gamma)2", "sqrts"), + "INC": ("0", "mu2", "sqrts"), + "EWK_RAP": ("etay", "M2", "sqrts"), + "EWK_PT": ("p_T", "M2", "sqrts"), + "EWK_PTRAP": ("etay", "p_T2", "sqrts"), + "EWK_MLL": ("M_ll", "M_ll2", "sqrts"), + "EWJ_RAP": ("etay", "M2", "sqrts"), + "EWJ_PT": ("p_T", "M2", "sqrt(s)"), + "EWJ_PTRAP": ("etay", "p_T2", "sqrts"), + "EWJ_JRAP": ("etay", "M2", "sqrts"), + "EWJ_JPT": ("p_T", "M2", "sqrts"), + "EWJ_MLL": ("M_ll", "M_ll2", "sqrts"), + "HQP_YQQ": ("yQQ", "mu2", "sqrts"), + "HQP_MQQ": ("MQQ", "mu2", "sqrts"), + "HQP_PTQQ": ("p_TQQ", "mu2", "sqrts"), + "HQP_YQ": ("yQ", "mu2", "sqrts"), + "HQP_PTQ": ("p_TQ", "mu2", "sqrts"), + "HIG_RAP": ("y", "M_H2", "sqrts"), + "SIA": ("z", "Q2", "y"), +} + + class RuleProcessingError(Exception): """Exception raised when we couldn't process a rule.""" @@ -46,12 +75,6 @@ def default_filter_rules_input(): return yaml.safe_load(read_text(validphys.cuts, "filters.yaml")) -@make_argcheck -def check_rngalgo(rngalgo: int): - """Check rngalgo content""" - check(0 <= rngalgo < 17, - "Invalid rngalgo. Must be int between [0, 16].") - def check_nonnegative(var: str): """Ensure that `var` is positive""" @@ -74,27 +97,8 @@ def export_mask(path, mask): """Dump mask to file""" np.savetxt(path, mask, fmt='%d') -@check_rngalgo -@check_nonnegative('filterseed') -@check_nonnegative('seed') -def prepare_nnpdf_rng(filterseed:int, rngalgo:int, seed:int): - """Initialise the internal NNPDF RNG, specified by ``rngalgo`` which must - be an integer between 0 and 16, seeded with ``filterseed``. - The RNG can then be subsequently used to i.e generate pseudodata. - """ - try: - from NNPDF import RandomGenerator - except ImportError as e: - logging.error("Generating closure data needs a valid installation of libNNPDF") - raise e - - log.warning("Importing libNNPDF") - log.info("Initialising RNG") - RandomGenerator.InitRNG(rngalgo, seed) - RandomGenerator.GetRNG().SetSeed(filterseed) - -@check_positive('errorsize') -def filter_closure_data(filter_path, data, fakepdf, fakenoise, errorsize, prepare_nnpdf_rng): + +def filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed): """Filter closure data. In addition to cutting data points, the data is generated from an underlying ``fakepdf``, applying a shift to the data if ``fakenoise`` is ``True``, which emulates the experimental central values @@ -102,13 +106,11 @@ def filter_closure_data(filter_path, data, fakepdf, fakenoise, errorsize, prepar """ log.info('Filtering closure-test data.') - return _filter_closure_data( - filter_path, data, fakepdf, fakenoise, errorsize) + return _filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed) -@check_positive("errorsize") def filter_closure_data_by_experiment( - filter_path, experiments_data, fakepdf, fakenoise, errorsize, prepare_nnpdf_rng, + filter_path, experiments_data, fakepdf, fakenoise, filterseed, experiments_index ): """ Like :py:func:`filter_closure_data` except filters data by experiment. @@ -119,10 +121,19 @@ def filter_closure_data_by_experiment( not reproducible. """ - return [ - _filter_closure_data(filter_path, exp, fakepdf, fakenoise, errorsize) - for exp in experiments_data - ] + + res = [] + for exp in experiments_data: + experiment_index = experiments_index[ + experiments_index.isin([exp.name], level=0) + ] + res.append( + _filter_closure_data( + filter_path, exp, fakepdf, fakenoise, filterseed, experiment_index + ) + ) + + return res def filter_real_data(filter_path, data): @@ -157,6 +168,7 @@ def _write_ds_cut_data(path, dataset): def _filter_real_data(filter_path, data): """Filter real experimental data.""" + total_data_points = 0 total_cut_data_points = 0 for dataset in data.datasets: @@ -164,28 +176,97 @@ def _filter_real_data(filter_path, data): nfull, ncut = _write_ds_cut_data(path, dataset) total_data_points += nfull total_cut_data_points += ncut - dataset.load_commondata().Export(str(path)) + dataset.load_commondata().export(path) return total_data_points, total_cut_data_points -def _filter_closure_data(filter_path, data, fakepdf, fakenoise, errorsize): - """Filter closure test data.""" +def _filter_closure_data( + filter_path, data, fakepdf, fakenoise, filterseed, experiments_index +): + """ + This function is accessed within a closure test only, that is, the fakedata + namespace has to be True (If fakedata = False, the _filter_real_data function + will be used to write the commondata files). + + The function writes commondata and systypes files within the + name_closure_test/filter folder. + If fakenoise is True, Level 1 type data is written to the filter folder, otherwise + Level 0 data is written. + + Level 1 data is generated from the Level 0 data by adding noise sampled from + the experimental covariance matrix using the validphys.pseudodata.make_replica + function. + + Parameters + ---------- + + filter_path : str + path to filter folder + + data : validphys.core.DataGroupSpec + + fakepdf : validphys.core.PDF + + fakenoise : bool + if fakenoise perform level1 shift of central data values + + filterseed : int + random seed used for the generation of + random noise added to Level 0 data + + + experiments_index : pandas.MultiIndex + + + Returns + ------- + tuple + total data points and points passing the cuts + + """ total_data_points = 0 total_cut_data_points = 0 - fakeset = fakepdf.legacy_load() - # Load data, don't cache result - loaded_data = data.load.__wrapped__(data) - # generate level 1 shift if fakenoise - loaded_data.MakeClosure(fakeset, fakenoise) - for j, dataset in enumerate(data.datasets): + + # circular import generated @ core.py + from validphys.pseudodata import level0_commondata_wc, make_level1_data + + closure_data = level0_commondata_wc(data, fakepdf) + + for dataset in data.datasets: + #== print number of points passing cuts, make dataset directory and write FKMASK ==# path = filter_path / dataset.name nfull, ncut = _write_ds_cut_data(path, dataset) + make_dataset_dir(path / "systypes") total_data_points += nfull total_cut_data_points += ncut - loaded_ds = loaded_data.GetSet(j) - if errorsize != 1.0: - loaded_ds.RescaleErrors(errorsize) - loaded_ds.Export(str(path)) + + if fakenoise: + #======= Level 1 closure test =======# + + closure_data = make_level1_data( + data, + closure_data, + filterseed, + experiments_index, + ) + + #====== write commondata and systype files ======# + if fakenoise: + log.info("Writing Level1 data") + else: + log.info("Writing Level0 data") + + for cd in closure_data: + path_cd = filter_path / cd.setname / f"DATA_{cd.setname}.dat" + path_sys = ( + filter_path + / cd.setname + / "systypes" + / f"SYSTYPE_{cd.setname}_DEFAULT.dat" + ) + write_commondata_to_file(commondata=cd, path=path_cd) + write_systype_to_file(commondata=cd, path=path_sys) + return total_data_points, total_cut_data_points @@ -343,14 +424,14 @@ def __init__( f"Could not find dataset {self.dataset}" ) from e if cd.process_type[:3] == "DIS": - self.variables = CommonData.kinLabel["DIS"] + self.variables = KIN_LABEL["DIS"] else: - self.variables = CommonData.kinLabel[cd.process_type] + self.variables = KIN_LABEL[cd.process_type] else: if self.process_type[:3] == "DIS": - self.variables = CommonData.kinLabel["DIS"] + self.variables = KIN_LABEL["DIS"] else: - self.variables = CommonData.kinLabel[self.process_type] + self.variables = KIN_LABEL[self.process_type] if hasattr(self, "local_variables"): if not isinstance(self.local_variables, Mapping): @@ -422,19 +503,21 @@ def __hash__(self): return hash(self._properties) def __call__(self, dataset, idat): - central_value = dataset.GetData(idat) + central_value = dataset.get_cv()[idat] + process_name = dataset.commondataproc + # We return None if the rule doesn't apply. This # is different to the case where the rule does apply, # but the point was cut out by the rule. if ( - dataset.GetSetName() != self.dataset - and dataset.GetProc(idat) != self.process_type + dataset.setname != self.dataset + and process_name != self.process_type and self.process_type != "DIS_ALL" ): return None # Handle the generalised DIS cut - if self.process_type == "DIS_ALL" and dataset.GetProc(idat)[:3] != "DIS": + if self.process_type == "DIS_ALL" and not process_name.startswith("DIS"): return None ns = self._make_point_namespace(dataset, idat) @@ -468,7 +551,7 @@ def __repr__(self): # pragma: no cover def _make_kinematics_dict(self, dataset, idat) -> dict: """Fill in a dictionary with the kinematics for each point""" - kinematics = [dataset.GetKinematics(idat, j) for j in range(3)] + kinematics = dataset.kinematics.values[idat] return dict(zip(self.variables, kinematics)) def _make_point_namespace(self, dataset, idat) -> dict: @@ -488,7 +571,7 @@ def get_cuts_for_dataset(commondata, rules) -> list: Parameters ---------- - commondata: NNPDF CommonData spec + commondata: validphys.coredata.CommonData rules: List[Rule] A list of Rule objects specifying the filters. @@ -515,7 +598,7 @@ def get_cuts_for_dataset(commondata, rules) -> list: dataset = commondata.load() mask = [] - for idat in range(dataset.GetNData()): + for idat in range(dataset.ndata): broken = False for rule in rules: rule_result = rule(dataset, idat) diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index d518efcccd..b2d07b7170 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -323,7 +323,7 @@ def print_different_cuts(fits, test_for_same_cuts): res.write("The following datasets are both included but have different kinematical cuts:\n\n") for (first, second) in test_for_same_cuts: info = get_info(first.commondata) - total_points = len(first.commondata.load()) + total_points = first.commondata.ndata res.write(" - %s:\n" % info.dataset_label) first_len = len(first.cuts.load()) if first.cuts else total_points second_len = len(second.cuts.load()) if second.cuts else total_points @@ -440,10 +440,10 @@ def print_systype_overlap(groups_commondata, group_dataset_inputs_by_metadata): systype_groups = dict() for group_cd, group in zip(groups_commondata, group_dataset_inputs_by_metadata): systype_groups[group["group_name"]] = { - cd.load().GetSys(0, i).name + cd.load().systype_table.iloc[i]["name"] for cd in group_cd for i in range(cd.nsys) - if cd.load().GetSys(0, i).name not in allow_list + if cd.load().systype_table.iloc[i]["name"] not in allow_list } systype_overlap = set() diff --git a/validphys2/src/validphys/mc_gen.py b/validphys2/src/validphys/mc_gen.py index ff68d59c88..bf94b5b478 100644 --- a/validphys2/src/validphys/mc_gen.py +++ b/validphys2/src/validphys/mc_gen.py @@ -5,6 +5,7 @@ Tools to check the pseudo-data MC generation. """ # The functions in this module have been ported to not use libNNPDF +# but is still using it under the hood # it has been a direct port of the libnnpdf dependent structure # so they should not be used as an example import logging diff --git a/validphys2/src/validphys/n3fit_data.py b/validphys2/src/validphys/n3fit_data.py index 7f50c96992..e057a36a5b 100644 --- a/validphys2/src/validphys/n3fit_data.py +++ b/validphys2/src/validphys/n3fit_data.py @@ -347,8 +347,9 @@ def pseudodata_table(groups_replicas_indexed_make_replica, replicas): Notes ----- Whilst running ``n3fit``, this action will only be called if - `fitting::savepseudodata` is `true` and replicas are fitted one at a time. - The table can be found in the replica folder i.e. /nnfit/replica_*/ + `fitting::savepseudodata` is `true` (as per the default setting) and + replicas are fitted one at a time. The table can be found in the replica + folder i.e. /nnfit/replica_*/ """ # Concatenate over replicas @@ -360,7 +361,7 @@ def pseudodata_table(groups_replicas_indexed_make_replica, replicas): @table def training_pseudodata(pseudodata_table, training_mask): """Save the training data for the given replica. - Activate by setting ``fitting::savepseudodata: True`` + Deactivate by setting ``fitting::savepseudodata: False`` from within the fit runcard. See Also @@ -373,7 +374,7 @@ def training_pseudodata(pseudodata_table, training_mask): @table def validation_pseudodata(pseudodata_table, training_mask): """Save the training data for the given replica. - Activate by setting ``fitting::savepseudodata: True`` + Deactivate by setting ``fitting::savepseudodata: False`` from within the fit runcard. See Also diff --git a/validphys2/src/validphys/paramfits/dataops.py b/validphys2/src/validphys/paramfits/dataops.py index fed61f8b2f..81c0992fc4 100644 --- a/validphys2/src/validphys/paramfits/dataops.py +++ b/validphys2/src/validphys/paramfits/dataops.py @@ -57,7 +57,7 @@ def location(self): @property def scale(self): - return np.asscalar(np.diff(np.percentile(self.data, [15.87, 84.13])))/2 + return (np.diff(np.percentile(self.data, [15.87, 84.13]))).item()/2 def get_parabola(asvals, chi2vals): diff --git a/validphys2/src/validphys/pdfbases.py b/validphys2/src/validphys/pdfbases.py index ffa6af5bee..aac0f01ef1 100644 --- a/validphys2/src/validphys/pdfbases.py +++ b/validphys2/src/validphys/pdfbases.py @@ -35,6 +35,23 @@ (21 , r"g"), )) +PIDS_DICT = { + -6: "TBAR", + -5: "BBAR", + -4: "CBAR", + -3: "SBAR", + -2: "UBAR", + -1: "DBAR", + 21: "GLUON", + 1: "D", + 2: "U", + 3: "S", + 4: "C", + 5: "B", + 6: "T", + 22: "PHT", + } + # Canonical ordering of PDG codes (so flavour basis) ALL_FLAVOURS = (-6, -5, -4, -3, -2, -1, 21, 1, 2, 3, 4, 5, 6, 22) DEFAULT_FLARR = (-3,-2,-1,0,1,2,3,4) @@ -476,8 +493,7 @@ def f_(transform_func): default_elements = DEFAULT_FLARR, element_representations=PDG_PARTONS ) -#dicts are oredered in python 3.6+... code shouldn't vreak if they aren't -#though +# dicts are ordered in python 3.6+... code shouldn't break if they aren't though #see Eqs.(56),(57) https://arxiv.org/pdf/0808.1231.pdf for evolution basis definition evolution = LinearBasis.from_mapping({ r'\Sigma' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1 ,'b':1, 'bbar': 1, 't': 1, 'tbar': 1}, @@ -591,10 +607,14 @@ def f_(transform_func): 's': {'s': 1}, 'sbar': {'sbar': 1}, 'c': {'c': 1}, + 'cbar': {'cbar': 1}, 'g': {'g': 1}, }, default_elements=('u', 'ubar', 'd', 'dbar', 's', 'sbar', 'c', 'g', )) +CCBAR_ASYMM_FLAVOUR = copy.deepcopy(FLAVOUR) +CCBAR_ASYMM_FLAVOUR.default_elements=('u', 'ubar', 'd', 'dbar', 's', 'sbar', 'c', 'cbar', 'g') + pdg = LinearBasis.from_mapping({ 'g/10': {'g':0.1}, 'u_{v}': {'u':1, 'ubar':-1}, @@ -753,6 +773,17 @@ def fitbasis_to_NN31IC(flav_info, fitbasis): g = {'sng': 0, 'v': 0, 'v3': 0, 'v8': 0, 't3': 0, 't8': 0, 't15': 0, 'g': 1, 'v15': 0 } v15 = {'sng': 0, 'v': 0, 'v3': 0, 'v8': 0, 't3': 0, 't8': 0, 't15': 0, 'g': 0, 'v15': 1 } + elif fitbasis == 'CCBAR_ASYMM_FLAVOUR': + sng = {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1, 'g': 0 } + v = {'u': 1, 'ubar': -1, 'd': 1, 'dbar': -1, 's': 1, 'sbar': -1, 'c': 1, 'cbar': -1, 'g': 0 } + v3 = {'u': 1, 'ubar': -1, 'd': -1, 'dbar': 1, 's': 0, 'sbar': 0, 'c': 0, 'cbar': 0, 'g': 0 } + v8 = {'u': 1, 'ubar': -1, 'd': 1, 'dbar': -1, 's': -2, 'sbar': 2, 'c': 0, 'cbar': 0, 'g': 0 } + t3 = {'u': 1, 'ubar': 1, 'd': -1, 'dbar': -1, 's': 0, 'sbar': 0, 'c': 0, 'cbar': 0, 'g': 0 } + t8 = {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': -2, 'sbar': -2, 'c': 0, 'cbar': 0, 'g': 0 } + cp = {'u': 0, 'ubar': 0, 'd': 0, 'dbar': 0, 's': 0, 'sbar': 0, 'c': 1, 'cbar': 1, 'g': 0 } + g = {'u': 0, 'ubar': 0, 'd': 0, 'dbar': 0, 's': 0, 'sbar': 0, 'c': 0, 'cbar': 0, 'g': 1 } + v15 = {'u': 1, 'ubar': -1, 'd': 1, 'dbar': -1, 's': 1, 'sbar': -1, 'c': -3, 'cbar': 3, 'g': 0 } + flist = [sng, g, v, v3, v8, t3, t8, cp, v15] mat = [] diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index c788277355..ec08c0e1a9 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -8,6 +8,10 @@ from scipy.integrate import trapezoid from pathlib import Path +from eko.io import EKO +from eko.io.manipulate import xgrid_reshape +from eko.interpolation import XGrid + import yaml from os import remove import time @@ -184,12 +188,9 @@ def compute_photon_array(self, id): # TODO : the different x points could be even computed in parallel # Load eko and reshape it - from eko.io import EKO with EKO.edit(self.path_to_eko_photon) as eko: # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape - from eko.io.manipulate import xgrid_reshape - from eko.interpolation import XGrid xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) # construct PDFs diff --git a/validphys2/src/validphys/plotoptions/core.py b/validphys2/src/validphys/plotoptions/core.py index d96aef7290..f97fee6fac 100644 --- a/validphys2/src/validphys/plotoptions/core.py +++ b/validphys2/src/validphys/plotoptions/core.py @@ -19,8 +19,9 @@ from reportengine.compat import yaml from reportengine.utils import get_functions, ChainMap -from NNPDF import CommonData, DataSet +from NNPDF import DataSet from validphys.core import CommonDataSpec, DataSetSpec, Cuts, InternalCutsWrapper +from validphys.coredata import CommonData from validphys.plotoptions.utils import apply_to_all_columns, get_subclasses from validphys.plotoptions import labelers, kintransforms, resulttransforms from validphys.utils import parse_yaml_inp @@ -57,10 +58,11 @@ def get_info(data, *, normalize=False, cuts=None, use_plotfiles=True): if cuts is None: if isinstance(data, DataSetSpec): cuts = data.cuts.load() if data.cuts else None - elif isinstance(cuts, (Cuts, InternalCutsWrapper)): + elif hasattr(cuts, 'load'): cuts = cuts.load() - elif not cuts: - cuts = None + + if cuts is not None and not len(cuts): + raise NotImplementedError("No point passes the cuts. Cannot retieve info") if isinstance(data, DataSetSpec): data = data.commondata @@ -175,6 +177,11 @@ def from_commondata(cls, commondata, cuts=None, normalize=False): kinlabels = commondata.plot_kinlabels kinlabels = plot_params['kinematics_override'].new_labels(*kinlabels) + if "extra_labels" in plot_params and cuts is not None: + cut_extra_labels ={ + k: [v[i] for i in cuts] for k, v in plot_params["extra_labels"].items() + } + plot_params["extra_labels"] = cut_extra_labels return cls(kinlabels=kinlabels, **plot_params) diff --git a/validphys2/src/validphys/pseudodata.py b/validphys2/src/validphys/pseudodata.py index 142072e0d6..67d0144a7b 100644 --- a/validphys2/src/validphys/pseudodata.py +++ b/validphys2/src/validphys/pseudodata.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd -from validphys.covmats import INTRA_DATASET_SYS_NAME, sqrt_covmat +from validphys.covmats import INTRA_DATASET_SYS_NAME, sqrt_covmat, dataset_inputs_covmat_from_systematics from reportengine import collect @@ -236,10 +236,139 @@ def indexed_make_replica(groups_index, make_replica): return pd.DataFrame(make_replica, index=groups_index, columns=["data"]) +def level0_commondata_wc(data, fakepdf): + """ + Given a validphys.core.DataGroupSpec object, load commondata and + generate a new commondata instance with central values replaced + by fakepdf prediction + + Parameters + ---------- + + data : validphys.core.DataGroupSpec + + fakepdf: validphys.core.PDF + + Returns + ------- + list + list of validphys.coredata.CommonData instances corresponding to + all datasets within one experiment. The central value is replaced + by Level 0 fake data. + + Example + ------- + >>> from validphys.api import API + >>> API.level0_commondata_wc(dataset_inputs = [{"dataset":"NMC"}], use_cuts="internal", theoryid=200,fakepdf = "NNPDF40_nnlo_as_01180") + + [CommonData(setname='NMC', ndata=204, commondataproc='DIS_NCE', nkin=3, nsys=16)] + """ + from validphys.covmats import dataset_t0_predictions + level0_commondata_instances_wc = [] + + # ==== Load validphys.coredata.CommonData instance with cuts ====# + + for dataset in data.datasets: + commondata_wc = dataset.commondata.load_commondata_instance() + if dataset.cuts is not None: + cuts = dataset.cuts.load() + commondata_wc = commondata_wc.with_cuts(cuts) + + # == Generate a new CommonData instance with central value given by Level 0 data generated with fakepdf ==# + + t0_prediction = dataset_t0_predictions( + dataset=dataset, t0set=fakepdf + ) # N.B. cuts already applied to th. pred. + level0_commondata_instances_wc.append( + commondata_wc.with_central_value(t0_prediction) + ) + + return level0_commondata_instances_wc + -_group_recreate_pseudodata = collect('indexed_make_replica', ('group_dataset_inputs_by_experiment',)) -_recreate_fit_pseudodata = collect('_group_recreate_pseudodata', ('fitreplicas', 'fitenvironment')) -_recreate_pdf_pseudodata = collect('_group_recreate_pseudodata', ('pdfreplicas', 'fitenvironment')) +def make_level1_data( + data, level0_commondata_wc, filterseed, experiments_index +): + """ + Given a list of level0 commondata instances, return the same list + with central values replaced by level1 data + + + Parameters + ---------- + + data : validphys.core.DataGroupSpec + + level0_commondata_wc : list + list of validphys.coredata.CommonData instances corresponding to + all datasets within one experiment. The central value is replaced + by Level 0 fake data. Cuts already applied. + + filterseed: int + random seed used for the generation of Level 1 data + + + Returns + ------- + list + list of validphys.coredata.CommonData instances corresponding to + all datasets within one experiment. The central value is replaced + by Level 1 fake data. + + Example + ------- + + >>> from validphys.api import API + >>> dataset='NMC' + >>> l1_cd = API.make_level1_data(dataset_inputs = [{"dataset":dataset}],use_cuts="internal", theoryid=200, + fakepdf = "NNPDF40_nnlo_as_01180",filterseed=1) + >>> l1_cd + [CommonData(setname='NMC', ndata=204, commondataproc='DIS_NCE', nkin=3, nsys=16)] + """ + # =============== generate experimental covariance matrix ===============# + + dataset_input_list = list(data.dsinputs) + + commondata_wc = data.load_commondata_instance() + + covmat = dataset_inputs_covmat_from_systematics( + commondata_wc, + dataset_input_list, + use_weights_in_covmat=False, + norm_threshold=None, + _list_of_central_values=None, + _only_additive=False, + ) + + # ================== generation of pseudo data ======================# + # = generate pseudo data starting from theory predictions + level1_data = make_replica( + level0_commondata_wc, filterseed, covmat, sep_mult=False, genrep=True + ) + + indexed_level1_data = indexed_make_replica(experiments_index, level1_data) + + # ===== create commondata instances with central values given by pseudo_data =====# + level1_commondata_dict = {c.setname: c for c in level0_commondata_wc} + level1_commondata_instances_wc = [] + + for xx, grp in indexed_level1_data.groupby('dataset'): + level1_commondata_instances_wc.append( + level1_commondata_dict[xx].with_central_value(grp.values) + ) + + return level1_commondata_instances_wc + + +_group_recreate_pseudodata = collect( + 'indexed_make_replica', ('group_dataset_inputs_by_experiment',) +) +_recreate_fit_pseudodata = collect( + '_group_recreate_pseudodata', ('fitreplicas', 'fitenvironment') +) +_recreate_pdf_pseudodata = collect( + '_group_recreate_pseudodata', ('pdfreplicas', 'fitenvironment') +) fit_tr_masks = collect('replica_training_mask_table', ('fitreplicas', 'fitenvironment')) pdf_tr_masks = collect('replica_training_mask_table', ('pdfreplicas', 'fitenvironment')) @@ -304,4 +433,4 @@ def recreate_pdf_pseudodata(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks) pdf_tr_masks_no_table = collect('replica_training_mask', ('pdfreplicas', 'fitenvironment')) def recreate_pdf_pseudodata_no_table(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks_no_table): - return recreate_pdf_pseudodata(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks_no_table) \ No newline at end of file + return recreate_pdf_pseudodata(_recreate_pdf_pseudodata, pdfreplicas, pdf_tr_masks_no_table) diff --git a/validphys2/src/validphys/tests/conftest.py b/validphys2/src/validphys/tests/conftest.py index 74eb4315c3..b41e7258ad 100644 --- a/validphys2/src/validphys/tests/conftest.py +++ b/validphys2/src/validphys/tests/conftest.py @@ -23,6 +23,8 @@ def tmp(tmpdir): # Here define the default config items like the PDF, theory and experiment specs SINGLE_DATAPOINT = {'dataset': 'ATLASTTBARTOT8TEV', 'cfac': ['QCD']} +SINGLE_DATASET = {'dataset': 'NMC'} + DATA = [ {'dataset': 'NMC'}, {'dataset': 'ATLASTTBARTOT', 'cfac':['QCD']}, @@ -51,6 +53,7 @@ def tmp(tmpdir): THEORYID = 162 FIT = "NNPDF40_nnlo_lowprecision" FIT_3REPLICAS = "Basic_runcard_3replicas_lowprec_221130" +FIT_3REPLICAS_DCUTS = "Basic_runcard_3replicas_diffcuts_230221" FIT_ITERATED = "NNPDF40_nnlo_low_precision_iterated" PSEUDODATA_FIT = "pseudodata_test_fit_n3fit_221130" diff --git a/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv b/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv index f7ce521f40..a46ad353a5 100644 --- a/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv +++ b/validphys2/src/validphys/tests/regressions/test_filter_rebuild_closure_data.csv @@ -1,236 +1,236 @@ group dataset id data_central -NMC NMC 16 0.4007353489505 -NMC NMC 21 0.3596156369461 -NMC NMC 22 0.3696772738631 -NMC NMC 27 0.3662960538326 -NMC NMC 28 0.3671016068397 -NMC NMC 29 0.3911462081599 -NMC NMC 34 0.3873596614146 -NMC NMC 35 0.3685951249355 -NMC NMC 36 0.3652752448135 -NMC NMC 40 0.3782916051482 -NMC NMC 41 0.3910438628795 -NMC NMC 42 0.3733418699219 -NMC NMC 46 0.3810537704863 -NMC NMC 47 0.3708532044501 -NMC NMC 48 0.3593684720153 -NMC NMC 51 0.3693367172184 -NMC NMC 52 0.3737791523646 -NMC NMC 53 0.3386288622738 -NMC NMC 54 0.3500948571991 -NMC NMC 57 0.3418514086764 -NMC NMC 58 0.3489404958264 -NMC NMC 59 0.3488612312165 -NMC NMC 60 0.3505041034015 -NMC NMC 63 0.3333040875332 -NMC NMC 64 0.2944010260022 -NMC NMC 65 0.3099040341106 -NMC NMC 68 0.2811103711856 -NMC NMC 69 0.2595731013581 -NMC NMC 83 0.332503113838 -NMC NMC 84 0.3062223521694 -NMC NMC 87 0.3477451853918 -NMC NMC 88 0.352597827366 -NMC NMC 89 0.3689837642521 -NMC NMC 91 0.3410075219517 -NMC NMC 92 0.3380026268081 -NMC NMC 93 0.3630165854001 -NMC NMC 94 0.3692037594232 -NMC NMC 95 0.3379900856468 -NMC NMC 97 0.3498975977998 -NMC NMC 98 0.3455821206785 -NMC NMC 99 0.3561806744993 -NMC NMC 100 0.3732490901176 -NMC NMC 101 0.3500032818012 -NMC NMC 104 0.3619815034828 -NMC NMC 105 0.3577747301693 -NMC NMC 106 0.3448546937843 -NMC NMC 107 0.3970272458214 -NMC NMC 108 0.3194277040841 -NMC NMC 110 0.3205419458354 -NMC NMC 111 0.3367281886952 -NMC NMC 112 0.3386950891887 -NMC NMC 113 0.3225662456658 -NMC NMC 114 0.29975210255 -NMC NMC 115 0.3258513025439 -NMC NMC 116 0.3369058963301 -NMC NMC 117 0.3302714626432 -NMC NMC 118 0.3149870988695 -NMC NMC 119 0.2993100089626 -NMC NMC 120 0.3368228206002 -NMC NMC 121 0.3102225912073 -NMC NMC 122 0.3180821969045 -NMC NMC 123 0.2933430928775 -NMC NMC 124 0.3098840948832 -NMC NMC 125 0.3231255065332 -NMC NMC 126 0.2902405904389 -NMC NMC 127 0.2669059748255 -NMC NMC 128 0.2689476065289 -NMC NMC 129 0.2956116242333 -NMC NMC 130 0.288082548032 -NMC NMC 131 0.2226505072116 -NMC NMC 132 0.2198391302795 -NMC NMC 133 0.204048963916 -NMC NMC 134 0.2127696389248 -NMC NMC 136 0.1375618648213 -NMC NMC 137 0.1339894685257 -NMC NMC 147 0.3489969251822 -NMC NMC 148 0.3552399410999 -NMC NMC 152 0.3483599212619 -NMC NMC 153 0.3560033903034 -NMC NMC 154 0.3796442730621 -NMC NMC 157 0.3429642181623 -NMC NMC 158 0.3706142444086 -NMC NMC 159 0.355255409241 -NMC NMC 160 0.352842258266 -NMC NMC 161 0.3539020551091 -NMC NMC 162 0.3521469993182 -NMC NMC 163 0.3632537732513 -NMC NMC 164 0.3725308840612 -NMC NMC 165 0.3576604586946 -NMC NMC 166 0.3816526263355 -NMC NMC 167 0.3688809286074 -NMC NMC 168 0.3385882966462 -NMC NMC 169 0.3387628659638 -NMC NMC 170 0.3760634504695 -NMC NMC 171 0.3785323104752 -NMC NMC 172 0.3632421316272 -NMC NMC 173 0.3623753029918 -NMC NMC 174 0.3362183838135 -NMC NMC 175 0.3432266150754 -NMC NMC 176 0.3640134644566 -NMC NMC 177 0.3576634295374 -NMC NMC 178 0.3597463154209 -NMC NMC 179 0.3603372275518 -NMC NMC 180 0.3177940693752 -NMC NMC 181 0.3620568994759 -NMC NMC 182 0.3701314089822 -NMC NMC 183 0.2978652392955 -NMC NMC 184 0.3541449769575 -NMC NMC 185 0.3448722906877 -NMC NMC 186 0.3412588395999 -NMC NMC 187 0.3570684620173 -NMC NMC 188 0.3286891216355 -NMC NMC 189 0.3477526408617 -NMC NMC 190 0.335703925785 -NMC NMC 191 0.3444529842929 -NMC NMC 192 0.3403578691808 -NMC NMC 193 0.3347353981548 -NMC NMC 194 0.3203688496486 -NMC NMC 195 0.3159487761597 -NMC NMC 196 0.3443688712277 -NMC NMC 197 0.3124097261611 -NMC NMC 198 0.3154190766316 -NMC NMC 199 0.2942857811848 -NMC NMC 200 0.2799806891526 -NMC NMC 201 0.2775840928341 -NMC NMC 202 0.2532591445827 -NMC NMC 203 0.2708438289781 -NMC NMC 204 0.254383734349 -NMC NMC 205 0.2584578349333 -NMC NMC 206 0.2288369641399 -NMC NMC 207 0.2168810713002 -NMC NMC 208 0.2203718114199 -NMC NMC 209 0.1847450793115 -NMC NMC 210 0.1459140382753 -NMC NMC 211 0.1143166792701 -NMC NMC 212 0.1267472998114 -NMC NMC 221 0.377311655587 -NMC NMC 222 0.3786167921399 -NMC NMC 225 0.3880648558735 -NMC NMC 226 0.3835579098506 -NMC NMC 227 0.3700815811285 -NMC NMC 229 0.3784519070427 -NMC NMC 230 0.3982216265502 -NMC NMC 231 0.4017000200127 -NMC NMC 232 0.3922919247205 -NMC NMC 233 0.4124193564822 -NMC NMC 234 0.3590905662548 -NMC NMC 235 0.3967950366239 -NMC NMC 236 0.3971726043601 -NMC NMC 237 0.4020683891529 -NMC NMC 238 0.3814902989618 -NMC NMC 239 0.386741867313 -NMC NMC 240 0.369385519281 -NMC NMC 241 0.3814990929537 -NMC NMC 242 0.381564927955 -NMC NMC 243 0.3948804610387 -NMC NMC 244 0.3839173773074 -NMC NMC 245 0.3802441689135 -NMC NMC 246 0.3752499091384 -NMC NMC 247 0.3974188959257 -NMC NMC 248 0.3836237312771 -NMC NMC 249 0.3911270447289 -NMC NMC 250 0.3877800272259 -NMC NMC 251 0.3629868559682 -NMC NMC 252 0.3754088188714 -NMC NMC 253 0.380648396326 -NMC NMC 254 0.379643330629 -NMC NMC 255 0.3717431381859 -NMC NMC 256 0.3850277979529 -NMC NMC 257 0.357666057149 -NMC NMC 258 0.3843712169454 -NMC NMC 259 0.3677026936208 -NMC NMC 260 0.3655736280598 -NMC NMC 261 0.3441364575008 -NMC NMC 262 0.3972401996537 -NMC NMC 263 0.3645143376461 -NMC NMC 264 0.3510977851517 -NMC NMC 265 0.3539138992202 -NMC NMC 266 0.3493708856316 -NMC NMC 267 0.3320087427879 -NMC NMC 268 0.3284299620477 -NMC NMC 269 0.3321367638445 -NMC NMC 270 0.3209261313501 -NMC NMC 271 0.3152880392914 -NMC NMC 272 0.3067196941747 -NMC NMC 273 0.3051998380667 -NMC NMC 274 0.3111118466168 -NMC NMC 275 0.2900155628937 -NMC NMC 276 0.2883840542816 -NMC NMC 277 0.2756559342559 -NMC NMC 278 0.2855005116247 -NMC NMC 279 0.2662589115441 -NMC NMC 280 0.2559570780724 -NMC NMC 281 0.2504174862552 -NMC NMC 282 0.2307957738123 -NMC NMC 283 0.241680108918 -NMC NMC 284 0.2146215617884 -NMC NMC 285 0.211746450436 -NMC NMC 286 0.2128987035927 -NMC NMC 287 0.1935284098196 -NMC NMC 288 0.131946290253 -NMC NMC 289 0.1180789438391 -NMC NMC 290 0.1049547912561 -NMC NMC 291 0.1106025920512 -ATLAS ATLASTTBARTOT 0 162.5686864865 -ATLAS ATLASTTBARTOT 1 236.1859490199 -ATLAS ATLASTTBARTOT 2 830.4482576268 -CMS CMSZDIFF12 1 2914.124612961 -CMS CMSZDIFF12 2 1080.410756147 -CMS CMSZDIFF12 3 466.257652438 -CMS CMSZDIFF12 4 227.8725970988 -CMS CMSZDIFF12 5 112.9922431625 -CMS CMSZDIFF12 6 60.11148107363 -CMS CMSZDIFF12 7 31.01641512591 -CMS CMSZDIFF12 11 2841.234006315 -CMS CMSZDIFF12 12 1045.712255313 -CMS CMSZDIFF12 13 455.7116015988 -CMS CMSZDIFF12 14 208.6895742617 -CMS CMSZDIFF12 15 111.8661789107 -CMS CMSZDIFF12 16 61.01526352725 -CMS CMSZDIFF12 17 30.58163323228 -CMS CMSZDIFF12 21 2534.025669336 -CMS CMSZDIFF12 22 911.9673411774 -CMS CMSZDIFF12 23 415.3681972017 -CMS CMSZDIFF12 24 198.7630470755 -CMS CMSZDIFF12 25 100.2899391664 -CMS CMSZDIFF12 26 56.45171623313 -CMS CMSZDIFF12 27 28.6103128109 -CMS CMSZDIFF12 31 1990.217512849 -CMS CMSZDIFF12 32 728.4681533061 -CMS CMSZDIFF12 33 332.4189366359 -CMS CMSZDIFF12 34 165.4302576598 -CMS CMSZDIFF12 35 88.19728756934 -CMS CMSZDIFF12 36 47.85196748986 -CMS CMSZDIFF12 37 24.06201644101 +NMC NMC 16 0.28019164359912174 +NMC NMC 21 0.35499349354805787 +NMC NMC 22 0.36699960383943325 +NMC NMC 27 0.36514731639520354 +NMC NMC 28 0.3796019817168817 +NMC NMC 29 0.3557283684527493 +NMC NMC 34 0.358213498117281 +NMC NMC 35 0.38344229108403 +NMC NMC 36 0.3906176180402464 +NMC NMC 40 0.3565478540570389 +NMC NMC 41 0.35449556242092534 +NMC NMC 42 0.3477428857498088 +NMC NMC 46 0.34058587588303324 +NMC NMC 47 0.37528607055424945 +NMC NMC 48 0.35810702495585167 +NMC NMC 51 0.35948420833368 +NMC NMC 52 0.3252481995431795 +NMC NMC 53 0.34839699344491915 +NMC NMC 54 0.3305599715630647 +NMC NMC 57 0.33854839805110815 +NMC NMC 58 0.3475248192617279 +NMC NMC 59 0.3224033407693148 +NMC NMC 60 0.3557230755468301 +NMC NMC 63 0.3076864562936749 +NMC NMC 64 0.3249652996062988 +NMC NMC 65 0.3225373982130234 +NMC NMC 68 0.2698593749956943 +NMC NMC 69 0.28697589129899725 +NMC NMC 83 0.3314942368553224 +NMC NMC 84 0.3497035726336489 +NMC NMC 87 0.3561247503065496 +NMC NMC 88 0.36048612554048426 +NMC NMC 89 0.342867775009473 +NMC NMC 91 0.3462149531948183 +NMC NMC 92 0.33322645114254057 +NMC NMC 93 0.3496968402988778 +NMC NMC 94 0.37224312344590277 +NMC NMC 95 0.3501229381653187 +NMC NMC 97 0.34862218791470695 +NMC NMC 98 0.3513864997869911 +NMC NMC 99 0.3463189635741315 +NMC NMC 100 0.34999374714067055 +NMC NMC 101 0.3219866280166407 +NMC NMC 104 0.3136568526784361 +NMC NMC 105 0.3238255813447048 +NMC NMC 106 0.34905991139783304 +NMC NMC 107 0.34365171884104645 +NMC NMC 108 0.2700951102701827 +NMC NMC 110 0.3516749264080651 +NMC NMC 111 0.3420426414588959 +NMC NMC 112 0.3368571659898279 +NMC NMC 113 0.32766489680568944 +NMC NMC 114 0.3309229352262125 +NMC NMC 115 0.3087559452541232 +NMC NMC 116 0.30434091887231657 +NMC NMC 117 0.31985278442485376 +NMC NMC 118 0.3081369779237656 +NMC NMC 119 0.32428991893790204 +NMC NMC 120 0.29177949202546344 +NMC NMC 121 0.2986166917278112 +NMC NMC 122 0.28530172048925173 +NMC NMC 123 0.30960790490633666 +NMC NMC 124 0.2838914377920225 +NMC NMC 125 0.327108232453106 +NMC NMC 126 0.30800213354866085 +NMC NMC 127 0.27505741362840286 +NMC NMC 128 0.3057396895330445 +NMC NMC 129 0.2948720021522578 +NMC NMC 130 0.28654594665723143 +NMC NMC 131 0.24998019481392156 +NMC NMC 132 0.23305710852110711 +NMC NMC 133 0.2285884324760879 +NMC NMC 134 0.21022049472555798 +NMC NMC 136 0.14289441509543208 +NMC NMC 137 0.1467309459465361 +NMC NMC 147 0.3783814687867924 +NMC NMC 148 0.3660143252628356 +NMC NMC 152 0.37316750851562697 +NMC NMC 153 0.38611716465278784 +NMC NMC 154 0.3698410284359639 +NMC NMC 157 0.36961892116181694 +NMC NMC 158 0.40918582365959344 +NMC NMC 159 0.3772237079249307 +NMC NMC 160 0.36695342000619174 +NMC NMC 161 0.37837880076113106 +NMC NMC 162 0.3574940348440908 +NMC NMC 163 0.38209182150349047 +NMC NMC 164 0.38555197565482335 +NMC NMC 165 0.38435365356895423 +NMC NMC 166 0.39523350385236933 +NMC NMC 167 0.352962329300064 +NMC NMC 168 0.33498853883029145 +NMC NMC 169 0.3799963472642203 +NMC NMC 170 0.3626711192860542 +NMC NMC 171 0.3726539470995713 +NMC NMC 172 0.38205670828934024 +NMC NMC 173 0.3624890112249862 +NMC NMC 174 0.351766590223616 +NMC NMC 175 0.38120620112890435 +NMC NMC 176 0.376704707091722 +NMC NMC 177 0.35767966843982835 +NMC NMC 178 0.3739917821379859 +NMC NMC 179 0.3779713226850912 +NMC NMC 180 0.3540178478852946 +NMC NMC 181 0.3563337922341492 +NMC NMC 182 0.3495895799725437 +NMC NMC 183 0.40513857686448185 +NMC NMC 184 0.3737246199455833 +NMC NMC 185 0.34224373934105096 +NMC NMC 186 0.3572968130894208 +NMC NMC 187 0.3513259873499916 +NMC NMC 188 0.36191625620248563 +NMC NMC 189 0.3757225472452201 +NMC NMC 190 0.34496599482267015 +NMC NMC 191 0.3671454519682669 +NMC NMC 192 0.3355832294500817 +NMC NMC 193 0.32611756260750746 +NMC NMC 194 0.3264723091842311 +NMC NMC 195 0.31358299893261243 +NMC NMC 196 0.3063679612675748 +NMC NMC 197 0.3243783766742867 +NMC NMC 198 0.2951522224003636 +NMC NMC 199 0.30478606823022336 +NMC NMC 200 0.3162026195490957 +NMC NMC 201 0.24410226609924124 +NMC NMC 202 0.29437714007416727 +NMC NMC 203 0.29086153651126373 +NMC NMC 204 0.2618537028601226 +NMC NMC 205 0.24746460684350655 +NMC NMC 206 0.22851397088173026 +NMC NMC 207 0.23143418017058778 +NMC NMC 208 0.2083333904728336 +NMC NMC 209 0.1871220049907259 +NMC NMC 210 0.13694037697890943 +NMC NMC 211 0.1410254122612783 +NMC NMC 212 0.10913012127955796 +NMC NMC 221 0.38635187132002613 +NMC NMC 222 0.4142522089627604 +NMC NMC 225 0.40345790673282933 +NMC NMC 226 0.40425695161223724 +NMC NMC 227 0.39659246253718794 +NMC NMC 229 0.4111692386727388 +NMC NMC 230 0.40555697596449714 +NMC NMC 231 0.4035342194456604 +NMC NMC 232 0.4151050115450948 +NMC NMC 233 0.3925537185116469 +NMC NMC 234 0.3991798063407916 +NMC NMC 235 0.42128296044596675 +NMC NMC 236 0.39594826617785617 +NMC NMC 237 0.4132863843007187 +NMC NMC 238 0.4063071052878472 +NMC NMC 239 0.39412536731089337 +NMC NMC 240 0.38212850421302924 +NMC NMC 241 0.3914294452161547 +NMC NMC 242 0.40047857374987483 +NMC NMC 243 0.4008070275555427 +NMC NMC 244 0.40262694743529254 +NMC NMC 245 0.39621958826618797 +NMC NMC 246 0.39102221801161896 +NMC NMC 247 0.3886398830939979 +NMC NMC 248 0.38188378549251667 +NMC NMC 249 0.3988887017211007 +NMC NMC 250 0.39585021194541986 +NMC NMC 251 0.40126477628252327 +NMC NMC 252 0.3964150971271786 +NMC NMC 253 0.3902202435316766 +NMC NMC 254 0.38160232623513723 +NMC NMC 255 0.3839102218492622 +NMC NMC 256 0.39033449259461933 +NMC NMC 257 0.37931861168091624 +NMC NMC 258 0.3709117293446901 +NMC NMC 259 0.3732315660927265 +NMC NMC 260 0.3720221661101343 +NMC NMC 261 0.3517503669287229 +NMC NMC 262 0.36807144164962735 +NMC NMC 263 0.3693458573629922 +NMC NMC 264 0.3592451130180051 +NMC NMC 265 0.35632376909940633 +NMC NMC 266 0.35150220374085334 +NMC NMC 267 0.34496329443474294 +NMC NMC 268 0.33956152225234476 +NMC NMC 269 0.3485667202754675 +NMC NMC 270 0.3391216940975037 +NMC NMC 271 0.33736702477837127 +NMC NMC 272 0.3133373105891819 +NMC NMC 273 0.32982788223049275 +NMC NMC 274 0.3211288416924518 +NMC NMC 275 0.3060547284123289 +NMC NMC 276 0.335436416428492 +NMC NMC 277 0.3073804246714358 +NMC NMC 278 0.294093691234672 +NMC NMC 279 0.2884616638265187 +NMC NMC 280 0.26397391314307916 +NMC NMC 281 0.2444528193940454 +NMC NMC 282 0.27832660956025357 +NMC NMC 283 0.25803828744513574 +NMC NMC 284 0.23389825136222092 +NMC NMC 285 0.2146733551898643 +NMC NMC 286 0.21813290674472682 +NMC NMC 287 0.212151341826686 +NMC NMC 288 0.16141376202106558 +NMC NMC 289 0.13433721471473095 +NMC NMC 290 0.11551212545868192 +NMC NMC 291 0.10901466846254028 +ATLAS ATLASTTBARTOT 0 165.5652425198355 +ATLAS ATLASTTBARTOT 1 245.16706727829992 +ATLAS ATLASTTBARTOT 2 801.771432400817 +CMS CMSZDIFF12 1 2973.4725103674823 +CMS CMSZDIFF12 2 1056.3849990121557 +CMS CMSZDIFF12 3 456.57202617345774 +CMS CMSZDIFF12 4 221.42094319673168 +CMS CMSZDIFF12 5 107.23839580843014 +CMS CMSZDIFF12 6 58.24831168720775 +CMS CMSZDIFF12 7 29.594897586398673 +CMS CMSZDIFF12 11 2828.9994410871873 +CMS CMSZDIFF12 12 1042.0799324021702 +CMS CMSZDIFF12 13 443.3584194436015 +CMS CMSZDIFF12 14 204.46185503728015 +CMS CMSZDIFF12 15 105.84479937812591 +CMS CMSZDIFF12 16 59.79512008357787 +CMS CMSZDIFF12 17 30.620091935371985 +CMS CMSZDIFF12 21 2590.081496607627 +CMS CMSZDIFF12 22 912.4546874257419 +CMS CMSZDIFF12 23 410.2638687475507 +CMS CMSZDIFF12 24 203.38813364945474 +CMS CMSZDIFF12 25 103.36390836336159 +CMS CMSZDIFF12 26 54.7135715358862 +CMS CMSZDIFF12 27 28.40706067537687 +CMS CMSZDIFF12 31 2029.9154133864206 +CMS CMSZDIFF12 32 737.238837202754 +CMS CMSZDIFF12 33 331.45213196343286 +CMS CMSZDIFF12 34 172.74641496097794 +CMS CMSZDIFF12 35 88.99325953676879 +CMS CMSZDIFF12 36 49.525868163359064 +CMS CMSZDIFF12 37 24.187338718035164 diff --git a/validphys2/src/validphys/tests/test_fitdata.py b/validphys2/src/validphys/tests/test_fitdata.py index c4069ba798..ccabfd2e69 100644 --- a/validphys2/src/validphys/tests/test_fitdata.py +++ b/validphys2/src/validphys/tests/test_fitdata.py @@ -1,5 +1,26 @@ from validphys.api import API -from validphys.fitdata import print_systype_overlap +from validphys.fitdata import print_systype_overlap, print_different_cuts +from validphys.tests.conftest import FIT_3REPLICAS, FIT_3REPLICAS_DCUTS + + +def test_print_different_cuts(): + """Checks the print_different_cuts functions + using two fits with a different choice of q2min and w2min in the runcard + One of the datasets (SLACP) gets 0 points in in the most restrictive case + The different cuts are: + q2min: 3.49 - 13.49 + w2min: 12.5 - 22.5 + """ + fit_1 = API.fit(fit=FIT_3REPLICAS) + fit_2 = API.fit(fit=FIT_3REPLICAS_DCUTS) + fits = [fit_1, fit_2] + testi = API.test_for_same_cuts(fits=[FIT_3REPLICAS, FIT_3REPLICAS_DCUTS], use_cuts="fromfit") + res = print_different_cuts(fits, testi) + assert "121 out of 260" in res + assert "59 out of 260" in res + assert "33 out of 211" in res + assert "0 out of 211" in res + def test_print_systype_overlap(): """Test that print_systype_overlap does expected thing diff --git a/validphys2/src/validphys/tests/test_loader.py b/validphys2/src/validphys/tests/test_loader.py index decf0a020c..c0385ab109 100644 --- a/validphys2/src/validphys/tests/test_loader.py +++ b/validphys2/src/validphys/tests/test_loader.py @@ -17,18 +17,24 @@ #The sorted is to appease hypothesis dss = sorted(l.available_datasets - {"PDFEVOLTEST"}) +class MockCuts(): + def __init__(self, arr): + self.arr = arr + def load(self): + return self.arr + @composite -def commodata_and_cuts(draw): +def commondata_and_cuts(draw): cd = l.check_commondata(draw(sampled_from(dss))) ndata = cd.metadata.ndata - #TODO: Maybe upgrade to this - #https://github.com/HypothesisWorks/hypothesis/issues/1115 - mask = sorted(draw(sets(sampled_from(range(ndata))))) + # Get a cut mask with at least one selected datapoint + masks = sets(sampled_from(range(ndata)), min_size=1) + mask = sorted(draw(masks)) return cd, mask -@given(arg=commodata_and_cuts()) +@given(arg=commondata_and_cuts()) @settings(deadline=None) def test_rebuild_commondata_without_cuts(tmp_path_factory, arg): # We need to create a new directory for each call of the test @@ -42,8 +48,8 @@ def test_rebuild_commondata_without_cuts(tmp_path_factory, arg): cutpath = tmp / "cuts.txt" np.savetxt(cutpath, np.asarray(cuts, dtype=int), fmt="%u") cutspec = Cuts(cd, cutpath) - lcd = type(lcd)(lcd, cuts) - lcd.Export(str(tmp)) + lcd = lcd.with_cuts(cuts) + lcd.export(tmp) # We have to reconstruct the name here... with_cuts = tmp / f"DATA_{cd.name}.dat" newpath = tmp / "commondata.dat" @@ -60,6 +66,13 @@ def test_rebuild_commondata_without_cuts(tmp_path_factory, arg): nocuts[cuts] = False assert (lncd.get_cv()[nocuts] == 0).all() +@given(inp=commondata_and_cuts()) +def test_kitable_with_cuts(inp): + cd, cuts = inp + info = get_info(cd, cuts=cuts) + tb = kitable(cd, info, cuts=MockCuts(cuts)) + assert len(tb) == len(cuts) + def test_load_fit(): assert l.check_fit(FIT) with pytest.raises(FitNotFound): diff --git a/validphys2/src/validphys/tests/test_pseudodata.py b/validphys2/src/validphys/tests/test_pseudodata.py index 413975af7c..b6149006ff 100644 --- a/validphys2/src/validphys/tests/test_pseudodata.py +++ b/validphys2/src/validphys/tests/test_pseudodata.py @@ -8,11 +8,14 @@ recreation. """ import pandas as pd +import numpy as np +from numpy.testing import assert_allclose import pytest from validphys.api import API -from validphys.tests.conftest import FIT, PSEUDODATA_FIT - +from validphys.tests.conftest import FIT, PSEUDODATA_FIT, THEORYID, SINGLE_DATASET, PDF +from validphys.loader import Loader +from validphys.covmats import dataset_t0_predictions def test_read_fit_pseudodata(): fit_pseudodata = API.read_fit_pseudodata(fit=PSEUDODATA_FIT) @@ -80,3 +83,29 @@ def test_read_matches_recreate(): ) pd.testing.assert_index_equal(read.tr_idx, recreate.tr_idx, check_order=False) pd.testing.assert_index_equal(read.val_idx, recreate.val_idx, check_order=False) + + +def test_level0_commondata_wc(): + """ + check whether level0_commondata_wc and dataset_t0_predictions + coincide + """ + dataset = SINGLE_DATASET + pdfname = PDF + l = Loader() + datasetspec = l.check_dataset(list(dataset.values())[0], theoryid=THEORYID) + t0set = l.check_pdf(pdfname) + + l0_cd = API.level0_commondata_wc( + dataset_inputs=[dataset], + use_cuts="internal", + theoryid=THEORYID, + fakepdf=pdfname, + ) + l0_vals = l0_cd[0].central_values + assert_allclose( + dataset_t0_predictions(dataset=datasetspec, t0set=t0set), + l0_vals, + rtol=1e-07, + atol=0, + ) diff --git a/validphys2/src/validphys/tests/test_weights.py b/validphys2/src/validphys/tests/test_weights.py index 0c1043fa44..134cf47e2f 100644 --- a/validphys2/src/validphys/tests/test_weights.py +++ b/validphys2/src/validphys/tests/test_weights.py @@ -8,9 +8,15 @@ def test_weights_have_same_commondata(weighted_data_witht0_config): data = API.data(**weighted_data_witht0_config) normal, weighted = data.datasets - normalds, weightedds = normal.load(), weighted.load() - assert normalds.GetSys(0, 0).mult == weightedds.GetSys(0, 0).mult - assert normalds.GetSys(0, 0).add == weightedds.GetSys(0, 0).add + normalds, weightedds = normal.load_commondata(), weighted.load_commondata() + assert ( + normalds.systematics_table["MULT"].iloc[0][0] + == weightedds.systematics_table["MULT"].iloc[0][0] + ) + assert ( + normalds.systematics_table["ADD"].iloc[0][0] + == weightedds.systematics_table["ADD"].iloc[0][0] + ) def test_chi2_arithmetic(weighted_data_witht0_internal_cuts_config): diff --git a/validphys2/src/validphys/theorycovariance/tests.py b/validphys2/src/validphys/theorycovariance/tests.py index 1a90302cac..6dd14d85c2 100644 --- a/validphys2/src/validphys/theorycovariance/tests.py +++ b/validphys2/src/validphys/theorycovariance/tests.py @@ -219,7 +219,7 @@ def all_matched_data_lengths(all_matched_datasets): """Returns a list of the data sets lengths.""" lens = [] for rlist in all_matched_datasets: - lens.append(rlist[0].load_commondata().GetNData()) + lens.append(rlist[0].load_commondata().ndata) return lens From 536614822a21eb6fcf9efd8479ad4451d5f9b245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 20 Mar 2023 20:39:19 +0100 Subject: [PATCH 086/204] Import XGRID from n3fit.io.writer --- validphys2/src/validphys/photon/compute.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index ec08c0e1a9..f8acefcb27 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -6,12 +6,13 @@ from . import structure_functions as sf from scipy.interpolate import interp1d from scipy.integrate import trapezoid -from pathlib import Path from eko.io import EKO from eko.io.manipulate import xgrid_reshape from eko.interpolation import XGrid +from n3fit.io.writer import XGRID + import yaml from os import remove import time @@ -64,11 +65,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.lux[i].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) - # TODO : once that #1537 is merged do: - # from evolven3fit_new.cli import XGRID - # self.xgrid = XGRID - self.xgrid = np.array([1.00000000000000e-09, 1.29708482343957e-09, 1.68242903474257e-09, 2.18225315420583e-09, 2.83056741739819e-09, 3.67148597892941e-09, 4.76222862935315e-09, 6.17701427376180e-09, 8.01211109898438e-09, 1.03923870607245e-08, 1.34798064073805e-08, 1.74844503691778e-08, 2.26788118881103e-08, 2.94163370300835e-08, 3.81554746595878e-08, 4.94908707232129e-08, 6.41938295708371e-08, 8.32647951986859e-08, 1.08001422993829e-07, 1.40086873081130e-07, 1.81704331793772e-07, 2.35685551545377e-07, 3.05703512595323e-07, 3.96522309841747e-07, 5.14321257236570e-07, 6.67115245136676e-07, 8.65299922973143e-07, 1.12235875241487e-06, 1.45577995547683e-06, 1.88824560514613e-06, 2.44917352454946e-06, 3.17671650028717e-06, 4.12035415232797e-06, 5.34425265752090e-06, 6.93161897806315e-06, 8.99034258238145e-06, 1.16603030112258e-05, 1.51228312288769e-05, 1.96129529349212e-05, 2.54352207134502e-05, 3.29841683435992e-05, 4.27707053972016e-05, 5.54561248105849e-05, 7.18958313632514e-05, 9.31954227979614e-05, 1.20782367731330e-04, 1.56497209466554e-04, 2.02708936328495e-04, 2.62459799331951e-04, 3.39645244168985e-04, 4.39234443000422e-04, 5.67535660104533e-04, 7.32507615725537e-04, 9.44112105452451e-04, 1.21469317686978e-03, 1.55935306118224e-03, 1.99627451141338e-03, 2.54691493736552e-03, 3.23597510213126e-03, 4.09103436509565e-03, 5.14175977083962e-03, 6.41865096062317e-03, 7.95137940306351e-03, 9.76689999624100e-03, 1.18876139251364e-02, 1.43298947643919e-02, 1.71032279460271e-02, 2.02100733925079e-02, 2.36463971369542e-02, 2.74026915728357e-02, 3.14652506132444e-02, 3.58174829282429e-02, 4.04411060163317e-02, 4.53171343973807e-02, 5.04266347950069e-02, 5.57512610084339e-02, 6.12736019390519e-02, 6.69773829498255e-02, 7.28475589986517e-02, 7.88703322292727e-02, 8.50331197801452e-02, 9.13244910278679e-02, 9.77340879783772e-02, 1.04252538208639e-01, 1.10871366547237e-01, 1.17582909372878e-01, 1.24380233801599e-01, 1.31257062945031e-01, 1.38207707707289e-01, 1.45227005135651e-01, 1.52310263065985e-01, 1.59453210652156e-01, 1.66651954293987e-01, 1.73902938455578e-01, 1.81202910873333e-01, 1.88548891679097e-01, 1.95938145999193e-01, 2.03368159629765e-01, 2.10836617429103e-01, 2.18341384106561e-01, 2.25880487124065e-01, 2.33452101459503e-01, 2.41054536011681e-01, 2.48686221452762e-01, 2.56345699358723e-01, 2.64031612468684e-01, 2.71742695942783e-01, 2.79477769504149e-01, 2.87235730364833e-01, 2.95015546847664e-01, 3.02816252626866e-01, 3.10636941519503e-01, 3.18476762768082e-01, 3.26334916761672e-01, 3.34210651149156e-01, 3.42103257303627e-01, 3.50012067101685e-01, 3.57936449985571e-01, 3.65875810279643e-01, 3.73829584735962e-01, 3.81797240286494e-01, 3.89778271981947e-01, 3.97772201099286e-01, 4.05778573402340e-01, 4.13796957540671e-01, 4.21826943574548e-01, 4.29868141614175e-01, 4.37920180563205e-01, 4.45982706956990e-01, 4.54055383887562e-01, 4.62137890007651e-01, 4.70229918607142e-01, 4.78331176755675e-01, 4.86441384506059e-01, 4.94560274153348e-01, 5.02687589545177e-01, 5.10823085439086e-01, 5.18966526903235e-01, 5.27117688756998e-01, 5.35276355048428e-01, 5.43442318565661e-01, 5.51615380379768e-01, 5.59795349416641e-01, 5.67982042055800e-01, 5.76175281754088e-01, 5.84374898692498e-01, 5.92580729444440e-01, 6.00792616663950e-01, 6.09010408792398e-01, 6.17233959782450e-01, 6.25463128838069e-01, 6.33697780169485e-01, 6.41937782762089e-01, 6.50183010158361e-01, 6.58433340251944e-01, 6.66688655093089e-01, 6.74948840704708e-01, 6.83213786908386e-01, 6.91483387159697e-01, 6.99757538392251e-01, 7.08036140869916e-01, 7.16319098046733e-01, 7.24606316434025e-01, 7.32897705474271e-01, 7.41193177421404e-01, 7.49492647227008e-01, 7.57796032432224e-01, 7.66103253064927e-01, 7.74414231541921e-01, 7.82728892575836e-01, 7.91047163086478e-01, 7.99368972116378e-01, 8.07694250750291e-01, 8.16022932038457e-01, 8.24354950923382e-01, 8.32690244169987e-01, 8.41028750298844e-01, 8.49370409522600e-01, 8.57715163684985e-01, 8.66062956202683e-01, 8.74413732009721e-01, 8.82767437504206e-01, 8.91124020497459e-01, 8.99483430165226e-01, 9.07845617001021e-01, 9.16210532771399e-01, 9.24578130473112e-01, 9.32948364292029e-01, 9.41321189563734e-01, 9.49696562735755e-01, 9.58074441331298e-01, 9.66454783914439e-01, 9.74837550056705e-01, 9.83222700304978e-01, 9.91610196150662e-01, 1.00000000000000e+00]) - + self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() self.produce_interpolators() From 92a511709d16340162a0380814481b9d979a0788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 21 Mar 2023 11:06:12 +0100 Subject: [PATCH 087/204] Fix test_compute --- validphys2/src/validphys/tests/photon/test_compute.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index f2642bb8b5..7c7a88c740 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -4,8 +4,11 @@ import lhapdf import numpy as np from collections import namedtuple +from pathlib import Path class faketheory(): + def __init__(self): + self.path = Path("/fake/path/") def get_description(self): return { "alphaqed": 0.01, @@ -92,9 +95,9 @@ def test_init(monkeypatch): np.testing.assert_almost_equal(photon.alpha_em_ref, faketheory().get_description()["alphaqed"]) # test masses - np.testing.assert_equal(photon.ktThr, np.inf) - np.testing.assert_almost_equal(photon.kbThr, 4.92) - np.testing.assert_almost_equal(photon.kcThr, 1.3) + np.testing.assert_equal(photon.thresh_t, np.inf) + np.testing.assert_almost_equal(photon.thresh_b, 4.92) + np.testing.assert_almost_equal(photon.thresh_c, 1.3) # test set_thresholds_alpha_em np.testing.assert_almost_equal(photon.thresh[5], 91.2) From a2d7c4a231ceb9a834ec7af76048b6444ad291d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 21 Mar 2023 11:07:56 +0100 Subject: [PATCH 088/204] Fix test_structurefunctions.py --- .../src/validphys/tests/photon/test_structurefunctions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index 97b711e503..a60b280e0b 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -26,7 +26,7 @@ def xfxQ(self, x, Q): f2lo = sf.F2LO(pdfs, fake_theory) - np.testing.assert_equal(f2lo.ktThr, np.inf) + np.testing.assert_equal(f2lo.thresh_t, np.inf) for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): From 0843efd573e4d98d92701a75617d590bb02106d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 21 Mar 2023 11:43:53 +0100 Subject: [PATCH 089/204] Remove imports --- validphys2/src/validphys/tests/photon/test_compute.py | 3 --- .../src/validphys/tests/photon/test_structurefunctions.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 7c7a88c740..920e977855 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -1,4 +1,3 @@ -import pytest from validphys.photon.compute import Photon from validphys.photon import structure_functions import lhapdf @@ -26,8 +25,6 @@ def get_description(self): fiatlux_runcard = { "pdf_name": "no_pdf", - "path_to_F2": "/", - "path_to_FL": "/", "additional_errors": False, } diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index a60b280e0b..a82b881c54 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -1,7 +1,5 @@ -import pytest import validphys.photon.structure_functions as sf import numpy as np -from pathlib import Path import pineappl def test_zero_pdfs(): From 1211b60911b9024d2c9ec85745ff294ad054bbf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 21 Mar 2023 11:50:13 +0100 Subject: [PATCH 090/204] Copy theory.db from master --- nnpdfcpp/data/theory.db | Bin 102400 -> 106496 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/nnpdfcpp/data/theory.db b/nnpdfcpp/data/theory.db index 15ef2e27da09dbfaa563f1a32bca2bab0d21ba12..0d064ee46054b78e548595d0a9dd6e19af84d9f0 100644 GIT binary patch delta 486 zcmZozz}9epZGtpw3Q_l48wbljq8(GmCKqY-U%u z%gDqLF!_n1H**3<#KuAyHUV~KCML#y#(ogZl*5$MoS?BiL4$FLx+*u*Oa{K&yc>Di znPzh5aVc{=V4uSl$tuVEh%t#l6=*aA1M}X=yoGAp{R0@im{^-!oB6dSD<-KjHL_1v z^klT3)U$B9e=nnmZccu=LP1e#a%OR6ex5>lQD#c9jzV%;Vsc4-QL#c=QGTvMNQQ!u zfkBf+v)J^U3`Vu-KqL94%cL>BnrGyBJwnMRY2nVw$ zfA?f|0UhRv{N2+H9T+P&uQ#~ECj`_g(QFJP#hcZDq*$}$JYb*07Rf5d z{D^5DV-iq`fuYH=S#rAL97eV6{sD|$Oxxwt8NC>pEtX0AcG|YXATM From b2ef6597f3a2a1a6f674cd464f58e6e15cf29205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 21 Mar 2023 11:53:17 +0100 Subject: [PATCH 091/204] Add theory 522 to theory.db --- nnpdfcpp/data/theory.db | Bin 106496 -> 106496 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/nnpdfcpp/data/theory.db b/nnpdfcpp/data/theory.db index 0d064ee46054b78e548595d0a9dd6e19af84d9f0..a683bd2f9b578cb25a6b7ba0eb43c123bfe89e69 100644 GIT binary patch delta 139 zcmV;60CfL=zy^T829O&8U6C9^5nTW-V*m{g_y7V2@Bjz!55cz~fB}F9v+xHL7y}mw zNV5kZDgpsWvoInS29r=I1Ox&K0t$mrD7R230kl^P1_+-43i}Sg4Sxuqw+?^-j|c;T t7=)9N+$WR0d=ir&VbqtPB>@BsQAI=`ZfD7R230kl^N1qr(W3i}Sg4S%-~fB}yPv49={1cE$-Y?G0+ KCb#8@0UH*g@gWre From 2a7bbfc555fab32f1cd5e803a526f72ba720414a Mon Sep 17 00:00:00 2001 From: Zaharid Date: Wed, 22 Mar 2023 15:54:41 +0000 Subject: [PATCH 092/204] Update conda-recipe/meta.yaml Co-authored-by: Roy Stegeman --- conda-recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index e8fda71127..5ec22111d6 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -55,7 +55,7 @@ requirements: - eko >=0.12.0 - banana-hep >=0.6.8 - lz4 >=3.1.10 # see https://github.com/conda-forge/eko-feedstock/issues/10 - - fiatlux # [linux] + - fiatlux test: From 5ab37a4fd97c50c3f84085f9aed74633a1098c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 23 Mar 2023 10:49:12 +0100 Subject: [PATCH 093/204] Make sure theory.db is the one from master + 522 --- nnpdfcpp/data/theory.db | Bin 106496 -> 102400 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/nnpdfcpp/data/theory.db b/nnpdfcpp/data/theory.db index a683bd2f9b578cb25a6b7ba0eb43c123bfe89e69..3c25e981367d144f66a32f100da96acc13972a16 100644 GIT binary patch delta 587 zcmW-dO-K}B9LC@Oyt6wq>wFmWp(*&T(6oxi7F0`UuKBTUNt#gXVFOopY!oX47ZE8UF9`(=OD*bce%qnKB6p`Tx}aJDGSzVKg?BEyj!wp zqjiSY)?-GqX*J!(b-cgIkmHmpBm?(H`9@v7E@ zMxAWGQQK;|$4)SL!Ey(o%OWbQoQ=jZQ}M{S7A0ftsp&|1l4SL?meG^(Xd6jpbuB#; z%V;E$A;I3xGhT0@&rtry(De}@fIIT@W@%NGXLy0>AG$g?*xu6@>b9<>;fv*Og&N{( zBid|aJWDdk_;fawOpxS+-Jy|@Kwm-4Ck|+f1fhYx4}%Xkif9|jucVyV#K^vviGq%ucc&!hkHYwc!is{xA(=GsbPT9jd-7bC1LS_r0~_#@436jvw+i_IOy z3q@N;_#z2*71SUb6TV;sO1DK3L1`3GT|~QRmqt;Yll=kRycaJ#pZ9s5H+4r!tw}d4 z#7;uUo=l)|M<+RQnP{FZBFIG|7d`8_cv8-PlS>s^=9x5QZ;d$V$#|wS@zMMP?`yrp z68a3kHOq~oHpB>;%s)tdN^@Qgyn~>`%+LGIi_J>6(w2*tNM=(=bzly2`R33m81rF1 zAI5O7+AS;){jHW$r-TLfkgH1hATLVoVkP~EUQ%t5389ZoEgodCR%qtz&ldKu7Yppd zCAeAF4Tx-W1a3Y)h~wbD+HnrF-Y8?seb`p;u8U}uX{q5gGb=%aSnY(ktOm0@rxs?jjFY-g7dEaM7c`agq`GBCcZ4lw|Pwhuc#~CVC)Z* z^hsm)e_2ZkJllsx@Ph;Ji>Nefzgq4Fpz**JNC;?FN|kI7*(Yu~7HO+cPBye_>S5O> z=c2x@beOfVRjI+K2@{y!colmw2HkwV)xskq_%88=88m^f&f*Ll>8KH5mdPMXU5Czp z591z1-QS@0g1sL@fz9 Date: Thu, 23 Mar 2023 10:51:57 +0100 Subject: [PATCH 094/204] Take meat.yaml from master + fiatlux --- conda-recipe/meta.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 5ec22111d6..35e97c3a63 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -24,8 +24,7 @@ requirements: - python - numpy run: - # Only install tensorflow and fiatlux on linux. Select eigen build - - tensorflow >=2 *eigen* # [linux] + - tensorflow >=2 *eigen* - psutil # to ensure n3fit affinity is with the right processors - hyperopt - seaborn @@ -54,10 +53,8 @@ requirements: - pineappl >=0.5.8 - eko >=0.12.0 - banana-hep >=0.6.8 - - lz4 >=3.1.10 # see https://github.com/conda-forge/eko-feedstock/issues/10 - fiatlux - test: requires: - hypothesis From d35c557ee9fc1df806a30fbd465e8278b6dcfc63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 23 Mar 2023 10:54:27 +0100 Subject: [PATCH 095/204] Restore top level import of fiatlux --- validphys2/src/validphys/photon/compute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index f8acefcb27..0938260800 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -13,6 +13,8 @@ from n3fit.io.writer import XGRID +import fiatlux + import yaml from os import remove import time @@ -53,7 +55,6 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.path_to_eko_photon = theoryid.path / "eko_photon.tar" # set fiatlux - import fiatlux ff = open('fiatlux_runcard.yml', 'w+') yaml.dump(self.fiatlux_runcard, ff) self.lux = [fiatlux.FiatLux('fiatlux_runcard.yml') for i in range(len(replicas_id))] From 318e73552869c4fbb74d5e0c9b943e13b0843352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 23 Mar 2023 22:47:29 +0100 Subject: [PATCH 096/204] Change FxQ -> fxq --- .../src/validphys/photon/structure_functions.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 412acde066..8da165af04 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -24,12 +24,12 @@ def produce_interpolator(self): grid2D = predictions.reshape(len(x), len(q2)) self.interpolator = RectBivariateSpline(x, q2, grid2D) - def FxQ(self, x, Q): + def fxq(self, x, q): # here we are requiring that the grid that we pass to fiatlux # has Qmin = 1 (fiatlux doesn't go below Q=1) - if x < self.xmin or Q > self.qmax : + if x < self.xmin or q > self.qmax : return 0. - return self.interpolator(x, Q**2)[0, 0] + return self.interpolator(x, q**2)[0, 0] class F2LO : def __init__(self, pdfs, theory): @@ -48,7 +48,7 @@ def __init__(self, pdfs, theory): ed2 = 1. / 9 self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t - def FxQ(self, x, Q): + def fxq(self, x, q): r""" Compute the LO DIS structure function F2. @@ -65,16 +65,16 @@ def FxQ(self, x, Q): Structure function F2 at LO """ # at LO we use ZM-VFS - if Q < self.thresh_c : + if q < self.thresh_c : nf = 3 - elif Q < self.thresh_b : + elif q < self.thresh_b : nf = 4 - elif Q < self.thresh_t : + elif q < self.thresh_t : nf = 5 else : nf = 6 res = 0 - pdfs_values = self.pdfs.xfxQ(x, Q) + pdfs_values = self.pdfs.xfxQ(x, q) for i in range(1, nf+1): res += self.eq2[i-1] * (pdfs_values[i] + pdfs_values[-i]) return res From 011df81fd5324969626e97d40699fa9e5a83d150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 23 Mar 2023 23:32:42 +0100 Subject: [PATCH 097/204] Use LHAPDFSet in compute --- validphys2/src/validphys/photon/compute.py | 25 +++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 0938260800..5875ad0184 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -1,8 +1,8 @@ """Script that calls fiatlux to add the photon PDF.""" -import lhapdf - import numpy as np +from validphys.lhapdfset import LHAPDFSet + from . import structure_functions as sf from scipy.interpolate import interp1d from scipy.integrate import trapezoid @@ -24,7 +24,8 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard self.replicas_id = replicas_id - self.q_in2 = 100**2 + self.q_in = 10 + self.q_in2 = self.q_in**2 # parameters for the alphaem running self.alpha_em_ref = self.theory["alphaqed"] @@ -43,14 +44,14 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.set_thresholds_alpha_em() # structure functions - self.qcd_pdfs = [lhapdf.mkPDF(fiatlux_runcard["pdf_name"], id) for id in replicas_id] + self.qcd_pdfs = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") # TODO : maybe find a different name for fiatlux_dis_F2 path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" path_to_FL = theoryid.path / "fastkernel/fiatlux_dis_FL.pineappl.lz4" - f2 = [sf.StructureFunction(path_to_F2, pdfs) for pdfs in self.qcd_pdfs] - fl = [sf.StructureFunction(path_to_FL, pdfs) for pdfs in self.qcd_pdfs] - f2lo = [sf.F2LO(pdfs, self.theory) for pdfs in self.qcd_pdfs] + f2 = [sf.StructureFunction(path_to_F2, self.qcd_pdfs.members[id]) for id in self.replicas_id] + fl = [sf.StructureFunction(path_to_FL, self.qcd_pdfs.members[id]) for id in self.replicas_id] + f2lo = [sf.F2LO(self.qcd_pdfs.members[id], self.theory) for id in self.replicas_id] self.path_to_eko_photon = theoryid.path / "eko_photon.tar" @@ -64,7 +65,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): for i in range(len(replicas_id)): self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) self.lux[i].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) - self.lux[i].PlugStructureFunctions(f2[i].FxQ, fl[i].FxQ, f2lo[i].FxQ) + self.lux[i].PlugStructureFunctions(f2[i].fxq, fl[i].fxq, f2lo[i].fxq) self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() @@ -197,11 +198,11 @@ def compute_photon_array(self, id): if pid == 22 : pdfs[j] = photon_100GeV ph_id = j - if not self.qcd_pdfs[id].hasFlavor(pid): + if pid not in self.qcd_pdfs.flavors: continue pdfs[j] = np.array( [ - self.qcd_pdfs[id].xfxQ2(pid, x, self.q_in2) / x + self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x for x in self.xgrid ] ) @@ -248,10 +249,10 @@ def generate_error_matrix(self): """generate error matrix to be used for the additional errors.""" if not self.fiatlux_runcard["additional_errors"] : return None - extra_set = lhapdf.mkPDFs("LUXqed17_plus_PDF4LHC15_nnlo_100") + extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") return np.array( [ - [(extra_set[i].xfxQ2(22, x, self.q_in2) - extra_set[0].xfxQ2(22, x, self.q_in2)) for i in range(101, 107+1)] + [(extra_set.xfxQ(x, self.q_in, i, 22) - extra_set.xfxQ(x, self.q_in, 0, 22)) for i in range(101, 107+1)] for x in self.xgrid ] ) # first index must be x, while second one must be replica index From 50cdb52b93587dd882b4524a53b2a6dfa1b14c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 24 Mar 2023 10:35:00 +0100 Subject: [PATCH 098/204] Fix tests and add docstrings --- .../validphys/photon/structure_functions.py | 21 +++++++++++++++++++ .../validphys/tests/photon/test_compute.py | 4 ++-- .../tests/photon/test_structurefunctions.py | 4 ++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 8da165af04..5a1d6512a8 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -3,6 +3,10 @@ from scipy.interpolate import RectBivariateSpline class StructureFunction : + """ + Compute a given DIS structure function convoluting an FKtable + with a PDF. + """ def __init__(self, path_to_fktable, pdfs): self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs @@ -11,6 +15,7 @@ def __init__(self, path_to_fktable, pdfs): def produce_interpolator(self): + """Produce the interpolation function to be called in fxq""" x = np.unique(self.fktable.bin_left(1)) q2 = np.unique(self.fktable.bin_left(0)) self.xmin = min(x) @@ -25,6 +30,21 @@ def produce_interpolator(self): self.interpolator = RectBivariateSpline(x, q2, grid2D) def fxq(self, x, q): + r""" + Compute the DIS structure function. + + Parameters + ---------- + x : float + Bjorken's variable + Q : float + DIS hard scale + + Returns + ------- + F_{2,L}: float + Structure function F2 or FL + """ # here we are requiring that the grid that we pass to fiatlux # has Qmin = 1 (fiatlux doesn't go below Q=1) if x < self.xmin or q > self.qmax : @@ -32,6 +52,7 @@ def fxq(self, x, q): return self.interpolator(x, q**2)[0, 0] class F2LO : + """Compute the LO DIS structure function F2 given a PDF""" def __init__(self, pdfs, theory): self.pdfs = pdfs # TODO : maybe they shoud be kDIS instead of k, but usually they are the same diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 920e977855..5052c1e0f4 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -63,7 +63,7 @@ def __init__(self, path, pdfs): self.path = path self.pdfs = pdfs - def FxQ(self): + def fxq(self): return 0 class fakeF2LO(): @@ -71,7 +71,7 @@ def __init__(self, pdfs, theory): self.pdfs = pdfs self.theory = theory - def FxQ(self): + def fxq(self): return 0 diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index a82b881c54..376b4ff172 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -28,7 +28,7 @@ def xfxQ(self, x, Q): for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): - np.testing.assert_allclose(f2lo.FxQ(x, Q), 0.) + np.testing.assert_allclose(f2lo.fxq(x, Q), 0.) class FakeFKTable(): def __init__(self, path): @@ -70,6 +70,6 @@ def test_F2(monkeypatch): structurefunc = sf.StructureFunction("", fakepdf()) for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): - np.testing.assert_allclose(structurefunc.FxQ(x, Q), 0., rtol=1e-5) + np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) \ No newline at end of file From 5da88a64bc7c4dc7edd5d6b475492c9a87ff0c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 24 Mar 2023 14:30:43 +0100 Subject: [PATCH 099/204] Fix tests --- validphys2/src/validphys/photon/compute.py | 2 +- .../validphys/tests/photon/test_compute.py | 8 +-- .../tests/photon/test_structurefunctions.py | 67 ++++++++++++++----- 3 files changed, 57 insertions(+), 20 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 5875ad0184..82c1b9ac68 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -24,7 +24,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard self.replicas_id = replicas_id - self.q_in = 10 + self.q_in = 100 self.q_in2 = self.q_in**2 # parameters for the alphaem running diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 5052c1e0f4..0e5e839ef5 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -1,9 +1,10 @@ from validphys.photon.compute import Photon from validphys.photon import structure_functions -import lhapdf import numpy as np from collections import namedtuple from pathlib import Path +from validphys.lhapdfset import LHAPDFSet +import fiatlux class faketheory(): def __init__(self): @@ -24,7 +25,7 @@ def get_description(self): } fiatlux_runcard = { - "pdf_name": "no_pdf", + "pdf_name": "NNPDF40_nnlo_as_01180", "additional_errors": False, } @@ -78,8 +79,7 @@ def fxq(self): def test_init(monkeypatch): monkeypatch.setattr(structure_functions, "StructureFunction", fakeStructureFunction) monkeypatch.setattr(structure_functions, "F2LO", fakeF2LO) - monkeypatch.setattr(lhapdf, "mkPDF", lambda *args: fiatlux_runcard["pdf_name"]) - import fiatlux + # monkeypatch.setattr(LHAPDFSet, "mkPDF", lambda *args: fiatlux_runcard["pdf_name"]) monkeypatch.setattr(fiatlux, "FiatLux", fakeFiatlux) monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index 376b4ff172..d28635b193 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -2,15 +2,16 @@ import numpy as np import pineappl +class ZeroPdfs: + def xfxQ(self, x, Q): + res = {} + for i in range(1, 6+1): + res[i] = res[-i] = 0. + return res + def test_zero_pdfs(): - class fake_pdfs: - def xfxQ(self, x, Q): - res = {} - for i in range(1, 6+1): - res[i] = res[-i] = 0. - return res - pdfs = fake_pdfs() + pdfs = ZeroPdfs() fake_theory = { "mc": 1.3, @@ -30,7 +31,11 @@ def xfxQ(self, x, Q): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(f2lo.fxq(x, Q), 0.) -class FakeFKTable(): +class FakeSet(): + def get_entry(self, string): + return 0 + +class ZeroFKTable(): def __init__(self, path): self.path = path self.xgrid = np.geomspace(1e-4, 1., 10) @@ -47,11 +52,38 @@ def bin_left(self, i): def convolute_with_one(self, pdgid, xfxQ2): return np.zeros((10, 10)) -class fakeset(): - def get_entry(self, string): - return 0 +class OnePdf(): + def __init__(self): + self.ao = 1 + + def xfxQ(self, x, Q): + return 1. + + def xfxQ2(self, x, Q): + return 1.**2 + + def set(self): + return FakeSet() -class fakepdf(): +class OneFKTable(): + def __init__(self, path): + self.path = path + self.xgrid = np.geomspace(1e-4, 1., 10) + self.qgrid = np.geomspace(1.65, 1000, 10) + + def bin_left(self, i): + if i == 1: + return self.xgrid + if i == 0 : + return self.qgrid + else: + return 0 + + def convolute_with_one(self, pdgid, xfxQ2): + return np.zeros((10, 10)) + + +class ZeroPdf(): def __init__(self): self.ao = 1 @@ -62,12 +94,17 @@ def xfxQ2(self, x, Q): return 1.**2 def set(self): - return fakeset() + return FakeSet() def test_F2(monkeypatch): - monkeypatch.setattr(pineappl.fk_table.FkTable, "read", FakeFKTable) - structurefunc = sf.StructureFunction("", fakepdf()) + monkeypatch.setattr(pineappl.fk_table.FkTable, "read", ZeroFKTable) + structurefunc = sf.StructureFunction("", OnePdf()) + for x in np.geomspace(1e-4, 1., 10): + for Q in np.geomspace(10, 1000000, 10): + np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) + monkeypatch.setattr(pineappl.fk_table.FkTable, "read", OneFKTable) + structurefunc = sf.StructureFunction("", ZeroPdf()) for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) From b8a95f6b18f30498e7ad5a5442f4421828dc99b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 24 Mar 2023 17:01:38 +0100 Subject: [PATCH 100/204] Fix theory 522 in theory.db --- nnpdfcpp/data/theory.db | Bin 102400 -> 102400 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/nnpdfcpp/data/theory.db b/nnpdfcpp/data/theory.db index 3c25e981367d144f66a32f100da96acc13972a16..f3c94c9d3ef9e3d0ec07ce67f0037376f4e79eb2 100644 GIT binary patch delta 10547 zcma)C3s_TEwmv)0^GZVGjSzyO2qFS1pke{V7odPrd;yApqC6xP->3w%^+CnB)nli& z1!bm=*E*uuIzHysmTRqpXgk!lN~^Zo-de5NVtXA?XP<H29v>D|9Sc)R4-zm-qra0aV6CQ&~F zg9z*+N8dd)>v@T<=Dy~R=C@BvF?O@y3R`xX8!X9Fq*uCsdwvOGvRNM3t z9ip13z0^*sj@m#~QI%9FRY>JfSyUR8OvO{fsc6k4`d>ZoxK`7J5v>y<9R)965A49`|>qi=*ad;V9?5F2Z~ltMA4<5BRrioE^?lec-(r zLT>REp3N-5`mu76my z^kei<5i!Cneod$^s2bQcOb#7JK34uy@s@mG5eq9wjI89sow=ixKK!>xSJNvDE# z%rD0}vWj(Ft{+gs_yRohW+i7*f?znS5LX;62w5^p$SG48T+Bu=5TB^F)$F#8pJ8MI{?R#xJ9#4K%DhC2%K0wKM;8l?}c_JvK$i?HA{ChBcE ze|Txdd`vfLB|o2o)e(?dh0o>rtN0*;)%f~%wnPWdt=Wv^L)R?9Y`nYX0Fq@??ZRZ% zwbC6nuFVsbpIV2@tqvo$s}u6e3-yKhWfhCc7wVTRSm0PmYGNLtzNz|{-Otp~Ytf!5 zR_P&sD7zv(C20T~iFqbNJS?rAto0M@T&%r>!R^M$*r>3uKG?{>gEjcZ$WF<^bG1p@ zkHjroGWlR53vX^na@M^SgzsgrE-w9rfx6e6u;)2UPhsa0H)zb9KJO_u0^%w+fux|#5;uw@uv#~Q~}qT%Y` zI5^d}HQ^s45~}W*iUkR)_#_bKaXguCob!O~|Eq?byBS!$bF7Ot54Y-UZ3e@`yHZ_X zd+lq_2O-<8u)TKq4cFmaEY$50`k8b}2Zy|YZ^o=?;N=W_`A_4V5d$qF7R=H&$2sGr z4O72&x*(oOr$S-NTVnDsc=hcR7yX6}jTNRjkWimPD`5pQ3GHWO$~%h7D9u}>3nj@Q z&}4WD2RBaEt`?-s>TVcJ--j<780+}zG3<9*Jq+C0l%l;LZn?KGPW;(v_f`;=2JZd@ z;7P?N{CiBr{x9ixn7NT(Mfr%|eHV)ur=J8oVeWw~HYEwwf5ib#>;%EV2eGN%-cAsl z@?Vo&(vY9utrTZIVd2_Ci7xP#7~eP-2M@pZk_%kLC%j$l`3Tr?*cHxVkEb>B2_sxc z!ANVt&!b0EUdo4xHu<}<8flp%8;oo=cmrvT-%1#DXCS=Mrh;pl`+*q{OZS|kGpz!1 zz?UAIVg0`)QLnXRXv4(QjT(f(qsLRUSDorZ;g$EX1X+4U?Ly&`55(pAzl+PqpG?&* zb=ps*wx3fSTB=S9_-r0S%Ng;$zWzt5Gm7&*M-O+bM>R2kP=BHNko_xKPs^y8=nkTf z{AbxosY$W~q#`BEI4eGc%8#<+M4DY?_yeZf=zq;bI4y@)``1H#ZX9dv<3i>+5S!MTv#$R<>-c}F;i?6vDa9yV$f#%{n z`i;2!t8Yc+is-nSenurqWg;x!p>~WrZ!&V2;Brt!)D!BFdV1!BBT6PiKjL$f z;i#-8%^1!-U_c1$>>6rkI09xoTng%W<8d(SH~bRj;Z{54TV3|2O*`eiM=o04d!&UE z|0oda7cszm`QQt;{vUo-V(Z%q=nLB)<9DF~fG?z<;5P*}0Pk81Ozoa4*1yH?_Qy2b z{nY-Tg1Hw1&ppG}tf1%n(VBGfJ01M~d4Q}Y#u$WZF%bUr3=eHDXy46%FvsbY0a1h+ zs$!TS<8{7f+wgk9z79~>y+_RVR}SnCRz1F1 zW9!R71+brLFz*jIi`!&=5GrmY1?)L(ZJR^`r){Tkw;3RwF4SxqH&zAkn+3aMPU9NX z!2ZN*?GmNhJSsPNO|EfB7}Pu&z@1S5HN5{IOwBRGIXUI)iGt(FsMgT5Dh2mQVK$NR7j))RTv5Wvmt?@$AL z>Y4|FLEN@rkS6j1%d&ezh%oN>06Z@}ClaKovh$_KU?}%I1Q~RKtJc9q6$*V@~UjE!=vVij?p zP=BU6%&ujo(mL{i@>9i5`5f7J=@5yAxNkCCYRhk$o?jfPC#4CkV5R z^HZHrKsmvDJdD{Lg^#|6!+g_WJpjk@V!<%WjY~DktQUDvS*wVXJ(4y{%E2&x8zZZU zt@Y$rNiHKB#qwe}@as)cul;Cl`*ct&rg>9L7|qRHi}pSt*+6hi=DBXRCgLQ9Ko4iMZ6TXI^G1F&r|meUCCNG>9+U>>%ET+Kre z+WBNYND&=f6J#vmkL}!z0idtf<7@pc-=6QHMMSh<8f1y0Dy%Kq|i zWsTATNdR%(WcY7o&8XU-P=ewz0)RJ6oXBz+3sJiA=7YY1XrKs&aP@^qom~L%7ipGt zz^cYV(wk>X!`(}PoU3>pFx;IYV87+=nXP&{IJk|^@4KLLctUN;JmgQjyip%rZnmId^p#!WhfM>!yc^N6P{J;|^P(V!6DPfRV=4 zdU1a%M9ZhSM&@RU2i;71Ik$Hep!pwL**^TwepsTFP#;$nv;FCBs6FHyWq|xk*&EW? z5{76s8JalFQZQTKXMQCZ!A*QcWbQ|rJpy>+YA}qiJG0C7dv9~rR)aXsX9d9D6!tjE zH=MQ7VK>*)rpF}m3jgtpt>IqrVfa>wOk~yp3OQ+52b#nU{dp#Ela8+K2UP|XexCDz z`M_U@wQ}A}-GKRkw)=e@p8{ICj8Gp^6|n>9AE>v7g%LnO{(;FIv8&bXn!1CFM2o z8%oWSf&Nu6NHEfeuL4dWL=)IffsjwX)g6XAwpoPbW`>|DcjJEBB2J)DTx6ZdTxzq8 zs$7^?vsqNz!bBm>D0NgsWT?Jk$)fVgn&jF@F1!j=M)fObZGBz=`a5$&!`cc0efbX+ Cxt7rY literal 102400 zcmeI534Bvk*1&JlEiY@?%OX6=)2UPX=^!3xG@yW{0RUR~Z!G*rZW?$en*0DY`q|%23({IX zF3Gi8eA&Za&`W4p7i)!Egv zp{%pn*)O=OEm~Y$SXWHf70xd&rdJ7`Sw$!5sa0hQR#8-0WnJ;Y;>C1jRUKWqq`aJ7 zQdw5Bq&NjWUtL$_AFpgFK~G5kkI)yZaP(KU6!xz2{bY4>i}z=1tX_YaqrJP;0Y6aw z^=f*Y@T0svQS1#k`g-`o);gQLew?!zf1vyq)%GuT!f)X9|EadWst10Ym%mYIjn{`e zn!JB!ACIqTs%!1>`i_o9uMgMwczjKx&kySJ@j>1nU(@I7gNhbaz)zA}YsGS}zo25N z*M}E+H@e!gN_!oAhy2e|SK-s-k(0>ZP}J4Y;q2^#e^T^aQ|Q(0U5$?Rm7V=`S!bVf zwX-LMF0O5Iv^(n^J+#!y=>^3lg-gooXd5)B*)!?tp0*B0&jxyta|1ndww|h9Tvk!I zcp1H@cv(`}f@JPRMXk=JHNE)xqSE4`MM==*;V08cw3SYwZSbF+PDZcgo`r7DJv*ZW z-kkxTp0T_lwVvEBsj4h5Pc6icbNA;9?ibAveiVun!rvtLEoDVMp1_fU{{qDg-0vxF zsK&peZgHjP^I`0u&!?@P?+$~8_yH6xxDQK*|05ajUncyQCH#_n)KKy&7(V>Gti~$- z{agS+kI{vd3uykwd9=;*olztazp18p0rzXTzmI?#2_fPSgfWE{eVc7EYv(ch(`N&LHO&;xnGKNOBElE*q4 zrSF0dh~I_6h~%-3KIyyQ1LAj~m?3$rV!8NT6-&iGFO=937XZ*}@ZXXDQ3M@E0v~vx z6Cf+qCueG7$CORcfHrgu?_JvtNAoKk>-xIz&npF&?v*yd4^c3)(4p)i2L2mm`8m2f z*79G=Y0DRuPc0u?KCrxNIcj;$^0MW5%Tty=T7GYN*s{}dpXCnAt(F_Om!r3w>l+(y zP6pI?N;iU{G#U*R7pH^&4I2}O?xO#g$#L*T#pUTJK9E5Fj){rMKbV=n@v|FR3+i7& zx0my;*wggt$=B-hzw5m9msNkCTV9SWF8|c+4}4hr@4rM9#P6#)dDX;yZrxnpf^G%8 zwx;D~XV`OSYnt7jnQn#JYE93`Mt9%?cBFH&(=#&B6B#zEm3u`_mK{G}#qY??&9!CP z;WyIU*O~TcLEYH?*T4U2@A)Uzefhg*k8R0+dHZJ@yAI6A|2#V3roR1`q^DWw%JM22 ze@{ATmc6%aHQmve%nmmt1C0VC`8rC=F)0d+$!CzrD{qYnRVD%`sctCo#W=sK;VbB^IL_q{XNMjK^XmTt>$!vl!?}gS!@mL5xYIOB-sJE<_h< zINGXk^!7S?*^#lz45nBXVwB}AA^eI8-m9>zuuPY+|&~ zdVFJ&S&tei`GPboN@z=DXdt>i~%L%xtL9+1bC2 zW&xDLm>ekMnN%n(%w#Cd%t$CrOdOQP4Nw@+JN4*|6ndYo2#UBYDB!gS4*VxT5pCAP zK7N9(09d|*WB*f@e_8%vc@K{KU$wksdCqdk@(0Tv%R`nOmU}I~vTU_%vHZ+(rR54s zzop00ZfUWsvMjgMSt>21mid->mMn|aGSf2EGSQM?8E!FJVlBY@t@&&7=jIdU56thF z-!LCBzi9rG`Jj29`BC#O^MmHQ&9|9vHeYAnY-Y_H&FjtG=1a`Y<`w1!^I~(kxx`#x zo@351rl!H z@7H(hTlK5-4f<+*slGs;qqpjl^t3)fAFtQx0d9*)L>o)27b=|sF-6~y!u0U62t)`(2n=BYaZy^WRUUP{fOoftaod6WBst^VXPn0JcRWw z%`U8etNAU~ztQ{#>z$gNSnts6!1~vkUt|5C=0U6<&^&wkcMVErNZ5bF=X2Ux!k-pBelIF9w-!QZid z54?x@Mo-F1TSL!0(b%I!{9L1&x7Z& zehxf`^|Rnvtp5c5g!MDv8LXcMPh035)2KiH4;AHg57 z{sZ^})=z*Zu-*suVf{FG9P7OR+dcR_z;+Mz0BrZ*F@WtJJPNSggGT_id$1c|y9W;g zZ1>egN(S*bl%R0Q&*>6~KM~ZU@*8z-<8g0oV?( zAAoHD`vJHWU_Ss`0rmrM3&4H=ZU)#7z)b-A0k{!hKL9rX><3^Az>V2d=~4 z^OxY4SpNe20_&fHpJV+q@H4Eh1=nJ|8EnS-8gLEPSA(muz6xB0^_AdCtX;r`H49j* z8Ngt@32cgq(rAAAQzWmr0?ErSM{?svB!BV~Brm%R$qgHj?C(eN(o2zCzaGhT>yYg0 zL$bFQ$(|l0*RDmfyBo=_E+jiUk?iO|vb`P2HEWQ(#tbB< zPe*dvG$f}^MRLj%Boh;noIDvxnnrTcBqS$JMDo1zkeo09$?@Zn95)Whv15@;NI-JT z7$irJMsn0BBu9=!a>NKEhYv?`*f1pHM{KhwL$7Yz67FQ-~`6>*ud@HV6C(*GxOX9RX4>2pW=|722;xXT^s|GTGKN&4KQBK?0f zL6c6BaEbK)eKfiw{r^b+-|y_sNdI5q99XZDBqIHPG(#BKkf+-n>Hl++raeSLxc`67 zrjZLxgd6{t0`p1JQR6d)-TK?99^JyYDX}rp$D^LqZq;-FMZI2j>IC6b6K;+YH9|hI zLNkBOLvvU!2z z)uC)mbfcr!+1}QPH|Y>s6hxK9P8=%-<<0HpNC~1_p*|%e4Gr2DgeD*E3GB4yE3@^~*hH%`}{|-F$pT4xMua&OuYIoA> z9qsF2M4CQQ7?H@NTsf)>-631)6x47cg{N^33~KlY?=~e}>^bO_jh1Ef3EB#B$;^Z#DiqGx*wXDxOFVuEGLD32Zt%_O{uHY)!j;5rJrsPy-cQ2jP=q`|@ zXC<@arYd7qC}VYW!I2R=XR4LeEXazTHpLqpd1-Z#6wcR;6>wXJ@BG6FdL`sJ>4fm+ zgbhh%L>bBK=tO0l;7U<(i2C}HEB+seWd#q^V3}L7BJRvA04QU<}X$txQMml5U<)Pd|lv@;ONOR-=kAV3N(=OvR zhE{zR6%+Sn?9Lb_dRf#w?RnrM6%7B`%q(H$$9HgPlU=eDkNvUtoJm3jH6qbhmY`rn@@X z37LqS&hO<5_5~%EdR24dp|RsKP>Au5CsTTLS1lcQStoF&rQ`VP*LFM?biU0M7@TkL z)xR{K*P(YomsHzOoPZTJn$$$tY4&r%=|6m7j@MzsiPYNISvF;ymP(yXamfgW@UXNe zQYG?O)hz2dp%s=^<)-Dft+d&gmSpfDZw<&=@`__sIJX91wBnNk8t=Xi6NA}l2pHK$ zizHNXV4hJ(put^Lz*<_^38~5)GFKYD6q^_hU;l=iR>PgE^LZUEVV+G)QHIAeO^-LI zz@mFt;vw;hoiWQBB)QjlX2lN9>%zP%2m6;ZW}X{TVcw4`DZ=A_jUJc}n;tgaY^c4x=go z9M8e4e}7v^UVq!-JbRjzwaw=##l`r$P$Xw=l6& z-<_?Hj3g_}f;x*+SCVBa6Fe3^E>9Fw<|FN4M8 ztG-Y85FI+4a=dbynx~ADXJH<1PG_Jkjl1ihKXY<(y5rGQbA5FFe>m;7qmbm(xULUR zA@P{glzDPY-WGWb786;%7~Yi5n~Nxs^OpoU61)qqYwq)xW=Uhy_uJ9U?pvQoIC4{|cugj`|1#G&)%i*Va|Nxnpuy(&u({LK#h(L+02Q-lug!>jsxoqg-N z+0i)?14G%Qmz8?5SxG7-0~03>OlCH2Rr<}EhI)(F(b??j@V2yY>;Kz<`7_gBj1L>O z>92+R{}sj2G2cgj5cRxvm*xi0{zDZ2*u)xT?PNH@miRRpU?0Fa##rA6OND709-l>4 zOWSN&Y(llr!+j4HWs_GA@)LG_9&s3MY@lyf7jeh5K{R$Nn@}}aG-&IDhIsXujTntB zmrbY~B$@+X?Jd6N#ivw@CYw#D7%UpK1sjRgWU+KPLgRk|i!9cVc&x^l0}Gxc9K)DH zFA5fs3S*A|5sW?b1wkTGV+;}^%4X@ZU=gV>76}n$vBpxg%LXYe_|AKh-Meh?%}r%v zz3{1#<0#FJT!=Rv@jHD^hQl42{hb^cX6Ycr0vo&Ml_(qL$y+IE<${X?(3Rj6ojfZ$ zz8D7@zr$czOGUMKu-))<;ui=DOQ>M!y?P=qEJ=gRN-<`*2i;r+h(o5N)n zDEB5ccvl)&TjKiv@xXk{wAXlpp-;b*nxi`}_T!i*qM4{_ZHgxThZ_Bd$LFwPS17v- zc{Y3EFGGjm0kV1B_p4?gB^WI9($+Lvx|N-Lv9fEDj137cDr7{G7{Vy+B4LQ(IwamY z0I)4dYFvE3?R#ZgCy*9^B%#Uw>Xo-fUo%VVUTZ?uo`k@f6MjjXdpFGLBCZcWS5;DM z=7n-#xS}m-nZT_k1_#XhB#F@G<(?xUUg9PI$Ie=guJj~d_HAgM8Ix+avjNPEOCHLWaScM>^gSk%7wlD67(x@5#MS6=O9u@{Z6zTI99R+R8vuI9Y6B9fIS(CM}OSB+FD~>>+t3*9l9k4H}XfV~_w6 zj6sQugN3BZSR{m$&5i$O0rRVH{r^To9o+vfKJIkvJ8=BJHL6!zt(g@bKK=NvSshY?qRB#7)8EZr6?qPfk>C+yOx z77?_f)*ulbKvS33zo1q`@Q4h0b&!Zu7mUP?24m5f78I~aiNyEt74DTyN=pIt6u-^u zMn&IzUWa1oO}O0)9KLdECoXOMP6<*?%{``#%q|_ z4on(}?M!MqH-y5>h6o{OL(>|C-X$o_avewaDharjgv6b|s*OH+0HNq+(xPmMr}odL zI^>+ZO=karOaGyX$wxU>$ZN_fbjc^zGv9Xg1DE_mkk>#~HEX4uk~gdRVN3lX3CX`r zar6JDlm36Zej$|@_kHZqn8%{8i`t-V)D(m10I(A_D2w@US_z40Fq4Xoj&?7N|8wmF z-@ZS3JH#~X*o#|l%)eq!)2k<6tIz+g^VVNh{e7KCB49vnI zmUMRbdSRIIjYVWLh|1p{dwX}?imTN_h3l+gD2eNo#Tnnl|NYz+oa!0)qYjC%WNT#; z`+|k^#|J*F{r4+xsTPtohfVAa8dBnsk5(04t6oU3*VYp(q$f9ic0+5=Zq-7vr?ZJ` zgN1Yuo(BMJNxhJ4Y+`q?kPhHl+FI2@vSqS~UBN=ykG*ZOdLcn~>kJl>>N-f06aco6 z5<7y1q_!TC6q21yY!4Qa>bgi0NVW`a{67Qoq6$>Rnx<^=7tf$ z;V>aIjfp~53_E7>%$x z7eK2;A+PW;tN~js43J~;W>>Oai$Y9cV^)f+YUEFaUg*2zE3Smc#>V>FgE4F{iMl6T zB|M8AeudD8d}9$+!c|EHS`mRV;c`iWr1PF2$!*2w)n7dww=8ztMoDhu1CPvRR7(fi z4r~*^E=m?V@+X6dGpHY_$}TPEQW@;<%LWl=FclScXL%^x{D0X0f7`Ujc%9)AeF^OU z|1b9Kn7z?AMs;d0(7^70z`<{%lVt|KX7gnZ2uEF-neNs=&vyJRxPAQ`D{N|KlnoRLH`hF}?~ZAOxW5uA|3 z=#`Z{d`|#8q_!kd(zCTAN>tF4)Rsm{DZ$w`j4o(OjTNtcvGY6iB->I-Ie7dJt^($F zO#6&KH>}d9>dwR+jJ+miMKrBFrFjxQb@p#;7Bkt;pDG!)VBl_z4vlbqL8u=iN5fmX z6>grFo11IPv_mt}+}D}*XhGfB{@1_%YVY|c)_wWAXOC^ke|h_78@mq7$p1V#;ikU* zm&`1$T1r}+PNvLWnBa`Bw6Qf7*g&0rGb$h>)|=g{`R zkcE$f`teUTnHeK@NpGmC2J4dH!^&SUgVm(ba=Y>tzkGKjRkmR{e|aPwNjMjqW<~|e zNNoiGamCG!$Nxs~I50nFy2ZHEpr_u{-5OUKtBHOjs#Ti+p9swxdohzblRLV~OTMTP zoJ7lv(dj(1qe-4f^bejdY70VOzjab6fPZYhP>@15(tWKx&feCp_U2yNKC>f*?(8bA zZF00b>m5BQG|I(cLW%AyMt9h9CWQw~#lf))Hv^W$7ZKnX_q84llp|mzEXiE;VrCZM ziH3qod1{BUg(9Q0r&&3pBp}O~nZ)y{06<0zlQT?rKp>7@#3b`PHGX(%gpNMs4O0Zt z&Msn-d>1f;@v_&Gyb9vw&@fMwqPe+AuSobBM67ZiGlR#8qU8YLa;*sGSty)^)D?X| zfSiLPf_6AkpDrIudAodHDB`E?tIdG)*fM4sKU(mUQbLo9WR&uZQslpLn5jHWKmOY# zgsFD-9Vq+>y*HJa;(OpKyqLiCdLC0$xAP9f4%WdFeTO@^qKIOKbOYJytAVUX{eOdI zJ}}>BDl>el-%8Dk`)BM;F{`4!*WL*qj@lg6ncRsbXrMems=|!b=>$zalJcoM zlOFJ*>!k|e9`KYVe4mbGTAFzb%(@2(YjM_l4Y-CvIt&3MIP^_OH4R|*=HSX zHk0Z5$|Ss$M-p+qGRamk8KffNC!mUIyF)v9lvQ)xaTMNR7c=Push>DHNmwpd`K4)~ zl+w$UNfbU2)q2KGsQL+?VnZDoVKYFw?(PF;uQr}6GS`7}s))V%?B}>60B^YLGR8`% z`mt9L)rh%z2?lO+C2;GR**si-!vIk*AaEn0R+vJ$O&1LUmN99BY#u{o!g(%HGCzQ@`t5D&&0pCDg$>BsP6HzKWCzMVjQ+oRPd|rnW zr3^%=;G9tU8ktbi>3m*?6Qy)SDgT^M`ihU*yoG96BYz>pg zmr&6_b)b=|xP+?qYE_`MX?MnthW~ZUJOb5E#xE}Wt9%d@C{t0gkGM84a|u^J1ptY$ z4vn!?pnTmm0M2IT@NE5+tRgl$z$9dM93XX-3z=NP)sM}JxkgNOz&YMLUT*)tX~6ux z>1pFu!&-egHAAO~JqGvx+Y+@#TdJAnd-ywKo?$03HOf*UxLv*X`8Wr@+FN|ji%+Ry zCy9(SD~&=WUTM<{R}0u|*$`26kcgCx^}}Nn>jhB`JDI8S-3k_#L*SAv!8{RB{c>N; z$*U&rQ;!*m?Z}SK&!|)e$@dOG2l0ub&1!{Y%guzQRH2O00H(D5@Sfa;c=b|(rc|yB z(g3DZkGEt|EuiqJCNpJ0LwYpT zTpvAFwWgE>A(aLTNp+c$#FBE@Bxa$qY_i5z!mTDo>1j5caJ%a{_3^Mi3c_tQ+?gxs>~Ll|vKs$fL@@_p`P*Y}@2*>MwR*5XUm{d8nPtjijbJZ3 zZZ?%W)~G4zb6bs|%$k!zxAi%D9DUAadJ)~?XzJ_g>4jU_H7En;@LgBT%-;x)7FX@C z3xaED>vYm3i|9T_W4p7Lox&_t7MtL{rjlZ;GAldXf^Ye}mZya~x5L|Q1x`-GO@gyC z>^X3=V7omNZ%1mire|cM-Gku+cD##lc6vr8{Mnjevs%#uR%=d{9Y0_VY6DYrMT1`0 z(UfOP%dpZ(aQ~o;^3o!^HJP2s)F%U-f>7nuQOhe*>q{zYd7o59DSXQLkWsg@r*B

Zs-N9RNO=PU={mj3+|ova^^*Wn+$|!ZEFg*Z!Hw z`O$~e!wcG!Ph6>tBUt--@x+F*j7cx4hZk}}AHcGqeXR=8zFvH9{tLzL?N<-4fpAS= zRw}bEaStjsHdftT1jTH2n$>z zc!V1H0y5v8JER$64~8I#2j{znlcXa6hC>=5_GAIVg|1=5lLvutNTb9aEkIe~iYH4# z2M&r173Ag$GXi#V$NVZ4eLP1!a(2*F@sM0ni(w@UR05;1#;g zBMuw@JmSDZ6e9Lt0mS*P3B-Q~u7HUBcZgHOek>?{6uQO}KMsK65Qm8USpc!jHIDf6 zARrELnAoocFmqjFiC+(#-H3V=Ln*bE_QnKBO^z#pxN{)zhR!8Lzz%z{fL(!W4DsRs z*b(m?LTR;tA$DF5LvH;4JTO0I>NFaTF2nwA)Ys{IMgC-OAvMI%*dc2VS-O=@NXUl_$4hdM@SezC}v#$)tymawo~ z;vM5q$0^M)=DJddcMlxs&-!WLq+&w!k6AepyR{(3W2g4Scy^9!7IEr;@C~7$D4-|J zGUmHx5?>C49`Rvyrx`wNP?Bdr8%!oHJa7d?;=)7I2C)Z2LnP(*TvrnD;6UgNO?cRW z1@H=8Gl&BR0*^TG(1eKnR{(LoYdZ1YffGZC{|;@6()7B}HI4XjAQXo-MC{Md9!W4P zb4?}wJQ#>W8>S?+M&o}f`fXr-!gQH&j{ZyPQQg|;x8t&6PKU|JKk_vr1R?|?1Y`u* zge&#rT03ysH(OK^RCBslQ@nug>}>C1=Uu6AYMfxYsyg@Dt98IdOWcHL_q5+dWQK2B zXH)ySW;mU?trO2wZCS(b9+!Qk!Y%hY)NWhercL9y=yB@pje4IXPKu zo-2zCBnB?0h)bg?Zx@Q|DDo1$pqe+&l}X}SKmgSo*s#UNN8EU* JVwBYN{vX%xIbi?* From ccd50de547bf040eb84db2ed5dda663d9bcbd79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Sun, 26 Mar 2023 17:57:56 +0200 Subject: [PATCH 101/204] add seed to random generator in compute --- validphys2/src/validphys/photon/compute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 82c1b9ac68..de59ac1922 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -262,5 +262,6 @@ def generate_errors(self): if self.error_matrix is None : return np.zeros_like(self.xgrid) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) - errors = u @ (s * np.random.normal(size=7)) + rng = np.random.default_rng(seed=self.fiatlux_runcard["luxseed"]) + errors = u @ (s * rng.normal(size=7)) return errors From 952156b9b658c76ac3c95c19b383d0ad752220e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Sun, 26 Mar 2023 19:58:33 +0200 Subject: [PATCH 102/204] add abstract class StructureFunction --- validphys2/src/validphys/photon/compute.py | 50 +++++++++++-------- .../validphys/photon/structure_functions.py | 21 +++++--- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index de59ac1922..3ae1c74fee 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -20,6 +20,7 @@ import time class Photon: + """Photon class computing the photon array with the LuxQED approach.""" def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard @@ -30,16 +31,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # parameters for the alphaem running self.alpha_em_ref = self.theory["alphaqed"] self.qref = self.theory["Qref"] - # TODO : maybe they shoud be kDIS instead of k, but usually they are the same - self.thresh_c = self.theory["kcThr"] * self.theory["mc"] - self.thresh_b = self.theory["kbThr"] * self.theory["mb"] - self.thresh_t = self.theory["ktThr"] * self.theory["mt"] - if self.theory["MaxNfAs"] <= 5 : - self.thresh_t = np.inf - if self.theory["MaxNfAs"] <= 4 : - self.thresh_b = np.inf - if self.theory["MaxNfAs"] <= 3 : - self.thresh_c = np.inf + self.set_betas() self.set_thresholds_alpha_em() @@ -49,19 +41,24 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # TODO : maybe find a different name for fiatlux_dis_F2 path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" path_to_FL = theoryid.path / "fastkernel/fiatlux_dis_FL.pineappl.lz4" - f2 = [sf.StructureFunction(path_to_F2, self.qcd_pdfs.members[id]) for id in self.replicas_id] - fl = [sf.StructureFunction(path_to_FL, self.qcd_pdfs.members[id]) for id in self.replicas_id] - f2lo = [sf.F2LO(self.qcd_pdfs.members[id], self.theory) for id in self.replicas_id] - self.path_to_eko_photon = theoryid.path / "eko_photon.tar" # set fiatlux - ff = open('fiatlux_runcard.yml', 'w+') - yaml.dump(self.fiatlux_runcard, ff) - self.lux = [fiatlux.FiatLux('fiatlux_runcard.yml') for i in range(len(replicas_id))] - remove('fiatlux_runcard.yml') + self.lux = [] + f2 = [] + fl = [] + f2lo = [] + for id in replicas_id: + f2.append(sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id])) + fl.append(sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id])) + f2lo.append(sf.F2LO(self.qcd_pdfs.members[id], self.theory)) + ff = open(f'fiatlux_runcard_{id}.yml', 'w+') + yaml.dump(self.fiatlux_runcard, ff) + self.lux.append(fiatlux.FiatLux(f'fiatlux_runcard_{id}.yml')) + remove(f'fiatlux_runcard_{id}.yml') # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick + # we print different runcards for every replica so they do not interfere with each other for i in range(len(replicas_id)): self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) self.lux[i].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) @@ -129,6 +126,17 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" + # TODO : maybe they shoud be kDIS instead of k, but usually they are the same + self.thresh_c = self.theory["kcThr"] * self.theory["mc"] + self.thresh_b = self.theory["kbThr"] * self.theory["mb"] + self.thresh_t = self.theory["ktThr"] * self.theory["mt"] + if self.theory["MaxNfAs"] <= 5 : + self.thresh_t = np.inf + if self.theory["MaxNfAs"] <= 4 : + self.thresh_b = np.inf + if self.theory["MaxNfAs"] <= 3 : + self.thresh_c = np.inf + thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] if self.qref < self.thresh_c : nfref = 3 @@ -219,6 +227,7 @@ def compute_photon_array(self, id): return self.xgrid * photon_Q0 def produce_interpolators(self): + """Produce the interpolation functions to be called in compute.""" self.photons_array = [self.compute_photon_array(i) for i in range(len(self.replicas_id))] self.interpolator = [interp1d(self.xgrid, photon_array, fill_value=0., kind='cubic') for photon_array in self.photons_array] @@ -250,6 +259,8 @@ def generate_error_matrix(self): if not self.fiatlux_runcard["additional_errors"] : return None extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") + # random generator must be set previously in case of parallel replicas + self.rng = np.random.default_rng(seed=self.fiatlux_runcard["luxseed"]) return np.array( [ [(extra_set.xfxQ(x, self.q_in, i, 22) - extra_set.xfxQ(x, self.q_in, 0, 22)) for i in range(101, 107+1)] @@ -262,6 +273,5 @@ def generate_errors(self): if self.error_matrix is None : return np.zeros_like(self.xgrid) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) - rng = np.random.default_rng(seed=self.fiatlux_runcard["luxseed"]) - errors = u @ (s * rng.normal(size=7)) + errors = u @ (s * self.rng.normal(size=7)) return errors diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 5a1d6512a8..f8e767f78d 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -3,8 +3,13 @@ from scipy.interpolate import RectBivariateSpline class StructureFunction : + """Abstract class for the DIS structure functions""" + def fxq(self, x, Q) : + pass + +class InterpStructureFunction(StructureFunction): """ - Compute a given DIS structure function convoluting an FKtable + Compute an interpolated structure function convoluting an FKtable with a PDF. """ def __init__(self, path_to_fktable, pdfs): @@ -15,7 +20,7 @@ def __init__(self, path_to_fktable, pdfs): def produce_interpolator(self): - """Produce the interpolation function to be called in fxq""" + """Produce the interpolation function to be called in fxq.""" x = np.unique(self.fktable.bin_left(1)) q2 = np.unique(self.fktable.bin_left(0)) self.xmin = min(x) @@ -31,7 +36,7 @@ def produce_interpolator(self): def fxq(self, x, q): r""" - Compute the DIS structure function. + Compute the DIS structure function interpolating the grid. Parameters ---------- @@ -47,12 +52,12 @@ def fxq(self, x, q): """ # here we are requiring that the grid that we pass to fiatlux # has Qmin = 1 (fiatlux doesn't go below Q=1) - if x < self.xmin or q > self.qmax : - return 0. + # if x < self.xmin or q > self.qmax : + # return 0. return self.interpolator(x, q**2)[0, 0] -class F2LO : - """Compute the LO DIS structure function F2 given a PDF""" +class F2LO(StructureFunction) : + """Compute analytically the leading order structure function for F2.""" def __init__(self, pdfs, theory): self.pdfs = pdfs # TODO : maybe they shoud be kDIS instead of k, but usually they are the same @@ -71,7 +76,7 @@ def __init__(self, pdfs, theory): def fxq(self, x, q): r""" - Compute the LO DIS structure function F2. + Compute the analytical form of F2LO. Parameters ---------- From debb85264b53617b147a7ae982f0f7a35d7b61f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Sun, 26 Mar 2023 20:52:51 +0200 Subject: [PATCH 103/204] Fix tests --- .../src/validphys/tests/photon/test_structurefunctions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index d28635b193..7405624f81 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -99,12 +99,12 @@ def set(self): def test_F2(monkeypatch): monkeypatch.setattr(pineappl.fk_table.FkTable, "read", ZeroFKTable) - structurefunc = sf.StructureFunction("", OnePdf()) + structurefunc = sf.InterpStructureFunction("", OnePdf()) for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) monkeypatch.setattr(pineappl.fk_table.FkTable, "read", OneFKTable) - structurefunc = sf.StructureFunction("", ZeroPdf()) + structurefunc = sf.InterpStructureFunction("", ZeroPdf()) for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) From bcea68826993ab86842b03f732f61101e88cf423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Sun, 26 Mar 2023 22:43:18 +0200 Subject: [PATCH 104/204] Fix test_compute.py --- validphys2/src/validphys/tests/photon/test_compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 0e5e839ef5..b79cfeb56e 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -77,7 +77,7 @@ def fxq(self): def test_init(monkeypatch): - monkeypatch.setattr(structure_functions, "StructureFunction", fakeStructureFunction) + monkeypatch.setattr(structure_functions, "InterpStructureFunction", fakeStructureFunction) monkeypatch.setattr(structure_functions, "F2LO", fakeF2LO) # monkeypatch.setattr(LHAPDFSet, "mkPDF", lambda *args: fiatlux_runcard["pdf_name"]) monkeypatch.setattr(fiatlux, "FiatLux", fakeFiatlux) From d706cf41794df7b98b399139ae4d3c9edf7a589a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 27 Mar 2023 12:44:09 +0200 Subject: [PATCH 105/204] Use different seeds for the different replicas --- validphys2/src/validphys/photon/compute.py | 57 +++++++++++++--------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 3ae1c74fee..234f7e44c5 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -8,8 +8,8 @@ from scipy.integrate import trapezoid from eko.io import EKO -from eko.io.manipulate import xgrid_reshape -from eko.interpolation import XGrid +# from eko.io.manipulate import xgrid_reshape +# from eko.interpolation import XGrid from n3fit.io.writer import XGRID @@ -44,25 +44,25 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.path_to_eko_photon = theoryid.path / "eko_photon.tar" # set fiatlux - self.lux = [] - f2 = [] - fl = [] - f2lo = [] + self.lux = {} + f2 = {} + fl = {} + f2lo = {} for id in replicas_id: - f2.append(sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id])) - fl.append(sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id])) - f2lo.append(sf.F2LO(self.qcd_pdfs.members[id], self.theory)) + f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) + fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) + f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) ff = open(f'fiatlux_runcard_{id}.yml', 'w+') yaml.dump(self.fiatlux_runcard, ff) - self.lux.append(fiatlux.FiatLux(f'fiatlux_runcard_{id}.yml')) + self.lux[id] = fiatlux.FiatLux(f'fiatlux_runcard_{id}.yml') remove(f'fiatlux_runcard_{id}.yml') # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick # we print different runcards for every replica so they do not interfere with each other - for i in range(len(replicas_id)): - self.lux[i].PlugAlphaQED(self.alpha_em, self.qref) - self.lux[i].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) - self.lux[i].PlugStructureFunctions(f2[i].fxq, fl[i].fxq, f2lo[i].fxq) + for id in replicas_id : + self.lux[id].PlugAlphaQED(self.alpha_em, self.qref) + self.lux[id].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) + self.lux[id].PlugStructureFunctions(f2[id].fxq, fl[id].fxq, f2lo[id].fxq) self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() @@ -189,16 +189,17 @@ def compute_photon_array(self, id): # Compute photon PDF start_time = time.perf_counter() photon_100GeV = np.array([self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid]) - photon_100GeV += self.generate_errors() + photon_100GeV += self.generate_errors(id) photon_100GeV /= self.xgrid print("Time to compute photon:", time.perf_counter() - start_time) # TODO : the different x points could be even computed in parallel # Load eko and reshape it - with EKO.edit(self.path_to_eko_photon) as eko: + with EKO.read(self.path_to_eko_photon) as eko: # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape - xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) + # TODO : move the reshape inside vp-setupfit + # xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) # construct PDFs pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) @@ -228,8 +229,10 @@ def compute_photon_array(self, id): def produce_interpolators(self): """Produce the interpolation functions to be called in compute.""" - self.photons_array = [self.compute_photon_array(i) for i in range(len(self.replicas_id))] - self.interpolator = [interp1d(self.xgrid, photon_array, fill_value=0., kind='cubic') for photon_array in self.photons_array] + self.photons_array = [self.compute_photon_array(id) for id in self.replicas_id] + self.interpolator = [ + interp1d(self.xgrid, photon_array, fill_value=0., kind='cubic') for photon_array in self.photons_array + ] def compute(self, xgrid): """ @@ -260,7 +263,6 @@ def generate_error_matrix(self): return None extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") # random generator must be set previously in case of parallel replicas - self.rng = np.random.default_rng(seed=self.fiatlux_runcard["luxseed"]) return np.array( [ [(extra_set.xfxQ(x, self.q_in, i, 22) - extra_set.xfxQ(x, self.q_in, 0, 22)) for i in range(101, 107+1)] @@ -268,10 +270,21 @@ def generate_error_matrix(self): ] ) # first index must be x, while second one must be replica index - def generate_errors(self): + def generate_errors(self, replica_id): """generate LUX additional errors.""" if self.error_matrix is None : return np.zeros_like(self.xgrid) + seed = replica_luxseed(replica_id, self.fiatlux_runcard["luxseed"]) + rng = np.random.default_rng(seed=seed) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) - errors = u @ (s * self.rng.normal(size=7)) + errors = u @ (s * rng.normal(size=7)) return errors + +# TODO : move it in n3fit_data.py with the others replica_seed generators? +# or use directly replica_nnseed? +def replica_luxseed(replica, luxseed): + """Generates the ``luxseed`` for a ``replica``.""" + np.random.seed(seed=luxseed) + for _ in range(replica): + res = np.random.randint(0, pow(2, 31)) + return res From d9e3df45cc293c9b9bcde5b6065af204e7e23b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 27 Mar 2023 23:12:07 +0200 Subject: [PATCH 106/204] Add qed to evolven3fit_new --- n3fit/src/evolven3fit_new/evolve.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/n3fit/src/evolven3fit_new/evolve.py b/n3fit/src/evolven3fit_new/evolve.py index 4a9ac53b00..93d8e2e8b6 100644 --- a/n3fit/src/evolven3fit_new/evolve.py +++ b/n3fit/src/evolven3fit_new/evolve.py @@ -6,8 +6,7 @@ import eko from eko import basis_rotation from eko import runner -from ekobox import genpdf, info_file -from ekomark import apply +from ekobox import genpdf, info_file, apply from reportengine.compat import yaml from validphys.loader import Loader @@ -85,6 +84,7 @@ def evolve_fit( theory, op = eko_utils.construct_eko_cards( theoryID, q_fin, q_points, x_grid, op_card_dict, theory_card_dict ) + qed = theory.order[1] > 0 if eko_path is not None: eko_path = pathlib.Path(eko_path) _logger.info(f"Loading eko from : {eko_path}") @@ -108,7 +108,7 @@ def evolve_fit( info["AlphaS_Qs"] = eko_op.mu2grid.tolist() dump_info_file(usr_path, info) for replica in initial_PDFs_dict.keys(): - evolved_block = evolve_exportgrid(initial_PDFs_dict[replica], eko_op, x_grid) + evolved_block = evolve_exportgrid(initial_PDFs_dict[replica], eko_op, x_grid, qed) dump_evolved_replica( evolved_block, usr_path, int(replica.removeprefix("replica_")) ) @@ -142,7 +142,7 @@ def load_fit(usr_path): return pdf_dict -def evolve_exportgrid(exportgrid, eko, x_grid): +def evolve_exportgrid(exportgrid, eko, x_grid, qed): """ Evolves the provided exportgrid for the desired replica with the eko and returns the evolved block @@ -163,7 +163,7 @@ def evolve_exportgrid(exportgrid, eko, x_grid): pdf_grid = np.array(exportgrid["pdfgrid"]).transpose() pdf_to_evolve = utils.LhapdfLike(pdf_grid, exportgrid["q20"], x_grid) # evolve pdf - evolved_pdf = apply.apply_pdf(eko, pdf_to_evolve) + evolved_pdf = apply.apply_pdf(eko, pdf_to_evolve, qed) # generate block to dump targetgrid = eko.rotations.targetgrid.tolist() From 144696562afa555537629ccdd6d79e8a162737e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 28 Mar 2023 11:58:35 +0200 Subject: [PATCH 107/204] Fix opcard for exa evolution in evolven3fit_new --- n3fit/src/evolven3fit_new/eko_utils.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/n3fit/src/evolven3fit_new/eko_utils.py b/n3fit/src/evolven3fit_new/eko_utils.py index 17467dc1d9..0dc4872979 100644 --- a/n3fit/src/evolven3fit_new/eko_utils.py +++ b/n3fit/src/evolven3fit_new/eko_utils.py @@ -10,7 +10,7 @@ _logger = logging.getLogger(__name__) -EVOLVEN3FIT_CONFIGS_DEFAULTS = { +EVOLVEN3FIT_CONFIGS_DEFAULTS_TRN = { "ev_op_iterations": 1, "ev_op_max_order": (1, 0), "evolution_method": "truncated", @@ -18,6 +18,14 @@ "n_integration_cores": 1, } +EVOLVEN3FIT_CONFIGS_DEFAULTS_EXA = { + "ev_op_iterations": 10, + "ev_op_max_order": (1, 0), + "evolution_method": "iterate-exact", + "inversion_method": "exact", + "n_integration_cores": 1, +} + NFREF_DEFAULT = 5 NF0_DEFAULT = 4 @@ -68,7 +76,10 @@ def construct_eko_cards( ) op_card["rotations"]["xgrid"] = x_grid # Specific defaults for evolven3fit evolution - op_card["configs"].update(EVOLVEN3FIT_CONFIGS_DEFAULTS) + if theory["ModEv"] == "TRN": + op_card["configs"].update(EVOLVEN3FIT_CONFIGS_DEFAULTS_TRN) + if theory["ModEv"] == "EXA": + op_card["configs"].update(EVOLVEN3FIT_CONFIGS_DEFAULTS_EXA) # User can still change the configs via op_card_dict # Note that every entry that is not a dictionary should not be From ab9818a1254412f216b78410cf9019a9347d7270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 29 Mar 2023 10:36:18 +0200 Subject: [PATCH 108/204] Modify _register_photon --- n3fit/src/n3fit/vpinterface.py | 20 +++++++++---------- validphys2/src/validphys/photon/compute.py | 1 - .../validphys/photon/structure_functions.py | 1 - 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index c7ba392852..88f139f63e 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -80,17 +80,15 @@ def xfxQ(self, x, Q, n, fl): def _register_photon(self, xgrid): """If the PDF models contain photons, register the xgrid with them""" - try: - for m in self._lhapdf_set: - pl = m.get_layer("add_photon") - pl.register_photon(xgrid) - # Recompile the model if necessary - # (relies on the caching mechanism inside the add_photon layer) - if not pl.built: - m.compile() - except ValueError: - # There's no photon I guess - pass + for m in self._lhapdf_set: + pl = m.get_layer_re("add_photon") + # if pl is an empy list there's no photon + if not pl : + continue + pl.register_photon(xgrid) + # Recompile the model if necessary + if not pl.built: + m.compile() def __call__(self, xarr, flavours=None, replica=None): """Uses the internal model to produce pdf values for the grid diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 234f7e44c5..38254c073c 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -126,7 +126,6 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" - # TODO : maybe they shoud be kDIS instead of k, but usually they are the same self.thresh_c = self.theory["kcThr"] * self.theory["mc"] self.thresh_b = self.theory["kbThr"] * self.theory["mb"] self.thresh_t = self.theory["ktThr"] * self.theory["mt"] diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index f8e767f78d..3b851c2af3 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -60,7 +60,6 @@ class F2LO(StructureFunction) : """Compute analytically the leading order structure function for F2.""" def __init__(self, pdfs, theory): self.pdfs = pdfs - # TODO : maybe they shoud be kDIS instead of k, but usually they are the same self.thresh_c = theory["kcThr"] * theory["mc"] self.thresh_b = theory["kbThr"] * theory["mb"] self.thresh_t = theory["ktThr"] * theory["mt"] From 0baf9b97bff1dcba53ba7e63024aac723bff514f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 30 Mar 2023 10:03:40 +0200 Subject: [PATCH 109/204] Fix apply_pdf --- n3fit/src/evolven3fit_new/evolve.py | 2 +- .../validphys/tests/photon/test_structurefunctions.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/n3fit/src/evolven3fit_new/evolve.py b/n3fit/src/evolven3fit_new/evolve.py index 93d8e2e8b6..de8cba242e 100644 --- a/n3fit/src/evolven3fit_new/evolve.py +++ b/n3fit/src/evolven3fit_new/evolve.py @@ -163,7 +163,7 @@ def evolve_exportgrid(exportgrid, eko, x_grid, qed): pdf_grid = np.array(exportgrid["pdfgrid"]).transpose() pdf_to_evolve = utils.LhapdfLike(pdf_grid, exportgrid["q20"], x_grid) # evolve pdf - evolved_pdf = apply.apply_pdf(eko, pdf_to_evolve, qed) + evolved_pdf = apply.apply_pdf(eko, pdf_to_evolve, qed=qed) # generate block to dump targetgrid = eko.rotations.targetgrid.tolist() diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index 7405624f81..57b51ea170 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -1,6 +1,7 @@ import validphys.photon.structure_functions as sf import numpy as np import pineappl +from validphys.lhapdfset import LHAPDFSet class ZeroPdfs: def xfxQ(self, x, Q): @@ -98,11 +99,21 @@ def set(self): def test_F2(monkeypatch): + # grid put to 0 and pdf put to 1 monkeypatch.setattr(pineappl.fk_table.FkTable, "read", ZeroFKTable) structurefunc = sf.InterpStructureFunction("", OnePdf()) for x in np.geomspace(1e-4, 1., 10): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) + + # grid put to 0 and real pdf + pdf = LHAPDFSet("NNPDF40_nnlo_as_01180", "replicas") + structurefunc = sf.InterpStructureFunction("", pdf.central_member) + for x in np.geomspace(1e-4, 1., 10): + for Q in np.geomspace(10, 1000000, 10): + np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) + + # grid put to 1 and pdf put to 0 monkeypatch.setattr(pineappl.fk_table.FkTable, "read", OneFKTable) structurefunc = sf.InterpStructureFunction("", ZeroPdf()) for x in np.geomspace(1e-4, 1., 10): From ad8080a29e1c7997c12c9ae64d73f30d32f9c72b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 11:31:41 +0200 Subject: [PATCH 110/204] Add Basic_runcard_qed.yml --- n3fit/runcards/examples/Basic_runcard_qed.yml | 204 ++++++++++++++++++ validphys2/src/validphys/photon/compute.py | 7 +- 2 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 n3fit/runcards/examples/Basic_runcard_qed.yml diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml new file mode 100644 index 0000000000..ef70d3fd19 --- /dev/null +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -0,0 +1,204 @@ +# +# Configuration file for n3fit +# +############################################################ +description: Basic runcard qed + +############################################################ +# frac: training fraction +# ewk: apply ewk k-factors +# sys: systematics treatment (see systypes) +dataset_inputs: +# - {dataset: NMCPD_dw_ite, frac: 0.75} +- {dataset: NMC, frac: 0.75} +- {dataset: SLACP_dwsh, frac: 0.75} +# - {dataset: SLACD_dw_ite, frac: 0.75} +# - {dataset: BCDMSP_dwsh, frac: 0.75} +# - {dataset: BCDMSD_dw_ite, frac: 0.75} +# - {dataset: CHORUSNUPb_dw_ite, frac: 0.75} +# - {dataset: CHORUSNBPb_dw_ite, frac: 0.75} +# - {dataset: NTVNUDMNFe_dw_ite, frac: 0.75, cfac: [MAS]} +# - {dataset: NTVNBDMNFe_dw_ite, frac: 0.75, cfac: [MAS]} +# - {dataset: HERACOMBNCEM, frac: 0.75} +# - {dataset: HERACOMBNCEP460, frac: 0.75} +# - {dataset: HERACOMBNCEP575, frac: 0.75} +# - {dataset: HERACOMBNCEP820, frac: 0.75} +# - {dataset: HERACOMBNCEP920, frac: 0.75} +# - {dataset: HERACOMBCCEM, frac: 0.75} +# - {dataset: HERACOMBCCEP, frac: 0.75} +# - {dataset: HERACOMB_SIGMARED_C, frac: 0.75} +# - {dataset: HERACOMB_SIGMARED_B, frac: 0.75} +# - {dataset: DYE886R_dw_ite, frac: 0.75, cfac: [QCD]} +# - {dataset: DYE886P, frac: 0.75, cfac: [QCD]} +# - {dataset: DYE605_dw_ite, frac: 0.75, cfac: [QCD]} +# - {dataset: DYE906R_dw_ite, frac: 0.75, cfac: [ACC, QCD]} +# - {dataset: CDFZRAP_NEW, frac: 0.75, cfac: [QCD]} +# - {dataset: D0ZRAP_40, frac: 0.75, cfac: [QCD]} +# - {dataset: D0WMASY, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASWZRAP36PB, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASZHIGHMASS49FB, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASLOMASSDY11EXT, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASWZRAP11CC, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASWZRAP11CF, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASDY2D8TEV, frac: 0.75, cfac: [QCDEWK]} +# - {dataset: ATLAS_DY_2D_8TEV_LOWMASS, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_WZ_TOT_13TEV, frac: 0.75, cfac: [NRM, QCD]} +# - {dataset: ATLAS_WP_JET_8TEV_PT, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_WM_JET_8TEV_PT, frac: 0.75, cfac: [QCD]} +- {dataset: ATLASZPT8TEVMDIST, frac: 0.75, cfac: [QCD], sys: 10} +# - {dataset: ATLASZPT8TEVYDIST, frac: 0.75, cfac: [QCD], sys: 10} +# - {dataset: ATLASTTBARTOT7TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASTTBARTOT8TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_TTBARTOT_13TEV_FULLLUMI, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_TTB_DIFF_8TEV_LJ_TRAPNORM, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_TTB_DIFF_8TEV_LJ_TTRAPNORM, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_TOPDIFF_DILEPT_8TEV_TTRAPNORM, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_1JET_8TEV_R06_DEC, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_2JET_7TEV_R06, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLASPHT15_SF, frac: 0.75, cfac: [QCD, EWK]} +# - {dataset: ATLAS_SINGLETOP_TCH_R_7TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_SINGLETOP_TCH_R_13TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_SINGLETOP_TCH_DIFF_7TEV_T_RAP_NORM, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_SINGLETOP_TCH_DIFF_7TEV_TBAR_RAP_NORM, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_SINGLETOP_TCH_DIFF_8TEV_T_RAP_NORM, frac: 0.75, cfac: [QCD]} +# - {dataset: ATLAS_SINGLETOP_TCH_DIFF_8TEV_TBAR_RAP_NORM, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSWEASY840PB, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSWMASY47FB, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSDY2D11, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSWMU8TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSZDIFF12, frac: 0.75, cfac: [QCD, NRM], sys: 10} +# - {dataset: CMS_2JET_7TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMS_1JET_8TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSTTBARTOT7TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSTTBARTOT8TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSTTBARTOT13TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSTOPDIFF8TEVTTRAPNORM, frac: 0.75, cfac: [QCD]} +# - {dataset: CMSTTBARTOT5TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMS_TTBAR_2D_DIFF_MTT_TRAP_NORM, frac: 0.75, cfac: [QCD]} +# - {dataset: CMS_TTB_DIFF_13TEV_2016_2L_TRAP, frac: 0.75, cfac: [QCD]} +# - {dataset: CMS_TTB_DIFF_13TEV_2016_LJ_TRAP, frac: 0.75, cfac: [QCD]} +# - {dataset: CMS_SINGLETOP_TCH_TOT_7TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMS_SINGLETOP_TCH_R_8TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: CMS_SINGLETOP_TCH_R_13TEV, frac: 0.75, cfac: [QCD]} +# - {dataset: LHCBZ940PB, frac: 0.75, cfac: [QCD]} +# - {dataset: LHCBZEE2FB_40, frac: 0.75, cfac: [QCD]} +# - {dataset: LHCBWZMU7TEV, frac: 0.75, cfac: [NRM, QCD]} +# - {dataset: LHCBWZMU8TEV, frac: 0.75, cfac: [NRM, QCD]} +# - {dataset: LHCB_Z_13TEV_DIMUON, frac: 0.75, cfac: [QCD]} +# - {dataset: LHCB_Z_13TEV_DIELECTRON, frac: 0.75, cfac: [QCD]} + +############################################################ +datacuts: + t0pdfset: NNPDF40_nnlo_as_01180_1000 # PDF set to generate t0 covmat + q2min: 3.49 # Q2 minimum + w2min: 12.5 # W2 minimum + +############################################################ +theory: + theoryid: 522 # database id + +############################################################ +trvlseed: 1551864071 +nnseed: 676150632 +mcseed: 619859729 +save: false +genrep: true # true = generate MC replicas, false = use real data + +parameters: # This defines the parameter dictionary that is passed to the Model Trainer + nodes_per_layer: [25, 20, 8] + activation_per_layer: [tanh, tanh, linear] + initializer: glorot_normal + optimizer: + clipnorm: 6.073e-6 + learning_rate: 2.621e-3 + optimizer_name: Nadam + epochs: 17000 + positivity: + initial: 184.8 + multiplier: + integrability: + initial: 10 + multiplier: + stopping_patience: 0.1 + layer_type: dense + dropout: 0.0 + threshold_chi2: 3.5 + +fitting: + fitbasis: EVOL # EVOL (7), EVOLQED (8), etc. + basis: + - {fl: sng, trainable: false, smallx: [1.089, 1.117], largex: [1.462, 3.008]} + - {fl: g, trainable: false, smallx: [0.7542, 1.105], largex: [2.826, 5.407]} + - {fl: v, trainable: false, smallx: [0.4715, 0.7253], largex: [1.564, 3.48]} + - {fl: v3, trainable: false, smallx: [0.1372, 0.4205], largex: [1.755, 3.451]} + - {fl: v8, trainable: false, smallx: [0.5641, 0.7702], largex: [1.513, 3.433]} + - {fl: t3, trainable: false, smallx: [-0.4942, 0.9992], largex: [1.751, 3.383]} + - {fl: t8, trainable: false, smallx: [0.532, 0.8572], largex: [1.541, 3.349]} + - {fl: t15, trainable: false, smallx: [1.052, 1.14], largex: [1.487, 3.09]} + +############################################################ +positivity: + posdatasets: + - {dataset: POSF2U, maxlambda: 1e6} # Positivity Lagrange Multiplier + # - {dataset: POSF2DW, maxlambda: 1e6} + # - {dataset: POSF2S, maxlambda: 1e6} + - {dataset: POSFLL, maxlambda: 1e6} + # - {dataset: POSDYU, maxlambda: 1e10} + # - {dataset: POSDYD, maxlambda: 1e10} + # - {dataset: POSDYS, maxlambda: 1e10} + # - {dataset: POSF2C, maxlambda: 1e6} + # - {dataset: POSXUQ, maxlambda: 1e6} # Positivity of MSbar PDFs + # - {dataset: POSXUB, maxlambda: 1e6} + # - {dataset: POSXDQ, maxlambda: 1e6} + # - {dataset: POSXDB, maxlambda: 1e6} + # - {dataset: POSXSQ, maxlambda: 1e6} + # - {dataset: POSXSB, maxlambda: 1e6} + # - {dataset: POSXGL, maxlambda: 1e6} + +############################################################ +integrability: + integdatasets: + # - {dataset: INTEGXT8, maxlambda: 1e2} + - {dataset: INTEGXT3, maxlambda: 1e2} + +############################################################ +debug: True +maxcores: 8 + +fiatlux: + apfel: off + + qed_running: off # determines the running of alpha. + q2_max: 60000 # the maximum allowed Q2. + eps_base: 1e-5 # precision on final integration of double integral. + eps_rel: 1e-1 # extra precision on any single integration. + mproton: 0.938272046 # proton mass from 2015 update of PDG review + mum_proton: 2.792847356 # proton magnetic moment, from + # http://pdglive.lbl.gov/DataBlock.action?node=S016MM which itself + # gets it from arXiv:1203.5425 (CODATA) + + # the elastic param type, options: + # dipole + # A1_world_spline + # A1_world_pol_spline + elastic_param: A1_world_pol_spline + elastic_electric_rescale: 1 + elastic_magnetic_rescale: 1 + + # the inelastic param type, options: + inelastic_param: LHAPDF_Hermes_ALLM_CLAS # Hermes_ALLM_CLAS, LHAPDF_Hermes_ALLM_CLAS + rescale_r_twist4: 0 + rescale_r: 1 + allm_limits: 0 + rescale_non_resonance: 1 + rescale_resonance: 1 + use_mu2_as_upper_limit: off + q2min_inel_override: 0.0 + q2max_inel_override: 1E300 + lhapdf_transition_q2: 9 + + # general + verbose: off + pdf_name: NNPDF40_nnlo_as_01180_1000 + additional_errors: true # should be set to true only for the last iteration + luxseed: 1234567890 \ No newline at end of file diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 38254c073c..f737b199c9 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -2,6 +2,7 @@ import numpy as np from validphys.lhapdfset import LHAPDFSet +from validphys.n3fit_data import replica_nnseed from . import structure_functions as sf from scipy.interpolate import interp1d @@ -282,8 +283,4 @@ def generate_errors(self, replica_id): # TODO : move it in n3fit_data.py with the others replica_seed generators? # or use directly replica_nnseed? def replica_luxseed(replica, luxseed): - """Generates the ``luxseed`` for a ``replica``.""" - np.random.seed(seed=luxseed) - for _ in range(replica): - res = np.random.randint(0, pow(2, 31)) - return res + return replica_nnseed(replica, luxseed) From 4c709ef4bdc38b719bfa0933c62407719f61e08f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 11:42:16 +0200 Subject: [PATCH 111/204] Call black on modified files --- n3fit/src/n3fit/layers/msr_normalization.py | 8 +- n3fit/src/n3fit/layers/rotations.py | 16 +- n3fit/src/n3fit/model_gen.py | 10 +- n3fit/src/n3fit/model_trainer.py | 93 ++++++++--- n3fit/src/n3fit/msr.py | 46 +++--- n3fit/src/n3fit/scripts/n3fit_exec.py | 112 +++++++++----- n3fit/src/n3fit/vpinterface.py | 10 +- validphys2/src/validphys/photon/compute.py | 144 +++++++++++------- .../validphys/photon/structure_functions.py | 45 +++--- 9 files changed, 303 insertions(+), 181 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 97111e83c6..579977692b 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -55,11 +55,13 @@ def call(self, pdf_integrated, ph_replica): if self._photons is not None: photon_integral = self._photons[ph_replica] - else : - photon_integral = 0. + else: + photon_integral = 0.0 if self._msr_enabled: - n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - photon_integral) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) + n_ag = [ + (1.0 - y[GLUON_IDX[0][0] - 1] - photon_integral) / y[GLUON_IDX[0][0]] + ] * len(GLUON_IDX) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index f98720e220..7e27ee9870 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -28,7 +28,7 @@ def __init__(self, rotation_matrix, axes=1, **kwargs): super().__init__(**kwargs) def is_identity(self): - """ Returns true if the rotation is an identity """ + """Returns true if the rotation is an identity""" # check whether it is a mxm matrix if self.rotation_matrix.shape[0] == self.rotation_matrix.shape[1]: # check whether it is the identity @@ -41,12 +41,15 @@ def call(self, x_raw): class FlavourToEvolution(Rotation): """ - Rotates from the flavour basis to - the evolution basis. + Rotates from the flavour basis to + the evolution basis. """ def __init__( - self, flav_info, fitbasis, **kwargs, + self, + flav_info, + fitbasis, + **kwargs, ): rotation_matrix = pdfbases.fitbasis_to_NN31IC(flav_info, fitbasis) super().__init__(rotation_matrix, axes=1, **kwargs) @@ -91,6 +94,7 @@ def call(self, pdf_raw): # Concatenating destroys the batch index so we have to regenerate it return op.batchit(ret) + class AddPhoton(MetaLayer): """ Changes the value of the photon component of the PDF to non-zero. @@ -111,11 +115,11 @@ def register_photon(self, xgrid): if self._photons_generator is not None: self._pdf_ph = self._photons_generator.compute(xgrid) self.built = False - + def call(self, pdfs, i): if self._pdf_ph is None: return pdfs - return op.concatenate([self._pdf_ph[i], pdfs[:,:,1:]], axis=-1) + return op.concatenate([self._pdf_ph[i], pdfs[:, :, 1:]], axis=-1) class ObsRotation(MetaLayer): diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index ecdea276a6..04240784b5 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -351,7 +351,9 @@ def generate_dense_per_flavour_network( initializers = [] for _ in range(basis_size): # select the initializer and move the seed - initializers.append(MetaLayer.select_initializer(initializer_name, seed=current_seed)) + initializers.append( + MetaLayer.select_initializer(initializer_name, seed=current_seed) + ) current_seed += 1 # set the arguments that will define the layer @@ -573,7 +575,9 @@ def pdfNN_layer_generator( # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photons=photons) + sumrule_layer, integrator_input = msr_impose( + mode=impose_sumrule, scaler=scaler, photons=photons + ) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x @@ -646,7 +650,7 @@ def layer_fitbasis(x): # Rotation layer, changes from the 8-basis to the 14-basis def layer_pdf(x): return layer_evln(layer_fitbasis(x)) - + # Final PDF (apply normalization) normalized_pdf = sumrule_layer(layer_pdf, i) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 848a9dc7b0..fbc7425bf0 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -37,6 +37,7 @@ # See ModelTrainer::_xgrid_generation for the definition of each field and how they are generated InputInfo = namedtuple("InputInfo", ["input", "split", "idx"]) + def _pdf_injection(pdf_layers, observables, masks): """ Takes as input a list of PDF layers each corresponding to one observable (also given as a list) @@ -186,7 +187,9 @@ def __init__( hyper_loss = kfold_parameters.get("target", None) if hyper_loss is None: hyper_loss = "average" - log.warning("No minimization target selected, defaulting to '%s'", hyper_loss) + log.warning( + "No minimization target selected, defaulting to '%s'", hyper_loss + ) log.info("Using '%s' as the target for hyperoptimization", hyper_loss) self._hyper_loss = getattr(n3fit.hyper_optimization.rewards, hyper_loss) @@ -462,7 +465,9 @@ def _model_generation(self, xinput, pdf_models, partition, partition_idx): validation = MetaModel(full_model_input_dict, output_vl) # Or the positivity in the total chi2 - output_ex = _pdf_injection(exp_pdfs, self.experimental["output"], experimental_mask) + output_ex = _pdf_injection( + exp_pdfs, self.experimental["output"], experimental_mask + ) experimental = MetaModel(full_model_input_dict, output_ex) if self.print_summary: @@ -558,7 +563,9 @@ def _generate_observables( all_pos_initial, all_pos_multiplier, max_lambda, positivity_steps ) - pos_layer = model_gen.observable_generator(pos_dict, positivity_initial=pos_initial) + pos_layer = model_gen.observable_generator( + pos_dict, positivity_initial=pos_initial + ) # The input list is still common self.input_list.append(pos_layer["inputs"]) @@ -573,13 +580,18 @@ def _generate_observables( if self.integ_info is not None: for integ_dict in self.integ_info: if not self.mode_hyperopt: - log.info("Generating integrability penalty for %s", integ_dict["name"]) + log.info( + "Generating integrability penalty for %s", integ_dict["name"] + ) integrability_steps = int(epochs / PUSH_INTEGRABILITY_EACH) max_lambda = integ_dict["lambda"] integ_initial, integ_multiplier = _LM_initial_and_multiplier( - all_integ_initial, all_integ_multiplier, max_lambda, integrability_steps + all_integ_initial, + all_integ_multiplier, + max_lambda, + integrability_steps, ) integ_layer = model_gen.observable_generator( @@ -605,10 +617,15 @@ def _generate_observables( force_set_smallest = input_arr.min() > 1e-9 if force_set_smallest: new_xgrid = np.linspace( - start=1 / input_arr_size, stop=1.0, endpoint=False, num=input_arr_size + start=1 / input_arr_size, + stop=1.0, + endpoint=False, + num=input_arr_size, ) else: - new_xgrid = np.linspace(start=0, stop=1.0, endpoint=False, num=input_arr_size) + new_xgrid = np.linspace( + start=0, stop=1.0, endpoint=False, num=input_arr_size + ) # When mapping the FK xgrids onto our new grid, we need to consider degeneracies among # the x-values in the FK grids @@ -628,7 +645,9 @@ def _generate_observables( # Select the indices of the points that will be used by the interpolator onein = map_from_complete.size / (int(interpolation_points) - 1) - selected_points = [round(i * onein - 1) for i in range(1, int(interpolation_points))] + selected_points = [ + round(i * onein - 1) for i in range(1, int(interpolation_points)) + ] if selected_points[0] != 0: selected_points = [0] + selected_points map_from = map_from_complete[selected_points] @@ -639,7 +658,8 @@ def _generate_observables( scaler = PchipInterpolator(map_from, map_to) except ValueError: raise ValueError( - "interpolation_points is larger than the number of unique " "input x-values" + "interpolation_points is larger than the number of unique " + "input x-values" ) self._scaler = lambda x: np.concatenate([scaler(np.log(x)), x], axis=-1) @@ -716,7 +736,14 @@ def _prepare_reporting(self, partition): to select the bits necessary for reporting the chi2. Receives the chi2 partition data to see whether any dataset is to be left out """ - reported_keys = ["name", "count_chi2", "positivity", "integrability", "ndata", "ndata_vl"] + reported_keys = [ + "name", + "count_chi2", + "positivity", + "integrability", + "ndata", + "ndata_vl", + ] reporting_list = [] for exp_dict in self.all_info: reporting_dict = {k: exp_dict.get(k) for k in reported_keys} @@ -817,7 +844,10 @@ def evaluate(self, stopping_object): # training and the validation which are actually `chi2` and not part of the penalty train_chi2 = stopping_object.evaluate_training(self.training["model"]) val_chi2 = stopping_object.vl_chi2 - exp_chi2 = self.experimental["model"].compute_losses()["loss"] / self.experimental["ndata"] + exp_chi2 = ( + self.experimental["model"].compute_losses()["loss"] + / self.experimental["ndata"] + ) return train_chi2, val_chi2, exp_chi2 def hyperparametrizable(self, params): @@ -878,16 +908,16 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() - + # Initialize all photon classes for the different replicas: if self.fiatlux_runcard is not None: - photons=Photon( + photons = Photon( theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard, replicas_id=self.replicas_id, ) else: - photons=None + photons = None ### Training loop for k, partition in enumerate(self.kpartitions): @@ -919,7 +949,6 @@ def hyperparametrizable(self, params): # There's no photon I guess pass - # Model generation joins all the different observable layers # together with pdf model generated above models = self._model_generation(xinput, pdf_models, partition, k) @@ -932,8 +961,12 @@ def hyperparametrizable(self, params): if k > 0: # Reset the positivity and integrability multipliers - pos_and_int = self.training["posdatasets"] + self.training["integdatasets"] - initial_values = self.training["posinitials"] + self.training["posinitials"] + pos_and_int = ( + self.training["posdatasets"] + self.training["integdatasets"] + ) + initial_values = ( + self.training["posinitials"] + self.training["posinitials"] + ) models["training"].reset_layer_weights_to(pos_and_int, initial_values) # Generate the list containing reporting info necessary for chi2 @@ -974,10 +1007,14 @@ def hyperparametrizable(self, params): validation_loss = np.mean(stopping_object.vl_chi2) # Compute experimental loss - exp_loss_raw = np.average(models["experimental"].compute_losses()["loss"]) + exp_loss_raw = np.average( + models["experimental"].compute_losses()["loss"] + ) # And divide by the number of active points in this fold # it would be nice to have a ndata_per_fold variable coming in the vp object... - ndata = np.sum([np.count_nonzero(i[k]) for i in self.experimental["folds"]]) + ndata = np.sum( + [np.count_nonzero(i[k]) for i in self.experimental["folds"]] + ) # If ndata == 0 then it's the opposite, all data is in! if ndata == 0: ndata = self.experimental["ndata"] @@ -985,12 +1022,18 @@ def hyperparametrizable(self, params): hyper_loss = experimental_loss if passed != self.pass_status: - log.info("Hyperparameter combination fail to find a good fit, breaking") + log.info( + "Hyperparameter combination fail to find a good fit, breaking" + ) # If the fit failed to fit, no need to add a penalty to the loss break for penalty in self.hyper_penalties: - hyper_loss += penalty(pdf_models=pdf_models, stopping_object=stopping_object) - log.info("Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed) + hyper_loss += penalty( + pdf_models=pdf_models, stopping_object=stopping_object + ) + log.info( + "Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed + ) # Now save all information from this fold l_hyper.append(hyper_loss) @@ -1039,5 +1082,9 @@ def hyperparametrizable(self, params): # In a normal run, the only information we need to output is the stopping object # (which contains metadata about the stopping) # and the pdf models (which are used to generate the PDF grids and compute arclengths) - dict_out = {"status": passed, "stopping_object": stopping_object, "pdf_models": pdf_models} + dict_out = { + "status": passed, + "stopping_object": stopping_object, + "pdf_models": pdf_models, + } return dict_out diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 6325d4afbb..09874b351e 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -36,26 +36,26 @@ def gen_integration_input(nx): return xgrid, weights_array -def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): +def msr_impose(nx=int(2e3), mode="All", scaler=None, photons=None): """ - This function receives: - Generates a function that applies a normalization layer to the fit. - - fit_layer: the 8-basis layer of PDF which we fit - The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) - - final_layer: the 14-basis which is fed to the fktable - and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). - It uses pdf_fit to compute the sum rule and returns a modified version of - the final_pdf layer with a normalisation by which the sum rule is imposed - - Parameters - ---------- - nx: int - number of points for the integration grid, default: 2000 - mode: str - what sum rules to compute (MSR, VSR or All), default: All - scaler: scaler - Function to apply to the input. If given the input to the model - will be a (1, None, 2) tensor where dim [:,:,0] is scaled + This function receives: + Generates a function that applies a normalization layer to the fit. + - fit_layer: the 8-basis layer of PDF which we fit + The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) + - final_layer: the 14-basis which is fed to the fktable + and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). + It uses pdf_fit to compute the sum rule and returns a modified version of + the final_pdf layer with a normalisation by which the sum rule is imposed + + Parameters + ---------- + nx: int + number of points for the integration grid, default: 2000 + mode: str + what sum rules to compute (MSR, VSR or All), default: All + scaler: scaler + Function to apply to the input. If given the input to the model + will be a (1, None, 2) tensor where dim [:,:,0] is scaled """ # 1. Generate the fake input which will be used to integrate @@ -84,14 +84,16 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): # and will return it appropiately normalized. def apply_normalization(layer_pdf, ph_replica): """ - layer_pdf: output of the PDF, unnormalized, ready for the fktable + layer_pdf: output of the PDF, unnormalized, ready for the fktable """ x_original = op.op_gather_keep_dims(xgrid_input, -1, axis=-1) - pdf_integrand = op.op_multiply([division_by_x(x_original), layer_pdf(xgrid_input)]) + pdf_integrand = op.op_multiply( + [division_by_x(x_original), layer_pdf(xgrid_input)] + ) normalization = normalizer(integrator(pdf_integrand), ph_replica) def ultimate_pdf(x): - return layer_pdf(x)*normalization + return layer_pdf(x) * normalization return ultimate_pdf diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 7ec49e0e00..d444690a07 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -20,11 +20,7 @@ from reportengine.namespaces import NSList -N3FIT_FIXED_CONFIG = dict( - use_cuts = 'internal', - use_t0 = True, - actions_ = [] -) +N3FIT_FIXED_CONFIG = dict(use_cuts="internal", use_t0=True, actions_=[]) FIT_NAMESPACE = "datacuts::theory::fitting " CLOSURE_NAMESPACE = "datacuts::theory::closuretest::fitting " @@ -45,6 +41,7 @@ INPUT_FOLDER = "input" TAB_FOLDER = "tables" + class N3FitError(Exception): """Exception raised when n3fit cannot succeed and knows why""" @@ -64,12 +61,12 @@ def init_output(self): # check if results folder exists self.output_path = pathlib.Path(self.output_path).absolute() - if not (self.output_path/"nnfit").is_dir(): + if not (self.output_path / "nnfit").is_dir(): if not re.fullmatch(r"[\w.\-]+", self.output_path.name): raise N3FitError("Invalid output folder name. Must be alphanumeric.") try: self.output_path.mkdir(exist_ok=True) - (self.output_path /"nnfit").mkdir(exist_ok=True) + (self.output_path / "nnfit").mkdir(exist_ok=True) except OSError as e: raise EnvironmentError_(e) from e @@ -124,18 +121,23 @@ def from_yaml(cls, o, *args, **kwargs): except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError(f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'.") + raise ConfigError( + f"Expecting input runcard to be a mapping, " + f"not '{type(file_content)}'." + ) - if file_content.get('closuretest') is not None: + if file_content.get("closuretest") is not None: namespace = CLOSURE_NAMESPACE else: namespace = FIT_NAMESPACE - N3FIT_FIXED_CONFIG['actions_'].append(namespace + "performfit") + N3FIT_FIXED_CONFIG["actions_"].append(namespace + "performfit") if fps := file_content["fitting"].get("savepseudodata", True): if fps != True: - raise TypeError(f"fitting::savepseudodata is neither True nor False ({fps})") + raise TypeError( + f"fitting::savepseudodata is neither True nor False ({fps})" + ) if len(kwargs["environment"].replicas) != 1: raise ConfigError( "Cannot request that multiple replicas are fitted and that " @@ -146,27 +148,40 @@ def from_yaml(cls, o, *args, **kwargs): training_action = namespace + "training_pseudodata" validation_action = namespace + "validation_pseudodata" - N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) - - if (thconfig:=file_content.get('fiatlux')) is not None: - N3FIT_FIXED_CONFIG['fiatlux']=thconfig - else : - N3FIT_FIXED_CONFIG['fiatlux']=None - #Theorycovmat flags and defaults - N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False - N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = False - N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = False - if (thconfig:=file_content.get('theorycovmatconfig')) is not None: - N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = thconfig.get('use_thcovmat_in_fitting', True) - N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = thconfig.get('use_thcovmat_in_sampling', True) - if N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] or N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting']: - N3FIT_FIXED_CONFIG['theory_covmat_flag'] = True - N3FIT_FIXED_CONFIG['use_user_uncertainties'] = thconfig.get('use_user_uncertainties', False) - N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = thconfig.get('use_scalevar_uncertainties', True) - #Sampling flags - if (sam_t0:=file_content.get('sampling')) is not None: - N3FIT_FIXED_CONFIG['separate_multiplicative'] = sam_t0.get('separate_multiplicative', True) - #Fitting flag + N3FIT_FIXED_CONFIG["actions_"].extend((training_action, validation_action)) + + if (thconfig := file_content.get("fiatlux")) is not None: + N3FIT_FIXED_CONFIG["fiatlux"] = thconfig + else: + N3FIT_FIXED_CONFIG["fiatlux"] = None + # Theorycovmat flags and defaults + N3FIT_FIXED_CONFIG["theory_covmat_flag"] = False + N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = False + N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = False + if (thconfig := file_content.get("theorycovmatconfig")) is not None: + N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = thconfig.get( + "use_thcovmat_in_fitting", True + ) + N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = thconfig.get( + "use_thcovmat_in_sampling", True + ) + if ( + N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] + or N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] + ): + N3FIT_FIXED_CONFIG["theory_covmat_flag"] = True + N3FIT_FIXED_CONFIG["use_user_uncertainties"] = thconfig.get( + "use_user_uncertainties", False + ) + N3FIT_FIXED_CONFIG["use_scalevar_uncertainties"] = thconfig.get( + "use_scalevar_uncertainties", True + ) + # Sampling flags + if (sam_t0 := file_content.get("sampling")) is not None: + N3FIT_FIXED_CONFIG["separate_multiplicative"] = sam_t0.get( + "separate_multiplicative", True + ) + # Fitting flag file_content.update(N3FIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -183,12 +198,13 @@ def parse_fakedata(self, fakedata: bool): """ if fakedata: log.warning("using filtered closure data") - if not (self.environment.output_path/'filter').is_dir(): + if not (self.environment.output_path / "filter").is_dir(): raise ConfigError( "Could not find filter result at " f"{self.environment.output_path/'filter'} " "to load commondata from. Did you run filter on the " - "runcard?") + "runcard?" + ) return fakedata def produce_use_fitcommondata(self, fakedata): @@ -198,8 +214,7 @@ def produce_use_fitcommondata(self, fakedata): return fakedata def produce_kfold_parameters(self, kfold=None, hyperopt=None): - """Return None even if there are kfolds in the runcard if the hyperopt flag is not active - """ + """Return None even if there are kfolds in the runcard if the hyperopt flag is not active""" if hyperopt is not None: return kfold return None @@ -237,18 +252,27 @@ def __init__(self): @property def argparser(self): parser = super().argparser - parser.add_argument("-o", "--output", help="Output folder and name of the fit", default=None) + parser.add_argument( + "-o", "--output", help="Output folder and name of the fit", default=None + ) def check_positive(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError("%s is an invalid positive int value." % value) + raise argparse.ArgumentTypeError( + "%s is an invalid positive int value." % value + ) return ivalue - parser.add_argument("--hyperopt", help="Enable hyperopt scan", default=None, type=int) + parser.add_argument( + "--hyperopt", help="Enable hyperopt scan", default=None, type=int + ) parser.add_argument("replica", help="MC replica number", type=check_positive) parser.add_argument( - "-r", "--replica_range", help="End of the range of replicas to compute", type=check_positive + "-r", + "--replica_range", + help="End of the range of replicas to compute", + type=check_positive, ) return parser @@ -260,7 +284,9 @@ def get_commandline_arguments(self, cmdline=None): def run(self): try: - self.environment.config_yml = pathlib.Path(self.args["config_yml"]).absolute() + self.environment.config_yml = pathlib.Path( + self.args["config_yml"] + ).absolute() replica = self.args["replica"] if self.args["replica_range"]: replicas = list(range(replica, self.args["replica_range"] + 1)) @@ -274,7 +300,9 @@ def run(self): sys.exit(1) except Exception as e: log.critical(f"Bug in n3fit ocurred. Please report it.") - print(colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr) + print( + colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr + ) sys.exit(1) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 88f139f63e..0a21099a38 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -74,7 +74,9 @@ def xfxQ(self, x, Q, n, fl): """Return the value of the PDF member for the given value in x""" if Q != self._fitting_q: log.warning( - "Querying N3LHAPDFSet at a value of Q=%f different from %f", Q, self._fitting_q + "Querying N3LHAPDFSet at a value of Q=%f different from %f", + Q, + self._fitting_q, ) return self.grid_values([fl], [x]).squeeze()[n] @@ -83,7 +85,7 @@ def _register_photon(self, xgrid): for m in self._lhapdf_set: pl = m.get_layer_re("add_photon") # if pl is an empy list there's no photon - if not pl : + if not pl: continue pl.register_photon(xgrid) # Recompile the model if necessary @@ -120,7 +122,9 @@ def __call__(self, xarr, flavours=None, replica=None): if replica is None or replica == 0: # We need generate output values for all replicas - result = np.concatenate([m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0) + result = np.concatenate( + [m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0 + ) if replica == 0: # We want _only_ the central value result = np.mean(result, axis=0, keepdims=True) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index f737b199c9..257545a59a 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -9,6 +9,7 @@ from scipy.integrate import trapezoid from eko.io import EKO + # from eko.io.manipulate import xgrid_reshape # from eko.interpolation import XGrid @@ -20,8 +21,10 @@ from os import remove import time + class Photon: """Photon class computing the photon array with the LuxQED approach.""" + def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard @@ -53,23 +56,28 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) - ff = open(f'fiatlux_runcard_{id}.yml', 'w+') + ff = open(f"fiatlux_runcard_{id}.yml", "w+") yaml.dump(self.fiatlux_runcard, ff) - self.lux[id] = fiatlux.FiatLux(f'fiatlux_runcard_{id}.yml') - remove(f'fiatlux_runcard_{id}.yml') + self.lux[id] = fiatlux.FiatLux(f"fiatlux_runcard_{id}.yml") + remove(f"fiatlux_runcard_{id}.yml") # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick # we print different runcards for every replica so they do not interfere with each other - for id in replicas_id : + for id in replicas_id: self.lux[id].PlugAlphaQED(self.alpha_em, self.qref) - self.lux[id].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) + self.lux[id].InsertInelasticSplitQ( + [ + self.thresh_b, + self.thresh_t if self.theory["MaxNfPdf"] == 6 else 1e100, + ] + ) self.lux[id].PlugStructureFunctions(f2[id].fxq, fl[id].fxq, f2lo[id].fxq) - + self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() self.produce_interpolators() - + def alpha_em(self, q): r""" Compute the value of alpha_em. @@ -78,27 +86,22 @@ def alpha_em(self, q): ---------- q: float value in which the coupling is computed - + Returns ------- alpha_em: float electromagnetic coupling """ - if q < self.thresh_c : + if q < self.thresh_c: nf = 3 - elif q < self.thresh_b : + elif q < self.thresh_b: nf = 4 - elif q < self.thresh_t : + elif q < self.thresh_t: nf = 5 - else : + else: nf = 6 - return self.alpha_em_nlo( - q, - self.alpha_thresh[nf], - self.thresh[nf], - nf - ) - + return self.alpha_em_nlo(q, self.alpha_thresh[nf], self.thresh[nf], nf) + def alpha_em_nlo(self, q, alpha_ref, qref, nf): """ Compute the alpha_em running for FFS at NLO. @@ -124,53 +127,66 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): alpha_LO = alpha_ref / den alpha_NLO = alpha_LO * (1 - self.b1[nf] * alpha_LO * np.log(den)) return alpha_NLO - + def set_thresholds_alpha_em(self): """Compute and store the couplings at thresholds""" self.thresh_c = self.theory["kcThr"] * self.theory["mc"] self.thresh_b = self.theory["kbThr"] * self.theory["mb"] self.thresh_t = self.theory["ktThr"] * self.theory["mt"] - if self.theory["MaxNfAs"] <= 5 : + if self.theory["MaxNfAs"] <= 5: self.thresh_t = np.inf - if self.theory["MaxNfAs"] <= 4 : + if self.theory["MaxNfAs"] <= 4: self.thresh_b = np.inf - if self.theory["MaxNfAs"] <= 3 : + if self.theory["MaxNfAs"] <= 3: self.thresh_c = np.inf thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] - if self.qref < self.thresh_c : + if self.qref < self.thresh_c: nfref = 3 - elif self.qref < self.thresh_b : + elif self.qref < self.thresh_b: nfref = 4 - elif self.qref < self.thresh_t : + elif self.qref < self.thresh_t: nfref = 5 - else : + else: nfref = 6 thresh_list.insert(nfref - 3, self.qref) - self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} + self.thresh = { + nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1) + } self.alpha_thresh = {nfref: self.alpha_em_ref} for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): - self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1) - + self.alpha_thresh[nf] = self.alpha_em_nlo( + self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1 + ) + for nf in reversed(range(3, nfref)): - self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1) - + self.alpha_thresh[nf] = self.alpha_em_nlo( + self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1 + ) + def set_betas(self): """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" nl = 3 nc = 3 - eu2 = 4. / 9 - ed2 = 1. / 9 + eu2 = 4.0 / 9 + ed2 = 1.0 / 9 self.beta0 = {} self.b1 = {} - for nf in range(3, 6+1): + for nf in range(3, 6 + 1): nu = nf // 2 nd = nf - nu - self.beta0[nf] = ( -4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2)) ) / (4 * np.pi) - self.b1[nf] = -4.0 * ( nl + nc * (nu * eu2**2 + nd * ed2**2) ) / self.beta0[nf] / (4 * np.pi)**2 + self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / ( + 4 * np.pi + ) + self.b1[nf] = ( + -4.0 + * (nl + nc * (nu * eu2**2 + nd * ed2**2)) + / self.beta0[nf] + / (4 * np.pi) ** 2 + ) def compute_photon_array(self, id): r""" @@ -180,7 +196,7 @@ def compute_photon_array(self, id): ---------- xgrids: numpy.array grid of the x points - + Returns ------- compute_photon_array: numpy.array @@ -188,7 +204,9 @@ def compute_photon_array(self, id): """ # Compute photon PDF start_time = time.perf_counter() - photon_100GeV = np.array([self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid]) + photon_100GeV = np.array( + [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] + ) photon_100GeV += self.generate_errors(id) photon_100GeV /= self.xgrid print("Time to compute photon:", time.perf_counter() - start_time) @@ -196,7 +214,7 @@ def compute_photon_array(self, id): # Load eko and reshape it with EKO.read(self.path_to_eko_photon) as eko: - # If we make sure that the grid of the precomputed EKO is the same of + # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape # TODO : move the reshape inside vp-setupfit # xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) @@ -204,18 +222,15 @@ def compute_photon_array(self, id): # construct PDFs pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) for j, pid in enumerate(eko.rotations.inputpids): - if pid == 22 : + if pid == 22: pdfs[j] = photon_100GeV ph_id = j if pid not in self.qcd_pdfs.flavors: continue pdfs[j] = np.array( - [ - self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x - for x in self.xgrid - ] + [self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x for x in self.xgrid] ) - + # Apply EKO to PDFs q2 = eko.mu2grid[0] with eko.operator(q2) as elem: @@ -226,14 +241,15 @@ def compute_photon_array(self, id): # we want x * gamma(x) return self.xgrid * photon_Q0 - + def produce_interpolators(self): """Produce the interpolation functions to be called in compute.""" self.photons_array = [self.compute_photon_array(id) for id in self.replicas_id] self.interpolator = [ - interp1d(self.xgrid, photon_array, fill_value=0., kind='cubic') for photon_array in self.photons_array + interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") + for photon_array in self.photons_array ] - + def compute(self, xgrid): """ Compute the photon interpolating the values of self.photon_array. @@ -242,44 +258,54 @@ def compute(self, xgrid): ---------- xgrid : nd.array array of x values with shape (1,xgrid,1) - + Returns ------- photon values : nd.array array of photon values with shape (1,xgrid,1) """ return [ - self.interpolator[id](xgrid[0,:,0])[np.newaxis,:,np.newaxis] + self.interpolator[id](xgrid[0, :, 0])[np.newaxis, :, np.newaxis] for id in range(len(self.replicas_id)) ] - + def integrate(self): """Compute the integral of the photon on the x range.""" - return [trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id))] - + return [ + trapezoid(self.photons_array[id], self.xgrid) + for id in range(len(self.replicas_id)) + ] + def generate_error_matrix(self): """generate error matrix to be used for the additional errors.""" - if not self.fiatlux_runcard["additional_errors"] : + if not self.fiatlux_runcard["additional_errors"]: return None extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") # random generator must be set previously in case of parallel replicas return np.array( [ - [(extra_set.xfxQ(x, self.q_in, i, 22) - extra_set.xfxQ(x, self.q_in, 0, 22)) for i in range(101, 107+1)] + [ + ( + extra_set.xfxQ(x, self.q_in, i, 22) + - extra_set.xfxQ(x, self.q_in, 0, 22) + ) + for i in range(101, 107 + 1) + ] for x in self.xgrid ] - ) # first index must be x, while second one must be replica index + ) # first index must be x, while second one must be replica index def generate_errors(self, replica_id): """generate LUX additional errors.""" - if self.error_matrix is None : + if self.error_matrix is None: return np.zeros_like(self.xgrid) seed = replica_luxseed(replica_id, self.fiatlux_runcard["luxseed"]) rng = np.random.default_rng(seed=seed) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) errors = u @ (s * rng.normal(size=7)) return errors - + + # TODO : move it in n3fit_data.py with the others replica_seed generators? # or use directly replica_nnseed? def replica_luxseed(replica, luxseed): diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 3b851c2af3..4745fefd7f 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -2,22 +2,25 @@ import numpy as np from scipy.interpolate import RectBivariateSpline -class StructureFunction : + +class StructureFunction: """Abstract class for the DIS structure functions""" - def fxq(self, x, Q) : + + def fxq(self, x, Q): pass + class InterpStructureFunction(StructureFunction): """ Compute an interpolated structure function convoluting an FKtable with a PDF. """ + def __init__(self, path_to_fktable, pdfs): self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs self.pdgid = int(pdfs.set().get_entry("Particle")) self.produce_interpolator() - def produce_interpolator(self): """Produce the interpolation function to be called in fxq.""" @@ -33,7 +36,7 @@ def produce_interpolator(self): grid2D = predictions.reshape(len(x), len(q2)) self.interpolator = RectBivariateSpline(x, q2, grid2D) - + def fxq(self, x, q): r""" Compute the DIS structure function interpolating the grid. @@ -44,7 +47,7 @@ def fxq(self, x, q): Bjorken's variable Q : float DIS hard scale - + Returns ------- F_{2,L}: float @@ -56,23 +59,25 @@ def fxq(self, x, q): # return 0. return self.interpolator(x, q**2)[0, 0] -class F2LO(StructureFunction) : + +class F2LO(StructureFunction): """Compute analytically the leading order structure function for F2.""" + def __init__(self, pdfs, theory): self.pdfs = pdfs self.thresh_c = theory["kcThr"] * theory["mc"] self.thresh_b = theory["kbThr"] * theory["mb"] self.thresh_t = theory["ktThr"] * theory["mt"] - if theory["MaxNfPdf"] <= 5 : + if theory["MaxNfPdf"] <= 5: self.thresh_t = np.inf - if theory["MaxNfPdf"] <= 4 : + if theory["MaxNfPdf"] <= 4: self.thresh_b = np.inf - if theory["MaxNfPdf"] <= 3 : + if theory["MaxNfPdf"] <= 3: self.thresh_c = np.inf - eu2 = 4. / 9 - ed2 = 1. / 9 - self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t - + eu2 = 4.0 / 9 + ed2 = 1.0 / 9 + self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t + def fxq(self, x, q): r""" Compute the analytical form of F2LO. @@ -83,23 +88,23 @@ def fxq(self, x, q): Bjorken's variable Q : float DIS hard scale - + Returns ------- F2_LO : float Structure function F2 at LO """ # at LO we use ZM-VFS - if q < self.thresh_c : + if q < self.thresh_c: nf = 3 - elif q < self.thresh_b : + elif q < self.thresh_b: nf = 4 - elif q < self.thresh_t : + elif q < self.thresh_t: nf = 5 - else : + else: nf = 6 res = 0 pdfs_values = self.pdfs.xfxQ(x, q) - for i in range(1, nf+1): - res += self.eq2[i-1] * (pdfs_values[i] + pdfs_values[-i]) + for i in range(1, nf + 1): + res += self.eq2[i - 1] * (pdfs_values[i] + pdfs_values[-i]) return res From 703598b263e96a30c56118a1cff66dddc3d516b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Mon, 3 Apr 2023 11:54:39 +0200 Subject: [PATCH 112/204] Update n3fit/src/n3fit/model_gen.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/model_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 04240784b5..33356cff51 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -499,7 +499,7 @@ def pdfNN_layer_generator( parallel_models: int How many models should be trained in parallel photon: Function - If given, gives the `add_photon` a function to compute a photon which will be added at the + If given, gives the ``add_photon`` a function to compute a photon which will be added at the index 0 of the 14-size FK basis This same function will also be used to compute the MSR component for the photon From e2cc2bcd0a21c7ea7870630059dcfa0779280f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 12:40:09 +0200 Subject: [PATCH 113/204] Fix again _register_photon --- n3fit/runcards/examples/Basic_runcard_qed.yml | 2 +- n3fit/src/n3fit/layers/rotations.py | 4 ++-- n3fit/src/n3fit/model_gen.py | 11 ++++++----- n3fit/src/n3fit/vpinterface.py | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index ef70d3fd19..80a6e8a3ab 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -112,7 +112,7 @@ parameters: # This defines the parameter dictionary that is passed to the Model clipnorm: 6.073e-6 learning_rate: 2.621e-3 optimizer_name: Nadam - epochs: 17000 + epochs: 900 positivity: initial: 184.8 multiplier: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 7e27ee9870..1b709d85e0 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -116,10 +116,10 @@ def register_photon(self, xgrid): self._pdf_ph = self._photons_generator.compute(xgrid) self.built = False - def call(self, pdfs, i): + def call(self, pdfs, ph_replica): if self._pdf_ph is None: return pdfs - return op.concatenate([self._pdf_ph[i], pdfs[:, :, 1:]], axis=-1) + return op.concatenate([self._pdf_ph[ph_replica], pdfs[:, :, 1:]], axis=-1) class ObsRotation(MetaLayer): diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 04240784b5..9a82119cb8 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -656,12 +656,13 @@ def layer_pdf(x): # Photon layer, changes the photon from zero to non-zero def apply_photon(x): - return layer_photon(normalized_pdf(x), i) + # if photon is None then the pothon_layer is not applied + if photons is None: + return normalized_pdf(x) + else: + return layer_photon(normalized_pdf(x), i) - if photons is None: - final_pdf = normalized_pdf - else: - final_pdf = apply_photon + final_pdf = apply_photon # Create the model pdf_model = MetaModel( diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 0a21099a38..10ea15d7fc 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -87,9 +87,10 @@ def _register_photon(self, xgrid): # if pl is an empy list there's no photon if not pl: continue - pl.register_photon(xgrid) + # TODO : not sure this will work in parallel mode + pl[0].register_photon(xgrid) # Recompile the model if necessary - if not pl.built: + if not pl[0].built: m.compile() def __call__(self, xarr, flavours=None, replica=None): From 2eff55a23b2c58a9f2e24fcf1bb018f460e1794b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Mon, 3 Apr 2023 12:40:33 +0200 Subject: [PATCH 114/204] Update n3fit/src/n3fit/model_trainer.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/model_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index fbc7425bf0..62f67a2801 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -700,7 +700,7 @@ def _generate_pdf( dictionary of arguments for the regularizer seed: int seed for the NN - photon: Photon + :py:class:`validphys.photon.compute.Photon` function to compute the photon PDF see model_gen.pdfNN_layer_generator for more information From ba091f6c9150d7dcd7fb6746fc94de5fabe4a917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Mon, 3 Apr 2023 12:51:46 +0200 Subject: [PATCH 115/204] Update n3fit/src/n3fit/model_trainer.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/model_trainer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 62f67a2801..4451e58cdf 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -940,14 +940,10 @@ def hyperparametrizable(self, params): photons, ) - # Register the fitting grid with the photon layer - try: + if photons is not None: for m in pdf_models: pl = m.get_layer("add_photon") pl.register_photon(xinput.input.tensor_content) - except ValueError: - # There's no photon I guess - pass # Model generation joins all the different observable layers # together with pdf model generated above From 520039364ade7607809b454ad473292c913e2d79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Mon, 3 Apr 2023 12:52:10 +0200 Subject: [PATCH 116/204] Update n3fit/src/n3fit/vpinterface.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/vpinterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 10ea15d7fc..3762d33a6b 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -118,7 +118,7 @@ def __call__(self, xarr, flavours=None, replica=None): # as the scaling is done by the model itself mod_xgrid = xarr.reshape(1, -1, 1) - # Try registering the grid with the photon + # Register the grid with the photon self._register_photon(mod_xgrid) if replica is None or replica == 0: From 3ffee28d5a961b60c2d7f812879879c2fc36b86b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 14:08:04 +0200 Subject: [PATCH 117/204] Modify generate_error_matrix --- validphys2/src/validphys/photon/compute.py | 30 ++++++++++------------ 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 257545a59a..beb592a14b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -7,6 +7,7 @@ from . import structure_functions as sf from scipy.interpolate import interp1d from scipy.integrate import trapezoid +import logging from eko.io import EKO @@ -21,6 +22,8 @@ from os import remove import time +log = logging.getLogger(__name__) + class Photon: """Photon class computing the photon array with the LuxQED approach.""" @@ -194,8 +197,8 @@ def compute_photon_array(self, id): Parameters ---------- - xgrids: numpy.array - grid of the x points + id: int + replica id Returns ------- @@ -209,7 +212,7 @@ def compute_photon_array(self, id): ) photon_100GeV += self.generate_errors(id) photon_100GeV /= self.xgrid - print("Time to compute photon:", time.perf_counter() - start_time) + log.info(f"Time to compute photon: {time.perf_counter() - start_time}") # TODO : the different x points could be even computed in parallel # Load eko and reshape it @@ -281,19 +284,14 @@ def generate_error_matrix(self): if not self.fiatlux_runcard["additional_errors"]: return None extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") - # random generator must be set previously in case of parallel replicas - return np.array( - [ - [ - ( - extra_set.xfxQ(x, self.q_in, i, 22) - - extra_set.xfxQ(x, self.q_in, 0, 22) - ) - for i in range(101, 107 + 1) - ] - for x in self.xgrid - ] - ) # first index must be x, while second one must be replica index + qs = [self.q_in]*len(self.xgrid) + res_central = np.array(extra_set.central_member.xfxQ(22, self.xgrid, qs)) + res = [] + for idx_member in range(101, 107+1): + tmp = np.array(extra_set.members[idx_member].xfxQ(22, self.xgrid, qs)) + res.append(tmp - res_central) + # first index must be x, while second one must be replica index + return np.stack(res, axis=1) def generate_errors(self, replica_id): """generate LUX additional errors.""" From b76bf28a8bc7bbcadba812960bca189e39e0a962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 14:33:42 +0200 Subject: [PATCH 118/204] Add docstrings in set_thresholds_alpha_em --- validphys2/src/validphys/photon/compute.py | 39 +++++++++++++--------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index beb592a14b..99559d4cd2 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -1,26 +1,23 @@ """Script that calls fiatlux to add the photon PDF.""" import numpy as np - -from validphys.lhapdfset import LHAPDFSet -from validphys.n3fit_data import replica_nnseed - -from . import structure_functions as sf from scipy.interpolate import interp1d from scipy.integrate import trapezoid -import logging +from os import remove +import time +import fiatlux +import yaml from eko.io import EKO - # from eko.io.manipulate import xgrid_reshape # from eko.interpolation import XGrid +from validphys.lhapdfset import LHAPDFSet +from validphys.n3fit_data import replica_nnseed +from . import structure_functions as sf +import logging from n3fit.io.writer import XGRID -import fiatlux -import yaml -from os import remove -import time log = logging.getLogger(__name__) @@ -132,7 +129,16 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): return alpha_NLO def set_thresholds_alpha_em(self): - """Compute and store the couplings at thresholds""" + """ + Compute and store the couplings at thresholds to speed up the calling + to alpha_em inside fiatlux: + when q is in a certain range (e.g. thresh_c < q < thresh_b) and qref in a different one + (e.g. thresh_b < q < thresh_t) we need to evolve from qref to thresh_b with nf=5 and then + from thresh_b to q with nf=4. Given that the value of alpha at thresh_b is always the same + we can avoid computing the first step storing the values of alpha in the threshold points. + It is done for qref in a generic range (not necessarly qref=91.2). + + """ self.thresh_c = self.theory["kcThr"] * self.theory["mc"] self.thresh_b = self.theory["kbThr"] * self.theory["mb"] self.thresh_t = self.theory["ktThr"] * self.theory["mt"] @@ -144,6 +150,7 @@ def set_thresholds_alpha_em(self): self.thresh_c = np.inf thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] + # determine nfref if self.qref < self.thresh_c: nfref = 3 elif self.qref < self.thresh_b: @@ -160,6 +167,7 @@ def set_thresholds_alpha_em(self): self.alpha_thresh = {nfref: self.alpha_em_ref} + # determine the values of alpha in the threshold points, depending on the value of qref for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): self.alpha_thresh[nf] = self.alpha_em_nlo( self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1 @@ -207,9 +215,10 @@ def compute_photon_array(self, id): """ # Compute photon PDF start_time = time.perf_counter() - photon_100GeV = np.array( - [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] - ) + # photon_100GeV = np.array( + # [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] + # ) + photon_100GeV = np.exp(-self.xgrid) photon_100GeV += self.generate_errors(id) photon_100GeV /= self.xgrid log.info(f"Time to compute photon: {time.perf_counter() - start_time}") From d1ca414bcbf3100b6fb25505c9d6a862ab4a42fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 14:38:20 +0200 Subject: [PATCH 119/204] Revert "Call black on modified files" This reverts commit 4c709ef4bdc38b719bfa0933c62407719f61e08f. --- n3fit/src/n3fit/layers/msr_normalization.py | 8 +- n3fit/src/n3fit/layers/rotations.py | 12 +- n3fit/src/n3fit/model_gen.py | 10 +- n3fit/src/n3fit/model_trainer.py | 93 +++--------- n3fit/src/n3fit/msr.py | 46 +++--- n3fit/src/n3fit/scripts/n3fit_exec.py | 112 ++++++--------- n3fit/src/n3fit/vpinterface.py | 10 +- validphys2/src/validphys/photon/compute.py | 132 ++++++++---------- .../validphys/photon/structure_functions.py | 45 +++--- 9 files changed, 177 insertions(+), 291 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 579977692b..97111e83c6 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -55,13 +55,11 @@ def call(self, pdf_integrated, ph_replica): if self._photons is not None: photon_integral = self._photons[ph_replica] - else: - photon_integral = 0.0 + else : + photon_integral = 0. if self._msr_enabled: - n_ag = [ - (1.0 - y[GLUON_IDX[0][0] - 1] - photon_integral) / y[GLUON_IDX[0][0]] - ] * len(GLUON_IDX) + n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - photon_integral) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 1b709d85e0..3f1f896e8d 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -28,7 +28,7 @@ def __init__(self, rotation_matrix, axes=1, **kwargs): super().__init__(**kwargs) def is_identity(self): - """Returns true if the rotation is an identity""" + """ Returns true if the rotation is an identity """ # check whether it is a mxm matrix if self.rotation_matrix.shape[0] == self.rotation_matrix.shape[1]: # check whether it is the identity @@ -41,15 +41,12 @@ def call(self, x_raw): class FlavourToEvolution(Rotation): """ - Rotates from the flavour basis to - the evolution basis. + Rotates from the flavour basis to + the evolution basis. """ def __init__( - self, - flav_info, - fitbasis, - **kwargs, + self, flav_info, fitbasis, **kwargs, ): rotation_matrix = pdfbases.fitbasis_to_NN31IC(flav_info, fitbasis) super().__init__(rotation_matrix, axes=1, **kwargs) @@ -94,7 +91,6 @@ def call(self, pdf_raw): # Concatenating destroys the batch index so we have to regenerate it return op.batchit(ret) - class AddPhoton(MetaLayer): """ Changes the value of the photon component of the PDF to non-zero. diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index dda8304c5d..5d2dfc4e27 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -351,9 +351,7 @@ def generate_dense_per_flavour_network( initializers = [] for _ in range(basis_size): # select the initializer and move the seed - initializers.append( - MetaLayer.select_initializer(initializer_name, seed=current_seed) - ) + initializers.append(MetaLayer.select_initializer(initializer_name, seed=current_seed)) current_seed += 1 # set the arguments that will define the layer @@ -575,9 +573,7 @@ def pdfNN_layer_generator( # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose( - mode=impose_sumrule, scaler=scaler, photons=photons - ) + sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photons=photons) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x @@ -650,7 +646,7 @@ def layer_fitbasis(x): # Rotation layer, changes from the 8-basis to the 14-basis def layer_pdf(x): return layer_evln(layer_fitbasis(x)) - + # Final PDF (apply normalization) normalized_pdf = sumrule_layer(layer_pdf, i) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 4451e58cdf..5411c55aba 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -37,7 +37,6 @@ # See ModelTrainer::_xgrid_generation for the definition of each field and how they are generated InputInfo = namedtuple("InputInfo", ["input", "split", "idx"]) - def _pdf_injection(pdf_layers, observables, masks): """ Takes as input a list of PDF layers each corresponding to one observable (also given as a list) @@ -187,9 +186,7 @@ def __init__( hyper_loss = kfold_parameters.get("target", None) if hyper_loss is None: hyper_loss = "average" - log.warning( - "No minimization target selected, defaulting to '%s'", hyper_loss - ) + log.warning("No minimization target selected, defaulting to '%s'", hyper_loss) log.info("Using '%s' as the target for hyperoptimization", hyper_loss) self._hyper_loss = getattr(n3fit.hyper_optimization.rewards, hyper_loss) @@ -465,9 +462,7 @@ def _model_generation(self, xinput, pdf_models, partition, partition_idx): validation = MetaModel(full_model_input_dict, output_vl) # Or the positivity in the total chi2 - output_ex = _pdf_injection( - exp_pdfs, self.experimental["output"], experimental_mask - ) + output_ex = _pdf_injection(exp_pdfs, self.experimental["output"], experimental_mask) experimental = MetaModel(full_model_input_dict, output_ex) if self.print_summary: @@ -563,9 +558,7 @@ def _generate_observables( all_pos_initial, all_pos_multiplier, max_lambda, positivity_steps ) - pos_layer = model_gen.observable_generator( - pos_dict, positivity_initial=pos_initial - ) + pos_layer = model_gen.observable_generator(pos_dict, positivity_initial=pos_initial) # The input list is still common self.input_list.append(pos_layer["inputs"]) @@ -580,18 +573,13 @@ def _generate_observables( if self.integ_info is not None: for integ_dict in self.integ_info: if not self.mode_hyperopt: - log.info( - "Generating integrability penalty for %s", integ_dict["name"] - ) + log.info("Generating integrability penalty for %s", integ_dict["name"]) integrability_steps = int(epochs / PUSH_INTEGRABILITY_EACH) max_lambda = integ_dict["lambda"] integ_initial, integ_multiplier = _LM_initial_and_multiplier( - all_integ_initial, - all_integ_multiplier, - max_lambda, - integrability_steps, + all_integ_initial, all_integ_multiplier, max_lambda, integrability_steps ) integ_layer = model_gen.observable_generator( @@ -617,15 +605,10 @@ def _generate_observables( force_set_smallest = input_arr.min() > 1e-9 if force_set_smallest: new_xgrid = np.linspace( - start=1 / input_arr_size, - stop=1.0, - endpoint=False, - num=input_arr_size, + start=1 / input_arr_size, stop=1.0, endpoint=False, num=input_arr_size ) else: - new_xgrid = np.linspace( - start=0, stop=1.0, endpoint=False, num=input_arr_size - ) + new_xgrid = np.linspace(start=0, stop=1.0, endpoint=False, num=input_arr_size) # When mapping the FK xgrids onto our new grid, we need to consider degeneracies among # the x-values in the FK grids @@ -645,9 +628,7 @@ def _generate_observables( # Select the indices of the points that will be used by the interpolator onein = map_from_complete.size / (int(interpolation_points) - 1) - selected_points = [ - round(i * onein - 1) for i in range(1, int(interpolation_points)) - ] + selected_points = [round(i * onein - 1) for i in range(1, int(interpolation_points))] if selected_points[0] != 0: selected_points = [0] + selected_points map_from = map_from_complete[selected_points] @@ -658,8 +639,7 @@ def _generate_observables( scaler = PchipInterpolator(map_from, map_to) except ValueError: raise ValueError( - "interpolation_points is larger than the number of unique " - "input x-values" + "interpolation_points is larger than the number of unique " "input x-values" ) self._scaler = lambda x: np.concatenate([scaler(np.log(x)), x], axis=-1) @@ -736,14 +716,7 @@ def _prepare_reporting(self, partition): to select the bits necessary for reporting the chi2. Receives the chi2 partition data to see whether any dataset is to be left out """ - reported_keys = [ - "name", - "count_chi2", - "positivity", - "integrability", - "ndata", - "ndata_vl", - ] + reported_keys = ["name", "count_chi2", "positivity", "integrability", "ndata", "ndata_vl"] reporting_list = [] for exp_dict in self.all_info: reporting_dict = {k: exp_dict.get(k) for k in reported_keys} @@ -844,10 +817,7 @@ def evaluate(self, stopping_object): # training and the validation which are actually `chi2` and not part of the penalty train_chi2 = stopping_object.evaluate_training(self.training["model"]) val_chi2 = stopping_object.vl_chi2 - exp_chi2 = ( - self.experimental["model"].compute_losses()["loss"] - / self.experimental["ndata"] - ) + exp_chi2 = self.experimental["model"].compute_losses()["loss"] / self.experimental["ndata"] return train_chi2, val_chi2, exp_chi2 def hyperparametrizable(self, params): @@ -908,16 +878,16 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() - + # Initialize all photon classes for the different replicas: if self.fiatlux_runcard is not None: - photons = Photon( + photons=Photon( theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard, replicas_id=self.replicas_id, ) else: - photons = None + photons=None ### Training loop for k, partition in enumerate(self.kpartitions): @@ -945,6 +915,7 @@ def hyperparametrizable(self, params): pl = m.get_layer("add_photon") pl.register_photon(xinput.input.tensor_content) + # Model generation joins all the different observable layers # together with pdf model generated above models = self._model_generation(xinput, pdf_models, partition, k) @@ -957,12 +928,8 @@ def hyperparametrizable(self, params): if k > 0: # Reset the positivity and integrability multipliers - pos_and_int = ( - self.training["posdatasets"] + self.training["integdatasets"] - ) - initial_values = ( - self.training["posinitials"] + self.training["posinitials"] - ) + pos_and_int = self.training["posdatasets"] + self.training["integdatasets"] + initial_values = self.training["posinitials"] + self.training["posinitials"] models["training"].reset_layer_weights_to(pos_and_int, initial_values) # Generate the list containing reporting info necessary for chi2 @@ -1003,14 +970,10 @@ def hyperparametrizable(self, params): validation_loss = np.mean(stopping_object.vl_chi2) # Compute experimental loss - exp_loss_raw = np.average( - models["experimental"].compute_losses()["loss"] - ) + exp_loss_raw = np.average(models["experimental"].compute_losses()["loss"]) # And divide by the number of active points in this fold # it would be nice to have a ndata_per_fold variable coming in the vp object... - ndata = np.sum( - [np.count_nonzero(i[k]) for i in self.experimental["folds"]] - ) + ndata = np.sum([np.count_nonzero(i[k]) for i in self.experimental["folds"]]) # If ndata == 0 then it's the opposite, all data is in! if ndata == 0: ndata = self.experimental["ndata"] @@ -1018,18 +981,12 @@ def hyperparametrizable(self, params): hyper_loss = experimental_loss if passed != self.pass_status: - log.info( - "Hyperparameter combination fail to find a good fit, breaking" - ) + log.info("Hyperparameter combination fail to find a good fit, breaking") # If the fit failed to fit, no need to add a penalty to the loss break for penalty in self.hyper_penalties: - hyper_loss += penalty( - pdf_models=pdf_models, stopping_object=stopping_object - ) - log.info( - "Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed - ) + hyper_loss += penalty(pdf_models=pdf_models, stopping_object=stopping_object) + log.info("Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed) # Now save all information from this fold l_hyper.append(hyper_loss) @@ -1078,9 +1035,5 @@ def hyperparametrizable(self, params): # In a normal run, the only information we need to output is the stopping object # (which contains metadata about the stopping) # and the pdf models (which are used to generate the PDF grids and compute arclengths) - dict_out = { - "status": passed, - "stopping_object": stopping_object, - "pdf_models": pdf_models, - } + dict_out = {"status": passed, "stopping_object": stopping_object, "pdf_models": pdf_models} return dict_out diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 09874b351e..6325d4afbb 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -36,26 +36,26 @@ def gen_integration_input(nx): return xgrid, weights_array -def msr_impose(nx=int(2e3), mode="All", scaler=None, photons=None): +def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): """ - This function receives: - Generates a function that applies a normalization layer to the fit. - - fit_layer: the 8-basis layer of PDF which we fit - The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) - - final_layer: the 14-basis which is fed to the fktable - and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). - It uses pdf_fit to compute the sum rule and returns a modified version of - the final_pdf layer with a normalisation by which the sum rule is imposed - - Parameters - ---------- - nx: int - number of points for the integration grid, default: 2000 - mode: str - what sum rules to compute (MSR, VSR or All), default: All - scaler: scaler - Function to apply to the input. If given the input to the model - will be a (1, None, 2) tensor where dim [:,:,0] is scaled + This function receives: + Generates a function that applies a normalization layer to the fit. + - fit_layer: the 8-basis layer of PDF which we fit + The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) + - final_layer: the 14-basis which is fed to the fktable + and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). + It uses pdf_fit to compute the sum rule and returns a modified version of + the final_pdf layer with a normalisation by which the sum rule is imposed + + Parameters + ---------- + nx: int + number of points for the integration grid, default: 2000 + mode: str + what sum rules to compute (MSR, VSR or All), default: All + scaler: scaler + Function to apply to the input. If given the input to the model + will be a (1, None, 2) tensor where dim [:,:,0] is scaled """ # 1. Generate the fake input which will be used to integrate @@ -84,16 +84,14 @@ def msr_impose(nx=int(2e3), mode="All", scaler=None, photons=None): # and will return it appropiately normalized. def apply_normalization(layer_pdf, ph_replica): """ - layer_pdf: output of the PDF, unnormalized, ready for the fktable + layer_pdf: output of the PDF, unnormalized, ready for the fktable """ x_original = op.op_gather_keep_dims(xgrid_input, -1, axis=-1) - pdf_integrand = op.op_multiply( - [division_by_x(x_original), layer_pdf(xgrid_input)] - ) + pdf_integrand = op.op_multiply([division_by_x(x_original), layer_pdf(xgrid_input)]) normalization = normalizer(integrator(pdf_integrand), ph_replica) def ultimate_pdf(x): - return layer_pdf(x) * normalization + return layer_pdf(x)*normalization return ultimate_pdf diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index d444690a07..7ec49e0e00 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -20,7 +20,11 @@ from reportengine.namespaces import NSList -N3FIT_FIXED_CONFIG = dict(use_cuts="internal", use_t0=True, actions_=[]) +N3FIT_FIXED_CONFIG = dict( + use_cuts = 'internal', + use_t0 = True, + actions_ = [] +) FIT_NAMESPACE = "datacuts::theory::fitting " CLOSURE_NAMESPACE = "datacuts::theory::closuretest::fitting " @@ -41,7 +45,6 @@ INPUT_FOLDER = "input" TAB_FOLDER = "tables" - class N3FitError(Exception): """Exception raised when n3fit cannot succeed and knows why""" @@ -61,12 +64,12 @@ def init_output(self): # check if results folder exists self.output_path = pathlib.Path(self.output_path).absolute() - if not (self.output_path / "nnfit").is_dir(): + if not (self.output_path/"nnfit").is_dir(): if not re.fullmatch(r"[\w.\-]+", self.output_path.name): raise N3FitError("Invalid output folder name. Must be alphanumeric.") try: self.output_path.mkdir(exist_ok=True) - (self.output_path / "nnfit").mkdir(exist_ok=True) + (self.output_path /"nnfit").mkdir(exist_ok=True) except OSError as e: raise EnvironmentError_(e) from e @@ -121,23 +124,18 @@ def from_yaml(cls, o, *args, **kwargs): except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError( - f"Expecting input runcard to be a mapping, " - f"not '{type(file_content)}'." - ) + raise ConfigError(f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'.") - if file_content.get("closuretest") is not None: + if file_content.get('closuretest') is not None: namespace = CLOSURE_NAMESPACE else: namespace = FIT_NAMESPACE - N3FIT_FIXED_CONFIG["actions_"].append(namespace + "performfit") + N3FIT_FIXED_CONFIG['actions_'].append(namespace + "performfit") if fps := file_content["fitting"].get("savepseudodata", True): if fps != True: - raise TypeError( - f"fitting::savepseudodata is neither True nor False ({fps})" - ) + raise TypeError(f"fitting::savepseudodata is neither True nor False ({fps})") if len(kwargs["environment"].replicas) != 1: raise ConfigError( "Cannot request that multiple replicas are fitted and that " @@ -148,40 +146,27 @@ def from_yaml(cls, o, *args, **kwargs): training_action = namespace + "training_pseudodata" validation_action = namespace + "validation_pseudodata" - N3FIT_FIXED_CONFIG["actions_"].extend((training_action, validation_action)) - - if (thconfig := file_content.get("fiatlux")) is not None: - N3FIT_FIXED_CONFIG["fiatlux"] = thconfig - else: - N3FIT_FIXED_CONFIG["fiatlux"] = None - # Theorycovmat flags and defaults - N3FIT_FIXED_CONFIG["theory_covmat_flag"] = False - N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = False - N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = False - if (thconfig := file_content.get("theorycovmatconfig")) is not None: - N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = thconfig.get( - "use_thcovmat_in_fitting", True - ) - N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = thconfig.get( - "use_thcovmat_in_sampling", True - ) - if ( - N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] - or N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] - ): - N3FIT_FIXED_CONFIG["theory_covmat_flag"] = True - N3FIT_FIXED_CONFIG["use_user_uncertainties"] = thconfig.get( - "use_user_uncertainties", False - ) - N3FIT_FIXED_CONFIG["use_scalevar_uncertainties"] = thconfig.get( - "use_scalevar_uncertainties", True - ) - # Sampling flags - if (sam_t0 := file_content.get("sampling")) is not None: - N3FIT_FIXED_CONFIG["separate_multiplicative"] = sam_t0.get( - "separate_multiplicative", True - ) - # Fitting flag + N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) + + if (thconfig:=file_content.get('fiatlux')) is not None: + N3FIT_FIXED_CONFIG['fiatlux']=thconfig + else : + N3FIT_FIXED_CONFIG['fiatlux']=None + #Theorycovmat flags and defaults + N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False + N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = False + N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = False + if (thconfig:=file_content.get('theorycovmatconfig')) is not None: + N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = thconfig.get('use_thcovmat_in_fitting', True) + N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = thconfig.get('use_thcovmat_in_sampling', True) + if N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] or N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting']: + N3FIT_FIXED_CONFIG['theory_covmat_flag'] = True + N3FIT_FIXED_CONFIG['use_user_uncertainties'] = thconfig.get('use_user_uncertainties', False) + N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = thconfig.get('use_scalevar_uncertainties', True) + #Sampling flags + if (sam_t0:=file_content.get('sampling')) is not None: + N3FIT_FIXED_CONFIG['separate_multiplicative'] = sam_t0.get('separate_multiplicative', True) + #Fitting flag file_content.update(N3FIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -198,13 +183,12 @@ def parse_fakedata(self, fakedata: bool): """ if fakedata: log.warning("using filtered closure data") - if not (self.environment.output_path / "filter").is_dir(): + if not (self.environment.output_path/'filter').is_dir(): raise ConfigError( "Could not find filter result at " f"{self.environment.output_path/'filter'} " "to load commondata from. Did you run filter on the " - "runcard?" - ) + "runcard?") return fakedata def produce_use_fitcommondata(self, fakedata): @@ -214,7 +198,8 @@ def produce_use_fitcommondata(self, fakedata): return fakedata def produce_kfold_parameters(self, kfold=None, hyperopt=None): - """Return None even if there are kfolds in the runcard if the hyperopt flag is not active""" + """Return None even if there are kfolds in the runcard if the hyperopt flag is not active + """ if hyperopt is not None: return kfold return None @@ -252,27 +237,18 @@ def __init__(self): @property def argparser(self): parser = super().argparser - parser.add_argument( - "-o", "--output", help="Output folder and name of the fit", default=None - ) + parser.add_argument("-o", "--output", help="Output folder and name of the fit", default=None) def check_positive(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError( - "%s is an invalid positive int value." % value - ) + raise argparse.ArgumentTypeError("%s is an invalid positive int value." % value) return ivalue - parser.add_argument( - "--hyperopt", help="Enable hyperopt scan", default=None, type=int - ) + parser.add_argument("--hyperopt", help="Enable hyperopt scan", default=None, type=int) parser.add_argument("replica", help="MC replica number", type=check_positive) parser.add_argument( - "-r", - "--replica_range", - help="End of the range of replicas to compute", - type=check_positive, + "-r", "--replica_range", help="End of the range of replicas to compute", type=check_positive ) return parser @@ -284,9 +260,7 @@ def get_commandline_arguments(self, cmdline=None): def run(self): try: - self.environment.config_yml = pathlib.Path( - self.args["config_yml"] - ).absolute() + self.environment.config_yml = pathlib.Path(self.args["config_yml"]).absolute() replica = self.args["replica"] if self.args["replica_range"]: replicas = list(range(replica, self.args["replica_range"] + 1)) @@ -300,9 +274,7 @@ def run(self): sys.exit(1) except Exception as e: log.critical(f"Bug in n3fit ocurred. Please report it.") - print( - colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr - ) + print(colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr) sys.exit(1) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 3762d33a6b..c06fbc61e6 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -74,9 +74,7 @@ def xfxQ(self, x, Q, n, fl): """Return the value of the PDF member for the given value in x""" if Q != self._fitting_q: log.warning( - "Querying N3LHAPDFSet at a value of Q=%f different from %f", - Q, - self._fitting_q, + "Querying N3LHAPDFSet at a value of Q=%f different from %f", Q, self._fitting_q ) return self.grid_values([fl], [x]).squeeze()[n] @@ -85,7 +83,7 @@ def _register_photon(self, xgrid): for m in self._lhapdf_set: pl = m.get_layer_re("add_photon") # if pl is an empy list there's no photon - if not pl: + if not pl : continue # TODO : not sure this will work in parallel mode pl[0].register_photon(xgrid) @@ -123,9 +121,7 @@ def __call__(self, xarr, flavours=None, replica=None): if replica is None or replica == 0: # We need generate output values for all replicas - result = np.concatenate( - [m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0 - ) + result = np.concatenate([m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0) if replica == 0: # We want _only_ the central value result = np.mean(result, axis=0, keepdims=True) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 99559d4cd2..b69414d2ee 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -21,10 +21,8 @@ log = logging.getLogger(__name__) - class Photon: """Photon class computing the photon array with the LuxQED approach.""" - def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard @@ -56,28 +54,23 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) - ff = open(f"fiatlux_runcard_{id}.yml", "w+") + ff = open(f'fiatlux_runcard_{id}.yml', 'w+') yaml.dump(self.fiatlux_runcard, ff) - self.lux[id] = fiatlux.FiatLux(f"fiatlux_runcard_{id}.yml") - remove(f"fiatlux_runcard_{id}.yml") + self.lux[id] = fiatlux.FiatLux(f'fiatlux_runcard_{id}.yml') + remove(f'fiatlux_runcard_{id}.yml') # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick # we print different runcards for every replica so they do not interfere with each other - for id in replicas_id: + for id in replicas_id : self.lux[id].PlugAlphaQED(self.alpha_em, self.qref) - self.lux[id].InsertInelasticSplitQ( - [ - self.thresh_b, - self.thresh_t if self.theory["MaxNfPdf"] == 6 else 1e100, - ] - ) + self.lux[id].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) self.lux[id].PlugStructureFunctions(f2[id].fxq, fl[id].fxq, f2lo[id].fxq) - + self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() self.produce_interpolators() - + def alpha_em(self, q): r""" Compute the value of alpha_em. @@ -86,22 +79,27 @@ def alpha_em(self, q): ---------- q: float value in which the coupling is computed - + Returns ------- alpha_em: float electromagnetic coupling """ - if q < self.thresh_c: + if q < self.thresh_c : nf = 3 - elif q < self.thresh_b: + elif q < self.thresh_b : nf = 4 - elif q < self.thresh_t: + elif q < self.thresh_t : nf = 5 - else: + else : nf = 6 - return self.alpha_em_nlo(q, self.alpha_thresh[nf], self.thresh[nf], nf) - + return self.alpha_em_nlo( + q, + self.alpha_thresh[nf], + self.thresh[nf], + nf + ) + def alpha_em_nlo(self, q, alpha_ref, qref, nf): """ Compute the alpha_em running for FFS at NLO. @@ -127,7 +125,7 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): alpha_LO = alpha_ref / den alpha_NLO = alpha_LO * (1 - self.b1[nf] * alpha_LO * np.log(den)) return alpha_NLO - + def set_thresholds_alpha_em(self): """ Compute and store the couplings at thresholds to speed up the calling @@ -142,62 +140,49 @@ def set_thresholds_alpha_em(self): self.thresh_c = self.theory["kcThr"] * self.theory["mc"] self.thresh_b = self.theory["kbThr"] * self.theory["mb"] self.thresh_t = self.theory["ktThr"] * self.theory["mt"] - if self.theory["MaxNfAs"] <= 5: + if self.theory["MaxNfAs"] <= 5 : self.thresh_t = np.inf - if self.theory["MaxNfAs"] <= 4: + if self.theory["MaxNfAs"] <= 4 : self.thresh_b = np.inf - if self.theory["MaxNfAs"] <= 3: + if self.theory["MaxNfAs"] <= 3 : self.thresh_c = np.inf thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] # determine nfref if self.qref < self.thresh_c: nfref = 3 - elif self.qref < self.thresh_b: + elif self.qref < self.thresh_b : nfref = 4 - elif self.qref < self.thresh_t: + elif self.qref < self.thresh_t : nfref = 5 - else: + else : nfref = 6 thresh_list.insert(nfref - 3, self.qref) - self.thresh = { - nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1) - } + self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} self.alpha_thresh = {nfref: self.alpha_em_ref} # determine the values of alpha in the threshold points, depending on the value of qref for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): - self.alpha_thresh[nf] = self.alpha_em_nlo( - self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1 - ) - + self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1) + for nf in reversed(range(3, nfref)): - self.alpha_thresh[nf] = self.alpha_em_nlo( - self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1 - ) - + self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1) + def set_betas(self): """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" nl = 3 nc = 3 - eu2 = 4.0 / 9 - ed2 = 1.0 / 9 + eu2 = 4. / 9 + ed2 = 1. / 9 self.beta0 = {} self.b1 = {} - for nf in range(3, 6 + 1): + for nf in range(3, 6+1): nu = nf // 2 nd = nf - nu - self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / ( - 4 * np.pi - ) - self.b1[nf] = ( - -4.0 - * (nl + nc * (nu * eu2**2 + nd * ed2**2)) - / self.beta0[nf] - / (4 * np.pi) ** 2 - ) + self.beta0[nf] = ( -4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2)) ) / (4 * np.pi) + self.b1[nf] = -4.0 * ( nl + nc * (nu * eu2**2 + nd * ed2**2) ) / self.beta0[nf] / (4 * np.pi)**2 def compute_photon_array(self, id): r""" @@ -215,10 +200,9 @@ def compute_photon_array(self, id): """ # Compute photon PDF start_time = time.perf_counter() - # photon_100GeV = np.array( - # [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] - # ) - photon_100GeV = np.exp(-self.xgrid) + photon_100GeV = np.array( + [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] + ) photon_100GeV += self.generate_errors(id) photon_100GeV /= self.xgrid log.info(f"Time to compute photon: {time.perf_counter() - start_time}") @@ -226,7 +210,7 @@ def compute_photon_array(self, id): # Load eko and reshape it with EKO.read(self.path_to_eko_photon) as eko: - # If we make sure that the grid of the precomputed EKO is the same of + # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape # TODO : move the reshape inside vp-setupfit # xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) @@ -234,15 +218,18 @@ def compute_photon_array(self, id): # construct PDFs pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) for j, pid in enumerate(eko.rotations.inputpids): - if pid == 22: + if pid == 22 : pdfs[j] = photon_100GeV ph_id = j if pid not in self.qcd_pdfs.flavors: continue pdfs[j] = np.array( - [self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x for x in self.xgrid] + [ + self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x + for x in self.xgrid + ] ) - + # Apply EKO to PDFs q2 = eko.mu2grid[0] with eko.operator(q2) as elem: @@ -253,15 +240,14 @@ def compute_photon_array(self, id): # we want x * gamma(x) return self.xgrid * photon_Q0 - + def produce_interpolators(self): """Produce the interpolation functions to be called in compute.""" self.photons_array = [self.compute_photon_array(id) for id in self.replicas_id] self.interpolator = [ - interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") - for photon_array in self.photons_array + interp1d(self.xgrid, photon_array, fill_value=0., kind='cubic') for photon_array in self.photons_array ] - + def compute(self, xgrid): """ Compute the photon interpolating the values of self.photon_array. @@ -270,27 +256,24 @@ def compute(self, xgrid): ---------- xgrid : nd.array array of x values with shape (1,xgrid,1) - + Returns ------- photon values : nd.array array of photon values with shape (1,xgrid,1) """ return [ - self.interpolator[id](xgrid[0, :, 0])[np.newaxis, :, np.newaxis] + self.interpolator[id](xgrid[0,:,0])[np.newaxis,:,np.newaxis] for id in range(len(self.replicas_id)) ] - + def integrate(self): """Compute the integral of the photon on the x range.""" - return [ - trapezoid(self.photons_array[id], self.xgrid) - for id in range(len(self.replicas_id)) - ] - + return [trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id))] + def generate_error_matrix(self): """generate error matrix to be used for the additional errors.""" - if not self.fiatlux_runcard["additional_errors"]: + if not self.fiatlux_runcard["additional_errors"] : return None extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") qs = [self.q_in]*len(self.xgrid) @@ -304,15 +287,14 @@ def generate_error_matrix(self): def generate_errors(self, replica_id): """generate LUX additional errors.""" - if self.error_matrix is None: + if self.error_matrix is None : return np.zeros_like(self.xgrid) seed = replica_luxseed(replica_id, self.fiatlux_runcard["luxseed"]) rng = np.random.default_rng(seed=seed) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) errors = u @ (s * rng.normal(size=7)) return errors - - + # TODO : move it in n3fit_data.py with the others replica_seed generators? # or use directly replica_nnseed? def replica_luxseed(replica, luxseed): diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 4745fefd7f..3b851c2af3 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -2,25 +2,22 @@ import numpy as np from scipy.interpolate import RectBivariateSpline - -class StructureFunction: +class StructureFunction : """Abstract class for the DIS structure functions""" - - def fxq(self, x, Q): + def fxq(self, x, Q) : pass - class InterpStructureFunction(StructureFunction): """ Compute an interpolated structure function convoluting an FKtable with a PDF. """ - def __init__(self, path_to_fktable, pdfs): self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs self.pdgid = int(pdfs.set().get_entry("Particle")) self.produce_interpolator() + def produce_interpolator(self): """Produce the interpolation function to be called in fxq.""" @@ -36,7 +33,7 @@ def produce_interpolator(self): grid2D = predictions.reshape(len(x), len(q2)) self.interpolator = RectBivariateSpline(x, q2, grid2D) - + def fxq(self, x, q): r""" Compute the DIS structure function interpolating the grid. @@ -47,7 +44,7 @@ def fxq(self, x, q): Bjorken's variable Q : float DIS hard scale - + Returns ------- F_{2,L}: float @@ -59,25 +56,23 @@ def fxq(self, x, q): # return 0. return self.interpolator(x, q**2)[0, 0] - -class F2LO(StructureFunction): +class F2LO(StructureFunction) : """Compute analytically the leading order structure function for F2.""" - def __init__(self, pdfs, theory): self.pdfs = pdfs self.thresh_c = theory["kcThr"] * theory["mc"] self.thresh_b = theory["kbThr"] * theory["mb"] self.thresh_t = theory["ktThr"] * theory["mt"] - if theory["MaxNfPdf"] <= 5: + if theory["MaxNfPdf"] <= 5 : self.thresh_t = np.inf - if theory["MaxNfPdf"] <= 4: + if theory["MaxNfPdf"] <= 4 : self.thresh_b = np.inf - if theory["MaxNfPdf"] <= 3: + if theory["MaxNfPdf"] <= 3 : self.thresh_c = np.inf - eu2 = 4.0 / 9 - ed2 = 1.0 / 9 - self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t - + eu2 = 4. / 9 + ed2 = 1. / 9 + self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t + def fxq(self, x, q): r""" Compute the analytical form of F2LO. @@ -88,23 +83,23 @@ def fxq(self, x, q): Bjorken's variable Q : float DIS hard scale - + Returns ------- F2_LO : float Structure function F2 at LO """ # at LO we use ZM-VFS - if q < self.thresh_c: + if q < self.thresh_c : nf = 3 - elif q < self.thresh_b: + elif q < self.thresh_b : nf = 4 - elif q < self.thresh_t: + elif q < self.thresh_t : nf = 5 - else: + else : nf = 6 res = 0 pdfs_values = self.pdfs.xfxQ(x, q) - for i in range(1, nf + 1): - res += self.eq2[i - 1] * (pdfs_values[i] + pdfs_values[-i]) + for i in range(1, nf+1): + res += self.eq2[i-1] * (pdfs_values[i] + pdfs_values[-i]) return res From 2f91a7b952729e5d4212701b4667cd5c8d5cd68f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 14:57:14 +0200 Subject: [PATCH 120/204] Add control on number of replicas generated --- n3fit/src/n3fit/model_gen.py | 2 +- validphys2/src/validphys/photon/compute.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 5d2dfc4e27..d8c60f228c 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -652,7 +652,7 @@ def layer_pdf(x): # Photon layer, changes the photon from zero to non-zero def apply_photon(x): - # if photon is None then the pothon_layer is not applied + # if photon is None then the photon layer is not applied if photons is None: return normalized_pdf(x) else: diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index b69414d2ee..64355dccda 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -4,6 +4,7 @@ from scipy.integrate import trapezoid from os import remove import time +import sys import fiatlux import yaml @@ -40,6 +41,10 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # structure functions self.qcd_pdfs = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") + if max(self.replicas_id) > self.qcd_pdfs.n_members - 1 : + log.error(f"Cannot generate a replica with id larger than the number of replicas of " + self.fiatlux_runcard["pdf_name"]) + sys.exit(1) + # TODO : maybe find a different name for fiatlux_dis_F2 path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" path_to_FL = theoryid.path / "fastkernel/fiatlux_dis_FL.pineappl.lz4" From 42a6ab87001842e5b0cf52aca22d7e3847acbfb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 15:24:26 +0200 Subject: [PATCH 121/204] Move previous control in n3fit_exec --- n3fit/src/n3fit/scripts/n3fit_exec.py | 10 ++++++++++ validphys2/src/validphys/photon/compute.py | 4 ---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 7ec49e0e00..d56c24d1f1 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -150,6 +150,16 @@ def from_yaml(cls, o, *args, **kwargs): if (thconfig:=file_content.get('fiatlux')) is not None: N3FIT_FIXED_CONFIG['fiatlux']=thconfig + from validphys.lhapdfset import LHAPDFSet + pdfs_fiatlux = LHAPDFSet(thconfig["pdf_name"], "replicas") + max_id = max(kwargs["environment"].replicas) + pdfs_ids = pdfs_fiatlux.n_members - 1 + if max_id > pdfs_ids : + log.error( + f"Cannot generate a replica with id larger than the number of replicas of the fiatlux PDFs set " + + thconfig["pdf_name"] + f": replica id={max_id}, replicas of " + thconfig["pdf_name"] + f"={pdfs_ids}" + ) + sys.exit(1) else : N3FIT_FIXED_CONFIG['fiatlux']=None #Theorycovmat flags and defaults diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 64355dccda..203f0dbcbe 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -41,10 +41,6 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # structure functions self.qcd_pdfs = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") - if max(self.replicas_id) > self.qcd_pdfs.n_members - 1 : - log.error(f"Cannot generate a replica with id larger than the number of replicas of " + self.fiatlux_runcard["pdf_name"]) - sys.exit(1) - # TODO : maybe find a different name for fiatlux_dis_F2 path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" path_to_FL = theoryid.path / "fastkernel/fiatlux_dis_FL.pineappl.lz4" From fe47937cd56d32b92180e65a92c47b2da6ebcf69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 3 Apr 2023 15:30:50 +0200 Subject: [PATCH 122/204] Run black on the two files of photon module --- validphys2/src/validphys/photon/compute.py | 131 ++++++++++-------- .../validphys/photon/structure_functions.py | 45 +++--- 2 files changed, 99 insertions(+), 77 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 203f0dbcbe..8451bf7e50 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -9,6 +9,7 @@ import fiatlux import yaml from eko.io import EKO + # from eko.io.manipulate import xgrid_reshape # from eko.interpolation import XGrid @@ -19,11 +20,12 @@ from n3fit.io.writer import XGRID - log = logging.getLogger(__name__) + class Photon: """Photon class computing the photon array with the LuxQED approach.""" + def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard @@ -55,23 +57,28 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) - ff = open(f'fiatlux_runcard_{id}.yml', 'w+') + ff = open(f"fiatlux_runcard_{id}.yml", "w+") yaml.dump(self.fiatlux_runcard, ff) - self.lux[id] = fiatlux.FiatLux(f'fiatlux_runcard_{id}.yml') - remove(f'fiatlux_runcard_{id}.yml') + self.lux[id] = fiatlux.FiatLux(f"fiatlux_runcard_{id}.yml") + remove(f"fiatlux_runcard_{id}.yml") # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick # we print different runcards for every replica so they do not interfere with each other - for id in replicas_id : + for id in replicas_id: self.lux[id].PlugAlphaQED(self.alpha_em, self.qref) - self.lux[id].InsertInelasticSplitQ([self.thresh_b, self.thresh_t if self.theory["MaxNfPdf"]==6 else 1e100]) + self.lux[id].InsertInelasticSplitQ( + [ + self.thresh_b, + self.thresh_t if self.theory["MaxNfPdf"] == 6 else 1e100, + ] + ) self.lux[id].PlugStructureFunctions(f2[id].fxq, fl[id].fxq, f2lo[id].fxq) - + self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() self.produce_interpolators() - + def alpha_em(self, q): r""" Compute the value of alpha_em. @@ -80,27 +87,22 @@ def alpha_em(self, q): ---------- q: float value in which the coupling is computed - + Returns ------- alpha_em: float electromagnetic coupling """ - if q < self.thresh_c : + if q < self.thresh_c: nf = 3 - elif q < self.thresh_b : + elif q < self.thresh_b: nf = 4 - elif q < self.thresh_t : + elif q < self.thresh_t: nf = 5 - else : + else: nf = 6 - return self.alpha_em_nlo( - q, - self.alpha_thresh[nf], - self.thresh[nf], - nf - ) - + return self.alpha_em_nlo(q, self.alpha_thresh[nf], self.thresh[nf], nf) + def alpha_em_nlo(self, q, alpha_ref, qref, nf): """ Compute the alpha_em running for FFS at NLO. @@ -126,7 +128,7 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): alpha_LO = alpha_ref / den alpha_NLO = alpha_LO * (1 - self.b1[nf] * alpha_LO * np.log(den)) return alpha_NLO - + def set_thresholds_alpha_em(self): """ Compute and store the couplings at thresholds to speed up the calling @@ -141,49 +143,62 @@ def set_thresholds_alpha_em(self): self.thresh_c = self.theory["kcThr"] * self.theory["mc"] self.thresh_b = self.theory["kbThr"] * self.theory["mb"] self.thresh_t = self.theory["ktThr"] * self.theory["mt"] - if self.theory["MaxNfAs"] <= 5 : + if self.theory["MaxNfAs"] <= 5: self.thresh_t = np.inf - if self.theory["MaxNfAs"] <= 4 : + if self.theory["MaxNfAs"] <= 4: self.thresh_b = np.inf - if self.theory["MaxNfAs"] <= 3 : + if self.theory["MaxNfAs"] <= 3: self.thresh_c = np.inf thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] # determine nfref if self.qref < self.thresh_c: nfref = 3 - elif self.qref < self.thresh_b : + elif self.qref < self.thresh_b: nfref = 4 - elif self.qref < self.thresh_t : + elif self.qref < self.thresh_t: nfref = 5 - else : + else: nfref = 6 thresh_list.insert(nfref - 3, self.qref) - self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} + self.thresh = { + nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1) + } self.alpha_thresh = {nfref: self.alpha_em_ref} # determine the values of alpha in the threshold points, depending on the value of qref for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): - self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1) - + self.alpha_thresh[nf] = self.alpha_em_nlo( + self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1 + ) + for nf in reversed(range(3, nfref)): - self.alpha_thresh[nf] = self.alpha_em_nlo(self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1) - + self.alpha_thresh[nf] = self.alpha_em_nlo( + self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1 + ) + def set_betas(self): """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" nl = 3 nc = 3 - eu2 = 4. / 9 - ed2 = 1. / 9 + eu2 = 4.0 / 9 + ed2 = 1.0 / 9 self.beta0 = {} self.b1 = {} - for nf in range(3, 6+1): + for nf in range(3, 6 + 1): nu = nf // 2 nd = nf - nu - self.beta0[nf] = ( -4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2)) ) / (4 * np.pi) - self.b1[nf] = -4.0 * ( nl + nc * (nu * eu2**2 + nd * ed2**2) ) / self.beta0[nf] / (4 * np.pi)**2 + self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / ( + 4 * np.pi + ) + self.b1[nf] = ( + -4.0 + * (nl + nc * (nu * eu2**2 + nd * ed2**2)) + / self.beta0[nf] + / (4 * np.pi) ** 2 + ) def compute_photon_array(self, id): r""" @@ -211,7 +226,7 @@ def compute_photon_array(self, id): # Load eko and reshape it with EKO.read(self.path_to_eko_photon) as eko: - # If we make sure that the grid of the precomputed EKO is the same of + # If we make sure that the grid of the precomputed EKO is the same of # self.xgrid then we don't need to reshape # TODO : move the reshape inside vp-setupfit # xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) @@ -219,18 +234,15 @@ def compute_photon_array(self, id): # construct PDFs pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) for j, pid in enumerate(eko.rotations.inputpids): - if pid == 22 : + if pid == 22: pdfs[j] = photon_100GeV ph_id = j if pid not in self.qcd_pdfs.flavors: continue pdfs[j] = np.array( - [ - self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x - for x in self.xgrid - ] + [self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x for x in self.xgrid] ) - + # Apply EKO to PDFs q2 = eko.mu2grid[0] with eko.operator(q2) as elem: @@ -241,14 +253,15 @@ def compute_photon_array(self, id): # we want x * gamma(x) return self.xgrid * photon_Q0 - + def produce_interpolators(self): """Produce the interpolation functions to be called in compute.""" self.photons_array = [self.compute_photon_array(id) for id in self.replicas_id] self.interpolator = [ - interp1d(self.xgrid, photon_array, fill_value=0., kind='cubic') for photon_array in self.photons_array + interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") + for photon_array in self.photons_array ] - + def compute(self, xgrid): """ Compute the photon interpolating the values of self.photon_array. @@ -257,30 +270,33 @@ def compute(self, xgrid): ---------- xgrid : nd.array array of x values with shape (1,xgrid,1) - + Returns ------- photon values : nd.array array of photon values with shape (1,xgrid,1) """ return [ - self.interpolator[id](xgrid[0,:,0])[np.newaxis,:,np.newaxis] + self.interpolator[id](xgrid[0, :, 0])[np.newaxis, :, np.newaxis] for id in range(len(self.replicas_id)) ] - + def integrate(self): """Compute the integral of the photon on the x range.""" - return [trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id))] - + return [ + trapezoid(self.photons_array[id], self.xgrid) + for id in range(len(self.replicas_id)) + ] + def generate_error_matrix(self): """generate error matrix to be used for the additional errors.""" - if not self.fiatlux_runcard["additional_errors"] : + if not self.fiatlux_runcard["additional_errors"]: return None extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") - qs = [self.q_in]*len(self.xgrid) + qs = [self.q_in] * len(self.xgrid) res_central = np.array(extra_set.central_member.xfxQ(22, self.xgrid, qs)) res = [] - for idx_member in range(101, 107+1): + for idx_member in range(101, 107 + 1): tmp = np.array(extra_set.members[idx_member].xfxQ(22, self.xgrid, qs)) res.append(tmp - res_central) # first index must be x, while second one must be replica index @@ -288,14 +304,15 @@ def generate_error_matrix(self): def generate_errors(self, replica_id): """generate LUX additional errors.""" - if self.error_matrix is None : + if self.error_matrix is None: return np.zeros_like(self.xgrid) seed = replica_luxseed(replica_id, self.fiatlux_runcard["luxseed"]) rng = np.random.default_rng(seed=seed) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) errors = u @ (s * rng.normal(size=7)) return errors - + + # TODO : move it in n3fit_data.py with the others replica_seed generators? # or use directly replica_nnseed? def replica_luxseed(replica, luxseed): diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 3b851c2af3..4745fefd7f 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -2,22 +2,25 @@ import numpy as np from scipy.interpolate import RectBivariateSpline -class StructureFunction : + +class StructureFunction: """Abstract class for the DIS structure functions""" - def fxq(self, x, Q) : + + def fxq(self, x, Q): pass + class InterpStructureFunction(StructureFunction): """ Compute an interpolated structure function convoluting an FKtable with a PDF. """ + def __init__(self, path_to_fktable, pdfs): self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) self.pdfs = pdfs self.pdgid = int(pdfs.set().get_entry("Particle")) self.produce_interpolator() - def produce_interpolator(self): """Produce the interpolation function to be called in fxq.""" @@ -33,7 +36,7 @@ def produce_interpolator(self): grid2D = predictions.reshape(len(x), len(q2)) self.interpolator = RectBivariateSpline(x, q2, grid2D) - + def fxq(self, x, q): r""" Compute the DIS structure function interpolating the grid. @@ -44,7 +47,7 @@ def fxq(self, x, q): Bjorken's variable Q : float DIS hard scale - + Returns ------- F_{2,L}: float @@ -56,23 +59,25 @@ def fxq(self, x, q): # return 0. return self.interpolator(x, q**2)[0, 0] -class F2LO(StructureFunction) : + +class F2LO(StructureFunction): """Compute analytically the leading order structure function for F2.""" + def __init__(self, pdfs, theory): self.pdfs = pdfs self.thresh_c = theory["kcThr"] * theory["mc"] self.thresh_b = theory["kbThr"] * theory["mb"] self.thresh_t = theory["ktThr"] * theory["mt"] - if theory["MaxNfPdf"] <= 5 : + if theory["MaxNfPdf"] <= 5: self.thresh_t = np.inf - if theory["MaxNfPdf"] <= 4 : + if theory["MaxNfPdf"] <= 4: self.thresh_b = np.inf - if theory["MaxNfPdf"] <= 3 : + if theory["MaxNfPdf"] <= 3: self.thresh_c = np.inf - eu2 = 4. / 9 - ed2 = 1. / 9 - self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t - + eu2 = 4.0 / 9 + ed2 = 1.0 / 9 + self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t + def fxq(self, x, q): r""" Compute the analytical form of F2LO. @@ -83,23 +88,23 @@ def fxq(self, x, q): Bjorken's variable Q : float DIS hard scale - + Returns ------- F2_LO : float Structure function F2 at LO """ # at LO we use ZM-VFS - if q < self.thresh_c : + if q < self.thresh_c: nf = 3 - elif q < self.thresh_b : + elif q < self.thresh_b: nf = 4 - elif q < self.thresh_t : + elif q < self.thresh_t: nf = 5 - else : + else: nf = 6 res = 0 pdfs_values = self.pdfs.xfxQ(x, q) - for i in range(1, nf+1): - res += self.eq2[i-1] * (pdfs_values[i] + pdfs_values[-i]) + for i in range(1, nf + 1): + res += self.eq2[i - 1] * (pdfs_values[i] + pdfs_values[-i]) return res From 1e53533677b2c4de21baed12ff440d3b60518508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:02:38 +0200 Subject: [PATCH 123/204] Update n3fit/runcards/examples/Basic_runcard_qed.yml Co-authored-by: Roy Stegeman --- n3fit/runcards/examples/Basic_runcard_qed.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index 80a6e8a3ab..a4b59f413f 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -201,4 +201,5 @@ fiatlux: verbose: off pdf_name: NNPDF40_nnlo_as_01180_1000 additional_errors: true # should be set to true only for the last iteration - luxseed: 1234567890 \ No newline at end of file + luxseed: 1234567890 + \ No newline at end of file From 00dbee5d6339d5b5e36c38604e5a5f5ad18dfe44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:02:59 +0200 Subject: [PATCH 124/204] Update validphys2/src/validphys/tests/photon/test_structurefunctions.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/tests/photon/test_structurefunctions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index 57b51ea170..829f154219 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -120,4 +120,5 @@ def test_F2(monkeypatch): for Q in np.geomspace(10, 1000000, 10): np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) + \ No newline at end of file From 714366e0f040879533216c1434d93f6b45731278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:53:21 +0200 Subject: [PATCH 125/204] Update validphys2/src/validphys/photon/compute.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/photon/compute.py | 1 - 1 file changed, 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 8451bf7e50..df5b1e6454 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -4,7 +4,6 @@ from scipy.integrate import trapezoid from os import remove import time -import sys import fiatlux import yaml From ecc3cdb353fd07fadd51fe177e3f9e4f265bca7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:55:02 +0200 Subject: [PATCH 126/204] Update validphys2/src/validphys/photon/compute.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/photon/compute.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index df5b1e6454..097080fcdb 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -9,9 +9,6 @@ import yaml from eko.io import EKO -# from eko.io.manipulate import xgrid_reshape -# from eko.interpolation import XGrid - from validphys.lhapdfset import LHAPDFSet from validphys.n3fit_data import replica_nnseed from . import structure_functions as sf From f1c6a7f36cd099e605fa33fbd7eb86e129c1508e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 4 Apr 2023 10:57:55 +0200 Subject: [PATCH 127/204] Rerun black --- n3fit/src/n3fit/model_trainer.py | 2 -- n3fit/src/n3fit/tests/test_layers.py | 4 ++-- validphys2/src/validphys/photon/compute.py | 11 +++-------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 5411c55aba..1537b06b24 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -878,7 +878,6 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() - # Initialize all photon classes for the different replicas: if self.fiatlux_runcard is not None: photons=Photon( @@ -888,7 +887,6 @@ def hyperparametrizable(self, params): ) else: photons=None - ### Training loop for k, partition in enumerate(self.kpartitions): # Each partition of the kfolding needs to have its own separate model diff --git a/n3fit/src/n3fit/tests/test_layers.py b/n3fit/src/n3fit/tests/test_layers.py index a42203d34f..c3a78f2dbb 100644 --- a/n3fit/src/n3fit/tests/test_layers.py +++ b/n3fit/src/n3fit/tests/test_layers.py @@ -268,12 +268,12 @@ def test_addphoton_init(): np.testing.assert_equal(addphoton._photons_generator, 1234) np.testing.assert_equal(addphoton._pdf_ph, None) -class fakePhoton(): +class FakePhoton(): def compute(self, xgrid): return [np.exp(-xgrid)] def test_compute_photon(): - photon = fakePhoton() + photon = FakePhoton() addphoton = layers.AddPhoton(photons=photon) xgrid = np.geomspace(1e-4, 1., 10) addphoton.register_photon(xgrid) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 8451bf7e50..f97e7b4095 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -162,9 +162,7 @@ def set_thresholds_alpha_em(self): nfref = 6 thresh_list.insert(nfref - 3, self.qref) - self.thresh = { - nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1) - } + self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} self.alpha_thresh = {nfref: self.alpha_em_ref} @@ -190,9 +188,7 @@ def set_betas(self): for nf in range(3, 6 + 1): nu = nf // 2 nd = nf - nu - self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / ( - 4 * np.pi - ) + self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / (4 * np.pi) self.b1[nf] = ( -4.0 * (nl + nc * (nu * eu2**2 + nd * ed2**2)) @@ -284,8 +280,7 @@ def compute(self, xgrid): def integrate(self): """Compute the integral of the photon on the x range.""" return [ - trapezoid(self.photons_array[id], self.xgrid) - for id in range(len(self.replicas_id)) + trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id)) ] def generate_error_matrix(self): From 86bc19bb3a945038343874f8ae467250b78e780f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:59:10 +0200 Subject: [PATCH 128/204] Update n3fit/src/n3fit/model_trainer.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/model_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 1537b06b24..53a855b421 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -908,7 +908,7 @@ def hyperparametrizable(self, params): photons, ) - if photons is not None: + if photons: for m in pdf_models: pl = m.get_layer("add_photon") pl.register_photon(xinput.input.tensor_content) From 829f45bfd7b7e3e33996cb1f14d0ac04bd4591a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:59:28 +0200 Subject: [PATCH 129/204] Update n3fit/src/n3fit/scripts/n3fit_exec.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/scripts/n3fit_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index d56c24d1f1..2c34f9a52e 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -148,7 +148,7 @@ def from_yaml(cls, o, *args, **kwargs): N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) - if (thconfig:=file_content.get('fiatlux')) is not None: + if (thconfig:=file_content.get('fiatlux')): N3FIT_FIXED_CONFIG['fiatlux']=thconfig from validphys.lhapdfset import LHAPDFSet pdfs_fiatlux = LHAPDFSet(thconfig["pdf_name"], "replicas") From 05e81cecc2259b8b5b95765fb359be03288862e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 4 Apr 2023 11:01:31 +0200 Subject: [PATCH 130/204] Add log.info --- validphys2/src/validphys/photon/compute.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index fef7a19bd9..a320d2990c 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -213,7 +213,8 @@ def compute_photon_array(self, id): ) photon_100GeV += self.generate_errors(id) photon_100GeV /= self.xgrid - log.info(f"Time to compute photon: {time.perf_counter() - start_time}") + log.info(f"Computing photon") + log.info(f"Computation time: {time.perf_counter() - start_time}") # TODO : the different x points could be even computed in parallel # Load eko and reshape it From 1adff94e6a5eec390006b3c8a90e58c02b45d0b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 4 Apr 2023 11:03:56 +0200 Subject: [PATCH 131/204] Update validphys2/src/validphys/photon/compute.py Co-authored-by: Felix Hekhorn --- validphys2/src/validphys/photon/compute.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index a320d2990c..d94f929224 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -53,8 +53,8 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) - ff = open(f"fiatlux_runcard_{id}.yml", "w+") - yaml.dump(self.fiatlux_runcard, ff) + with open(f"fiatlux_runcard_{id}.yml", "w+") as ff: + yaml.dump(self.fiatlux_runcard, ff) self.lux[id] = fiatlux.FiatLux(f"fiatlux_runcard_{id}.yml") remove(f"fiatlux_runcard_{id}.yml") # we have a dict but fiatlux wants a yaml file From dd924e42ce16d312598dc738f85b2ac7824a30dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 4 Apr 2023 11:55:59 +0200 Subject: [PATCH 132/204] Update validphys2/src/validphys/photon/compute.py Co-authored-by: Felix Hekhorn --- validphys2/src/validphys/photon/compute.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index d94f929224..d5f3da708f 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -53,10 +53,11 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) - with open(f"fiatlux_runcard_{id}.yml", "w+") as ff: + tmp_file = f"fiatlux_runcard_{id}.yml" + with open(tmp_file, "w+") as ff: yaml.dump(self.fiatlux_runcard, ff) - self.lux[id] = fiatlux.FiatLux(f"fiatlux_runcard_{id}.yml") - remove(f"fiatlux_runcard_{id}.yml") + self.lux[id] = fiatlux.FiatLux(tmp_file) + remove(tmp_file) # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick # we print different runcards for every replica so they do not interfere with each other From dd6e76bf0e242850b55e29704e92f1e2120a26fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 4 Apr 2023 16:04:17 +0200 Subject: [PATCH 133/204] Implement replica id control in n3fit.checks --- n3fit/src/n3fit/checks.py | 18 ++++++++++++++++++ n3fit/src/n3fit/model_trainer.py | 6 ++++++ n3fit/src/n3fit/performfit.py | 3 +++ n3fit/src/n3fit/scripts/n3fit_exec.py | 10 ---------- n3fit/src/n3fit/vpinterface.py | 1 - validphys2/src/validphys/photon/compute.py | 19 ++++++++++--------- .../validphys/photon/structure_functions.py | 2 +- 7 files changed, 38 insertions(+), 21 deletions(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index b651f512c5..f80e7da6b3 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -393,3 +393,21 @@ def check_deprecated_options(fitting): for option in nnfit_options: if option in fitting: log.warning("'fitting::%s' is an nnfit-only key, it will be ignored", option) + +@make_argcheck +def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): + if fiatlux : + import lhapdf + pdf_name = fiatlux["pdf_name"] + pdfs_fiatlux = lhapdf.getPDFSet(pdf_name) + max_id = max(replicas) + pdfs_ids = pdfs_fiatlux.size - 1 + if max_id > pdfs_ids : + for replica_id in replicas: + # At this point it should be always empty + os.rmdir(replica_path / f"replica_{replica_id}") + raise CheckError( + f"Cannot generate a replica with id larger than the number of replicas of the fiatlux PDFs set " + + pdf_name + f": replica id={max_id}, replicas of " + pdf_name + f"={pdfs_ids}" + +"\nRemoving replica output folders" + ) \ No newline at end of file diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 53a855b421..da5b788011 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -135,6 +135,12 @@ def __init__( whether sum rules should be enabled (All, MSR, VSR, False) parallel_models: int number of models to fit in parallel + theoryid : validphys.core.TheoryIDSpec + object contining info for generating the photon + fiatlux_runcard : dict + dictionary containing the fiatlux runcard + replicas_id : list + list with the replicas ids to be fitted """ # Save all input information self.exp_info = exp_info diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 3f4c1e7de7..4aba5964c7 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -15,6 +15,7 @@ # Action to be called by validphys # All information defining the NN should come here in the "parameters" dict @n3fit.checks.can_run_multiple_replicas +@n3fit.checks.check_fiatlux_pdfs_id def performfit( *, n3fit_checks_action, # wrapper for all checks @@ -85,6 +86,8 @@ def performfit( Theory which is used to generate theory predictions from model during fit. Object also contains some metadata on the theory settings. + fiatlux : dict + dictionary containing the fiatlux runcard basis: list[dict] preprocessing information for each flavour to be fitted. fitbasis: str diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 2c34f9a52e..a4a67578f7 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -150,16 +150,6 @@ def from_yaml(cls, o, *args, **kwargs): if (thconfig:=file_content.get('fiatlux')): N3FIT_FIXED_CONFIG['fiatlux']=thconfig - from validphys.lhapdfset import LHAPDFSet - pdfs_fiatlux = LHAPDFSet(thconfig["pdf_name"], "replicas") - max_id = max(kwargs["environment"].replicas) - pdfs_ids = pdfs_fiatlux.n_members - 1 - if max_id > pdfs_ids : - log.error( - f"Cannot generate a replica with id larger than the number of replicas of the fiatlux PDFs set " - + thconfig["pdf_name"] + f": replica id={max_id}, replicas of " + thconfig["pdf_name"] + f"={pdfs_ids}" - ) - sys.exit(1) else : N3FIT_FIXED_CONFIG['fiatlux']=None #Theorycovmat flags and defaults diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index c06fbc61e6..6c76f73d87 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -85,7 +85,6 @@ def _register_photon(self, xgrid): # if pl is an empy list there's no photon if not pl : continue - # TODO : not sure this will work in parallel mode pl[0].register_photon(xgrid) # Recompile the model if necessary if not pl[0].built: diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index a320d2990c..37ed0b6564 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -1,20 +1,20 @@ """Script that calls fiatlux to add the photon PDF.""" -import numpy as np -from scipy.interpolate import interp1d -from scipy.integrate import trapezoid -from os import remove +import logging import time +from os import remove import fiatlux +import numpy as np import yaml from eko.io import EKO - +from scipy.integrate import trapezoid +from scipy.interpolate import interp1d from validphys.lhapdfset import LHAPDFSet from validphys.n3fit_data import replica_nnseed -from . import structure_functions as sf -import logging + from n3fit.io.writer import XGRID +from . import structure_functions as sf log = logging.getLogger(__name__) @@ -207,14 +207,14 @@ def compute_photon_array(self, id): photon PDF at the scale 1 GeV """ # Compute photon PDF + log.info(f"Computing photon") start_time = time.perf_counter() photon_100GeV = np.array( [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] ) + log.info(f"Computation time: {time.perf_counter() - start_time}") photon_100GeV += self.generate_errors(id) photon_100GeV /= self.xgrid - log.info(f"Computing photon") - log.info(f"Computation time: {time.perf_counter() - start_time}") # TODO : the different x points could be even computed in parallel # Load eko and reshape it @@ -296,6 +296,7 @@ def generate_error_matrix(self): def generate_errors(self, replica_id): """generate LUX additional errors.""" + log.info(f"Generating additional errors") if self.error_matrix is None: return np.zeros_like(self.xgrid) seed = replica_luxseed(replica_id, self.fiatlux_runcard["luxseed"]) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 4745fefd7f..25ea915169 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -1,5 +1,5 @@ -import pineappl import numpy as np +import pineappl from scipy.interpolate import RectBivariateSpline From 681a1371d9bc198b139c14ee8a36c413046b187f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 4 Apr 2023 16:34:34 +0200 Subject: [PATCH 134/204] Fix check_fiatlux_pdfs_id --- n3fit/src/n3fit/checks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index f80e7da6b3..0deeb08ce7 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -6,6 +6,7 @@ import numbers import numpy as np from reportengine.checks import make_argcheck, CheckError +from validphys.core import PDF from validphys.pdfbases import check_basis from n3fit.hyper_optimization import penalties as penalties_module from n3fit.hyper_optimization import rewards as rewards_module @@ -397,17 +398,15 @@ def check_deprecated_options(fitting): @make_argcheck def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): if fiatlux : - import lhapdf pdf_name = fiatlux["pdf_name"] - pdfs_fiatlux = lhapdf.getPDFSet(pdf_name) + pdfs_ids = PDF(pdf_name).get_members() - 1 # get_members counts also replica0 max_id = max(replicas) - pdfs_ids = pdfs_fiatlux.size - 1 if max_id > pdfs_ids : for replica_id in replicas: # At this point it should be always empty os.rmdir(replica_path / f"replica_{replica_id}") raise CheckError( - f"Cannot generate a replica with id larger than the number of replicas of the fiatlux PDFs set " + f"Cannot generate a photon replica with id larger than the number of replicas of the PDFs set " + pdf_name + f": replica id={max_id}, replicas of " + pdf_name + f"={pdfs_ids}" +"\nRemoving replica output folders" ) \ No newline at end of file From cf778b7a3f9b28e9a9183894d97f437a3b6011a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 4 Apr 2023 16:35:55 +0200 Subject: [PATCH 135/204] Update n3fit/src/n3fit/checks.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 0deeb08ce7..57213be98f 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -409,4 +409,4 @@ def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): f"Cannot generate a photon replica with id larger than the number of replicas of the PDFs set " + pdf_name + f": replica id={max_id}, replicas of " + pdf_name + f"={pdfs_ids}" +"\nRemoving replica output folders" - ) \ No newline at end of file + ) From f42ae772ed89aec4553eabcc00d9d3cf392f38b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 5 Apr 2023 11:38:05 +0200 Subject: [PATCH 136/204] Dump fiatlux_runcard in temporary directory --- validphys2/src/validphys/photon/compute.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index dc7e6f9495..d3577971d0 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -1,7 +1,7 @@ """Script that calls fiatlux to add the photon PDF.""" import logging +import tempfile import time -from os import remove import fiatlux import numpy as np @@ -53,11 +53,11 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) - tmp_file = f"fiatlux_runcard_{id}.yml" - with open(tmp_file, "w+") as ff: - yaml.dump(self.fiatlux_runcard, ff) - self.lux[id] = fiatlux.FiatLux(tmp_file) - remove(tmp_file) + with tempfile.TemporaryDirectory() as tmp: + tmp_file = tmp + f"/fiatlux_runcard_{id}.yml" + with open(tmp_file, "w+") as ff: + yaml.dump(self.fiatlux_runcard, ff) + self.lux[id] = fiatlux.FiatLux(tmp_file) # we have a dict but fiatlux wants a yaml file # TODO : remove this dirty trick # we print different runcards for every replica so they do not interfere with each other @@ -297,7 +297,7 @@ def generate_error_matrix(self): def generate_errors(self, replica_id): """generate LUX additional errors.""" - log.info(f"Generating additional errors") + log.info(f"Generating photon additional errors") if self.error_matrix is None: return np.zeros_like(self.xgrid) seed = replica_luxseed(replica_id, self.fiatlux_runcard["luxseed"]) From f0cf02c96a23785ff00569e33b17f8685b3c4891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 11:40:22 +0200 Subject: [PATCH 137/204] Update n3fit/src/n3fit/model_trainer.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/model_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index da5b788011..8502c079d6 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -686,7 +686,7 @@ def _generate_pdf( dictionary of arguments for the regularizer seed: int seed for the NN - :py:class:`validphys.photon.compute.Photon` + photons: :py:class:`validphys.photon.compute.Photon` function to compute the photon PDF see model_gen.pdfNN_layer_generator for more information From 5ee4f759d609697c5d3d3ecffd95b8d186f4e653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 14:49:49 +0200 Subject: [PATCH 138/204] Update n3fit/src/n3fit/model_trainer.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/model_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 8502c079d6..647562fc8c 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -885,7 +885,7 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() # Initialize all photon classes for the different replicas: - if self.fiatlux_runcard is not None: + if self.fiatlux_runcard: photons=Photon( theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard, From 24c298236485732ca882a5780307495c6459b49e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 5 Apr 2023 16:23:25 +0200 Subject: [PATCH 139/204] Use temporary file --- n3fit/src/evolven3fit_new/evolve.py | 2 ++ validphys2/src/validphys/photon/compute.py | 13 ++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/n3fit/src/evolven3fit_new/evolve.py b/n3fit/src/evolven3fit_new/evolve.py index de8cba242e..f25b8b6cf8 100644 --- a/n3fit/src/evolven3fit_new/evolve.py +++ b/n3fit/src/evolven3fit_new/evolve.py @@ -154,6 +154,8 @@ def evolve_exportgrid(exportgrid, eko, x_grid, qed): eko operator for evolution xgrid: list xgrid to be used as the targetgrid + qed: bool + whether qed is activated or not Returns ------- : np.array diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index d3577971d0..590dd4ce3b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -53,14 +53,13 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) - with tempfile.TemporaryDirectory() as tmp: - tmp_file = tmp + f"/fiatlux_runcard_{id}.yml" - with open(tmp_file, "w+") as ff: - yaml.dump(self.fiatlux_runcard, ff) - self.lux[id] = fiatlux.FiatLux(tmp_file) + with tempfile.NamedTemporaryFile(mode="w") as tmp: + with tmp.file as tmp_file: + tmp_file.write(yaml.dump(self.fiatlux_runcard)) + self.lux[id] = fiatlux.FiatLux(tmp_file.name) # we have a dict but fiatlux wants a yaml file - # TODO : remove this dirty trick - # we print different runcards for every replica so they do not interfere with each other + # TODO : once that fiatlux will allow dictionaries + # pass directly self.fiatlux_runcard for id in replicas_id: self.lux[id].PlugAlphaQED(self.alpha_em, self.qref) self.lux[id].InsertInelasticSplitQ( From e55a1ee9b49735ded934e1f0d29e5704227388da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:36:31 +0200 Subject: [PATCH 140/204] Update n3fit/src/n3fit/model_gen.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/model_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index d8c60f228c..c1358973b0 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -653,7 +653,7 @@ def layer_pdf(x): # Photon layer, changes the photon from zero to non-zero def apply_photon(x): # if photon is None then the photon layer is not applied - if photons is None: + if photons: return normalized_pdf(x) else: return layer_photon(normalized_pdf(x), i) From 2714164970849bb6337e5e50b12bea8c07c8dbfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:37:56 +0200 Subject: [PATCH 141/204] Update validphys2/src/validphys/tests/photon/test_compute.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/tests/photon/test_compute.py | 1 - 1 file changed, 1 deletion(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index b79cfeb56e..363d88ba79 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -21,7 +21,6 @@ def get_description(self): "ktThr": 1., "MaxNfAs": 5, "MaxNfPdf": 5, - } fiatlux_runcard = { From b8cf0adc2cc8dda3222410bfc064b641be66d739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:38:45 +0200 Subject: [PATCH 142/204] Update n3fit/src/n3fit/model_gen.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/model_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index c1358973b0..df858440e5 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -496,7 +496,7 @@ def pdfNN_layer_generator( will be a (1, None, 2) tensor where dim [:,:,0] is scaled parallel_models: int How many models should be trained in parallel - photon: Function + photon: :py:class:`validphys.photon.compute.Photon` If given, gives the ``add_photon`` a function to compute a photon which will be added at the index 0 of the 14-size FK basis This same function will also be used to compute the MSR component for the photon From 9c6b776adfc4a42f1241e5b59ff0b8cd80a70b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:39:11 +0200 Subject: [PATCH 143/204] Update n3fit/src/n3fit/model_gen.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/model_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index df858440e5..f95192a0f1 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -497,7 +497,7 @@ def pdfNN_layer_generator( parallel_models: int How many models should be trained in parallel photon: :py:class:`validphys.photon.compute.Photon` - If given, gives the ``add_photon`` a function to compute a photon which will be added at the + If given, gives the AddPhoton layer a function to compute a photon which will be added at the index 0 of the 14-size FK basis This same function will also be used to compute the MSR component for the photon From 4f2ca5bb5727c54dabeb350a6f797a31a4396825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:39:48 +0200 Subject: [PATCH 144/204] Update n3fit/src/n3fit/layers/rotations.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/layers/rotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 3f1f896e8d..54bdb158ce 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -94,7 +94,7 @@ def call(self, pdf_raw): class AddPhoton(MetaLayer): """ Changes the value of the photon component of the PDF to non-zero. - The photon idx of the PDF is always index 0. + The photon idx in the dimension-14 PDF basis of the FKTables is always index 0. In order to avoid bottlenecks, this layer can only compute the photon for a given fixed shape. From 457fb3677bffc4fcec870ebb310ad3b3ac25f568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:40:05 +0200 Subject: [PATCH 145/204] Update n3fit/src/n3fit/layers/rotations.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/layers/rotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 54bdb158ce..a4e8325679 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -108,7 +108,7 @@ def __init__(self, photons, **kwargs): def register_photon(self, xgrid): """Compute the photon array and set the layer to be rebuilt""" - if self._photons_generator is not None: + if self._photons_generator: self._pdf_ph = self._photons_generator.compute(xgrid) self.built = False From 2b01732d776fae0c90470c4c9963f5334cffee0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:40:21 +0200 Subject: [PATCH 146/204] Update n3fit/src/n3fit/layers/msr_normalization.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/layers/msr_normalization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 97111e83c6..ea6f511c0a 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -53,7 +53,7 @@ def call(self, pdf_integrated, ph_replica): y = op.flatten(pdf_integrated) norm_constants = [] - if self._photons is not None: + if self._photons: photon_integral = self._photons[ph_replica] else : photon_integral = 0. From d06513e32ec3245955ec30310541b87d9ce953b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:40:43 +0200 Subject: [PATCH 147/204] Update n3fit/src/n3fit/msr.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/msr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 6325d4afbb..239ef55f7e 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -72,7 +72,7 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): # 3.1 If a photon is given, compute the photon component of the MSR photons_c = None - if photons is not None: + if photons: photons_c = photons.integrate() # 4. Now create the normalization by selecting the right integrations From a5aa945a0893da8cfbb030221e80a3184eb83ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 5 Apr 2023 17:45:06 +0200 Subject: [PATCH 148/204] Update validphys2/src/validphys/tests/photon/test_compute.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/tests/photon/test_compute.py | 1 - 1 file changed, 1 deletion(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 363d88ba79..115286af49 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -3,7 +3,6 @@ import numpy as np from collections import namedtuple from pathlib import Path -from validphys.lhapdfset import LHAPDFSet import fiatlux class faketheory(): From 0b2e2f4b86fc668976d4f5d978bea1933b6f69c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 5 Apr 2023 17:47:10 +0200 Subject: [PATCH 149/204] Fix docstring --- n3fit/src/n3fit/model_gen.py | 2 +- n3fit/src/n3fit/msr.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index f95192a0f1..fb42b85772 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -646,7 +646,7 @@ def layer_fitbasis(x): # Rotation layer, changes from the 8-basis to the 14-basis def layer_pdf(x): return layer_evln(layer_fitbasis(x)) - + # Final PDF (apply normalization) normalized_pdf = sumrule_layer(layer_pdf, i) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 239ef55f7e..f924e66982 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -55,7 +55,9 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): what sum rules to compute (MSR, VSR or All), default: All scaler: scaler Function to apply to the input. If given the input to the model - will be a (1, None, 2) tensor where dim [:,:,0] is scaled + will be a (1, None, 2) tensor where dim [:,:,0] is scaled + photon: :py:class:`validphys.photon.compute.Photon` + If given, gives the AddPhoton layer a function to compute the MSR component for the photon """ # 1. Generate the fake input which will be used to integrate From 1502482c639408f59fcef7eb7400190ec51c4423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 5 Apr 2023 17:52:16 +0200 Subject: [PATCH 150/204] Use abc module in structure_functions --- validphys2/src/validphys/photon/structure_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 25ea915169..78ea51a91e 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -1,11 +1,14 @@ +from abc import ABC, abstractmethod + import numpy as np import pineappl from scipy.interpolate import RectBivariateSpline -class StructureFunction: +class StructureFunction(ABC): """Abstract class for the DIS structure functions""" + @abstractmethod def fxq(self, x, Q): pass From 7f12b7185e493251e551144b55f4f7bfa17a0126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 6 Apr 2023 11:23:23 +0200 Subject: [PATCH 151/204] Call black and isort on validphys/tests/photon --- .../validphys/tests/photon/test_compute.py | 94 ++++++++++++------- .../tests/photon/test_structurefunctions.py | 83 ++++++++-------- 2 files changed, 103 insertions(+), 74 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 115286af49..ce18b1ce7d 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -1,35 +1,42 @@ -from validphys.photon.compute import Photon -from validphys.photon import structure_functions -import numpy as np from collections import namedtuple from pathlib import Path + import fiatlux +import numpy as np +from validphys.photon import structure_functions +from validphys.photon.compute import Photon + +from ..conftest import PDF + -class faketheory(): +class faketheory: def __init__(self): self.path = Path("/fake/path/") + def get_description(self): return { "alphaqed": 0.01, "Qref": 91.2, "mc": 1.3, - "mb": 4.92 , - "mt": 172., - "kcThr": 1., - "kbThr": 1., - "ktThr": 1., + "mb": 4.92, + "mt": 172.0, + "kcThr": 1.0, + "kbThr": 1.0, + "ktThr": 1.0, "MaxNfAs": 5, "MaxNfPdf": 5, } + fiatlux_runcard = { - "pdf_name": "NNPDF40_nnlo_as_01180", - "additional_errors": False, + "pdf_name": PDF, + "additional_errors": False, } -photon = namedtuple('photon', ['total', 'elastic', 'inelastic']) +photon = namedtuple("photon", ["total", "elastic", "inelastic"]) + -class fakeFiatlux(): +class fakeFiatlux: def __init__(self, runcard): self.runcard = runcard self.alphaem = None @@ -40,54 +47,60 @@ def __init__(self, runcard): self.fl = None self.f2lo = None self.res = photon(0, 0, 0) - + def PlugAlphaQED(self, alphaem, qref): self.alphaem = alphaem self.qref = qref - + def InsertInelasticSplitQ(self, args): self.trash1 = args[0] self.trash2 = args[1] - + def PlugStructureFunctions(self, f2, fl, f2lo): self.f2 = f2 self.fl = fl self.f2lo = f2lo - + def EvaluatePhoton(self, x, q): return self.res - -class fakeStructureFunction(): + + +class fakeStructureFunction: def __init__(self, path, pdfs): self.path = path self.pdfs = pdfs - + def fxq(self): return 0 -class fakeF2LO(): + +class fakeF2LO: def __init__(self, pdfs, theory): self.pdfs = pdfs self.theory = theory - + def fxq(self): return 0 def test_init(monkeypatch): - monkeypatch.setattr(structure_functions, "InterpStructureFunction", fakeStructureFunction) + monkeypatch.setattr( + structure_functions, "InterpStructureFunction", fakeStructureFunction + ) monkeypatch.setattr(structure_functions, "F2LO", fakeF2LO) # monkeypatch.setattr(LHAPDFSet, "mkPDF", lambda *args: fiatlux_runcard["pdf_name"]) monkeypatch.setattr(fiatlux, "FiatLux", fakeFiatlux) monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) - photon = Photon(faketheory(), fiatlux_runcard, [1,2,3]) - + photon = Photon(faketheory(), fiatlux_runcard, [1, 2, 3]) + # test the easy parameters - np.testing.assert_equal(photon.replicas_id, [1,2,3]) + np.testing.assert_equal(photon.replicas_id, [1, 2, 3]) np.testing.assert_equal(photon.fiatlux_runcard, fiatlux_runcard) np.testing.assert_almost_equal(photon.q_in2, 1e4) - np.testing.assert_almost_equal(photon.alpha_em_ref, faketheory().get_description()["alphaqed"]) + np.testing.assert_almost_equal( + photon.alpha_em_ref, faketheory().get_description()["alphaqed"] + ) # test masses np.testing.assert_equal(photon.thresh_t, np.inf) @@ -99,16 +112,29 @@ def test_init(monkeypatch): np.testing.assert_almost_equal(photon.thresh[4], 4.92) np.testing.assert_almost_equal(photon.thresh[3], 1.3) np.testing.assert_almost_equal(photon.alpha_thresh[5], 0.01) - np.testing.assert_almost_equal(photon.alpha_thresh[4], photon.alpha_em_nlo(4.92, 0.01, 91.2, 5)) - np.testing.assert_almost_equal(photon.alpha_thresh[3], photon.alpha_em_nlo(1.3, photon.alpha_thresh[4], 4.92, 4)) + np.testing.assert_almost_equal( + photon.alpha_thresh[4], photon.alpha_em_nlo(4.92, 0.01, 91.2, 5) + ) + np.testing.assert_almost_equal( + photon.alpha_thresh[3], + photon.alpha_em_nlo(1.3, photon.alpha_thresh[4], 4.92, 4), + ) np.testing.assert_equal(len(photon.alpha_thresh), 3) np.testing.assert_equal(len(photon.thresh), 3) # test betas - vec_beta0 = [-0.5305164769729844, -0.6719875374991137, -0.7073553026306458, -0.8488263631567751] - vec_b1 = [0.17507043740108488, 0.1605510390839295, 0.1538497783221655, 0.1458920311675707] - for nf in range(3, 6+1): + vec_beta0 = [ + -0.5305164769729844, + -0.6719875374991137, + -0.7073553026306458, + -0.8488263631567751, + ] + vec_b1 = [ + 0.17507043740108488, + 0.1605510390839295, + 0.1538497783221655, + 0.1458920311675707, + ] + for nf in range(3, 6 + 1): np.testing.assert_allclose(photon.beta0[nf], vec_beta0[nf - 3], rtol=1e-7) np.testing.assert_allclose(photon.b1[nf], vec_b1[nf - 3], rtol=1e-7) - - diff --git a/validphys2/src/validphys/tests/photon/test_structurefunctions.py b/validphys2/src/validphys/tests/photon/test_structurefunctions.py index 829f154219..984eb2eecd 100644 --- a/validphys2/src/validphys/tests/photon/test_structurefunctions.py +++ b/validphys2/src/validphys/tests/photon/test_structurefunctions.py @@ -1,26 +1,28 @@ -import validphys.photon.structure_functions as sf import numpy as np import pineappl +import validphys.photon.structure_functions as sf from validphys.lhapdfset import LHAPDFSet + class ZeroPdfs: def xfxQ(self, x, Q): res = {} - for i in range(1, 6+1): - res[i] = res[-i] = 0. + for i in range(1, 6 + 1): + res[i] = res[-i] = 0.0 return res + def test_zero_pdfs(): pdfs = ZeroPdfs() fake_theory = { "mc": 1.3, - "mb": 5. , - "mt": 172., - "kcThr": 1., - "kbThr": 1., - "ktThr": 1., + "mb": 5.0, + "mt": 172.0, + "kcThr": 1.0, + "kbThr": 1.0, + "ktThr": 1.0, "MaxNfPdf": 5, } @@ -28,24 +30,26 @@ def test_zero_pdfs(): np.testing.assert_equal(f2lo.thresh_t, np.inf) - for x in np.geomspace(1e-4, 1., 10): + for x in np.geomspace(1e-4, 1.0, 10): for Q in np.geomspace(10, 1000000, 10): - np.testing.assert_allclose(f2lo.fxq(x, Q), 0.) + np.testing.assert_allclose(f2lo.fxq(x, Q), 0.0) + -class FakeSet(): +class FakeSet: def get_entry(self, string): return 0 -class ZeroFKTable(): + +class ZeroFKTable: def __init__(self, path): self.path = path - self.xgrid = np.geomspace(1e-4, 1., 10) + self.xgrid = np.geomspace(1e-4, 1.0, 10) self.qgrid = np.geomspace(1.65, 1000, 10) def bin_left(self, i): if i == 1: return self.xgrid - if i == 0 : + if i == 0: return self.qgrid else: return 0 @@ -53,29 +57,31 @@ def bin_left(self, i): def convolute_with_one(self, pdgid, xfxQ2): return np.zeros((10, 10)) -class OnePdf(): + +class OnePdf: def __init__(self): self.ao = 1 - + def xfxQ(self, x, Q): - return 1. - + return 1.0 + def xfxQ2(self, x, Q): - return 1.**2 - + return 1.0**2 + def set(self): return FakeSet() -class OneFKTable(): + +class OneFKTable: def __init__(self, path): self.path = path - self.xgrid = np.geomspace(1e-4, 1., 10) + self.xgrid = np.geomspace(1e-4, 1.0, 10) self.qgrid = np.geomspace(1.65, 1000, 10) def bin_left(self, i): if i == 1: return self.xgrid - if i == 0 : + if i == 0: return self.qgrid else: return 0 @@ -84,16 +90,16 @@ def convolute_with_one(self, pdgid, xfxQ2): return np.zeros((10, 10)) -class ZeroPdf(): +class ZeroPdf: def __init__(self): self.ao = 1 - + def xfxQ(self, x, Q): - return 1. - + return 1.0 + def xfxQ2(self, x, Q): - return 1.**2 - + return 1.0**2 + def set(self): return FakeSet() @@ -102,23 +108,20 @@ def test_F2(monkeypatch): # grid put to 0 and pdf put to 1 monkeypatch.setattr(pineappl.fk_table.FkTable, "read", ZeroFKTable) structurefunc = sf.InterpStructureFunction("", OnePdf()) - for x in np.geomspace(1e-4, 1., 10): + for x in np.geomspace(1e-4, 1.0, 10): for Q in np.geomspace(10, 1000000, 10): - np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) - + np.testing.assert_allclose(structurefunc.fxq(x, Q), 0.0, rtol=1e-5) + # grid put to 0 and real pdf pdf = LHAPDFSet("NNPDF40_nnlo_as_01180", "replicas") structurefunc = sf.InterpStructureFunction("", pdf.central_member) - for x in np.geomspace(1e-4, 1., 10): + for x in np.geomspace(1e-4, 1.0, 10): for Q in np.geomspace(10, 1000000, 10): - np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) - + np.testing.assert_allclose(structurefunc.fxq(x, Q), 0.0, rtol=1e-5) + # grid put to 1 and pdf put to 0 monkeypatch.setattr(pineappl.fk_table.FkTable, "read", OneFKTable) structurefunc = sf.InterpStructureFunction("", ZeroPdf()) - for x in np.geomspace(1e-4, 1., 10): + for x in np.geomspace(1e-4, 1.0, 10): for Q in np.geomspace(10, 1000000, 10): - np.testing.assert_allclose(structurefunc.fxq(x, Q), 0., rtol=1e-5) - - - \ No newline at end of file + np.testing.assert_allclose(structurefunc.fxq(x, Q), 0.0, rtol=1e-5) From 092e2aa726d3d6ef7ecc93adbc07df04221c6bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Mon, 10 Apr 2023 21:53:13 +0200 Subject: [PATCH 152/204] Fix bug in apply_photon --- n3fit/src/n3fit/model_gen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index fb42b85772..d360b43ea3 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -654,9 +654,9 @@ def layer_pdf(x): def apply_photon(x): # if photon is None then the photon layer is not applied if photons: - return normalized_pdf(x) - else: return layer_photon(normalized_pdf(x), i) + else: + return normalized_pdf(x) final_pdf = apply_photon From 19f2deec3d13c1f391219bdbdbd20705457319b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 12 Apr 2023 12:31:20 +0200 Subject: [PATCH 153/204] Remove produce_interpolators from structure functions --- .../validphys/photon/structure_functions.py | 35 ++++++------ .../validphys/tests/photon/test_compute.py | 53 +++++++++++++------ 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 78ea51a91e..571ad2baa9 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -10,6 +10,21 @@ class StructureFunction(ABC): @abstractmethod def fxq(self, x, Q): + r""" + Abstract method returning the value of a DIS structure functions. + + Parameters + ---------- + x : float + Bjorken x + Q : float + DIS hard scale + + Returns + ------- + F_{2,L}: float + Structure function F2 or FL + """ pass @@ -21,19 +36,11 @@ class InterpStructureFunction(StructureFunction): def __init__(self, path_to_fktable, pdfs): self.fktable = pineappl.fk_table.FkTable.read(path_to_fktable) - self.pdfs = pdfs - self.pdgid = int(pdfs.set().get_entry("Particle")) - self.produce_interpolator() - - def produce_interpolator(self): - """Produce the interpolation function to be called in fxq.""" x = np.unique(self.fktable.bin_left(1)) q2 = np.unique(self.fktable.bin_left(0)) - self.xmin = min(x) - self.qmin = min(np.sqrt(q2)) - self.qmax = max(np.sqrt(q2)) - predictions = self.fktable.convolute_with_one(self.pdgid, self.pdfs.xfxQ2) + pdgid = int(pdfs.set().get_entry("Particle")) + predictions = self.fktable.convolute_with_one(pdgid, pdfs.xfxQ2) # here we require that the (x,Q2) couples that we passed # to pinefarm is a rectangular matrix @@ -47,7 +54,7 @@ def fxq(self, x, q): Parameters ---------- x : float - Bjorken's variable + Bjorken x Q : float DIS hard scale @@ -56,10 +63,6 @@ def fxq(self, x, q): F_{2,L}: float Structure function F2 or FL """ - # here we are requiring that the grid that we pass to fiatlux - # has Qmin = 1 (fiatlux doesn't go below Q=1) - # if x < self.xmin or q > self.qmax : - # return 0. return self.interpolator(x, q**2)[0, 0] @@ -88,7 +91,7 @@ def fxq(self, x, q): Parameters ---------- x : float - Bjorken's variable + Bjorken x Q : float DIS hard scale diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index ce18b1ce7d..ec464ad164 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -9,7 +9,7 @@ from ..conftest import PDF -class faketheory: +class FakeTheory: def __init__(self): self.path = Path("/fake/path/") @@ -36,7 +36,7 @@ def get_description(self): photon = namedtuple("photon", ["total", "elastic", "inelastic"]) -class fakeFiatlux: +class FakeFiatlux: def __init__(self, runcard): self.runcard = runcard self.alphaem = None @@ -65,7 +65,7 @@ def EvaluatePhoton(self, x, q): return self.res -class fakeStructureFunction: +class FakeStructureFunction: def __init__(self, path, pdfs): self.path = path self.pdfs = pdfs @@ -74,7 +74,7 @@ def fxq(self): return 0 -class fakeF2LO: +class FakeF2LO: def __init__(self, pdfs, theory): self.pdfs = pdfs self.theory = theory @@ -83,31 +83,46 @@ def fxq(self): return 0 -def test_init(monkeypatch): +def test_parameters_init(monkeypatch): monkeypatch.setattr( - structure_functions, "InterpStructureFunction", fakeStructureFunction + structure_functions, "InterpStructureFunction", FakeStructureFunction ) - monkeypatch.setattr(structure_functions, "F2LO", fakeF2LO) - # monkeypatch.setattr(LHAPDFSet, "mkPDF", lambda *args: fiatlux_runcard["pdf_name"]) - monkeypatch.setattr(fiatlux, "FiatLux", fakeFiatlux) + monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) + + monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) - photon = Photon(faketheory(), fiatlux_runcard, [1, 2, 3]) + photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) - # test the easy parameters np.testing.assert_equal(photon.replicas_id, [1, 2, 3]) np.testing.assert_equal(photon.fiatlux_runcard, fiatlux_runcard) np.testing.assert_almost_equal(photon.q_in2, 1e4) np.testing.assert_almost_equal( - photon.alpha_em_ref, faketheory().get_description()["alphaqed"] + photon.alpha_em_ref, FakeTheory().get_description()["alphaqed"] + ) + +def test_masses_init(monkeypatch): + monkeypatch.setattr( + structure_functions, "InterpStructureFunction", FakeStructureFunction ) + monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) - # test masses + monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) + monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) + photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) np.testing.assert_equal(photon.thresh_t, np.inf) np.testing.assert_almost_equal(photon.thresh_b, 4.92) np.testing.assert_almost_equal(photon.thresh_c, 1.3) - # test set_thresholds_alpha_em +def test_set_thresholds_alpha_em(monkeypatch): + monkeypatch.setattr( + structure_functions, "InterpStructureFunction", FakeStructureFunction + ) + monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) + + monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) + monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) + photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) np.testing.assert_almost_equal(photon.thresh[5], 91.2) np.testing.assert_almost_equal(photon.thresh[4], 4.92) np.testing.assert_almost_equal(photon.thresh[3], 1.3) @@ -122,7 +137,15 @@ def test_init(monkeypatch): np.testing.assert_equal(len(photon.alpha_thresh), 3) np.testing.assert_equal(len(photon.thresh), 3) - # test betas +def test_betas(monkeypatch): + monkeypatch.setattr( + structure_functions, "InterpStructureFunction", FakeStructureFunction + ) + monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) + + monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) + monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) + photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) vec_beta0 = [ -0.5305164769729844, -0.6719875374991137, From 2652406dd9eb612d52161c243d76157a3d0ad1fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 12 Apr 2023 12:32:33 +0200 Subject: [PATCH 154/204] Update validphys2/src/validphys/photon/structure_functions.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/photon/structure_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 571ad2baa9..3a1a391020 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -67,7 +67,7 @@ def fxq(self, x, q): class F2LO(StructureFunction): - """Compute analytically the leading order structure function for F2.""" + """Analytically compute the structure function F2 at leading order.""" def __init__(self, pdfs, theory): self.pdfs = pdfs From 6249e294a1cae31a56a1aed400f1c1039e760962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 12 Apr 2023 12:33:01 +0200 Subject: [PATCH 155/204] Update validphys2/src/validphys/photon/structure_functions.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/photon/structure_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 3a1a391020..a69d1188e5 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -86,7 +86,7 @@ def __init__(self, pdfs, theory): def fxq(self, x, q): r""" - Compute the analytical form of F2LO. + Analytically compute F2LO. Parameters ---------- From 3dd163ec0cfb64e24c654ac9e47db8ef0ee95f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 12 Apr 2023 14:26:12 +0200 Subject: [PATCH 156/204] Split alphaem running from Photon class --- n3fit/runcards/examples/Basic_runcard_qed.yml | 3 +- n3fit/src/n3fit/layers/rotations.py | 2 +- validphys2/src/validphys/photon/compute.py | 273 +++++++++--------- .../validphys/photon/structure_functions.py | 2 +- 4 files changed, 147 insertions(+), 133 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index a4b59f413f..9841ce29c1 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -168,8 +168,7 @@ maxcores: 8 fiatlux: apfel: off - qed_running: off # determines the running of alpha. - q2_max: 60000 # the maximum allowed Q2. + q2_max: 1e8 # the maximum allowed Q2. eps_base: 1e-5 # precision on final integration of double integral. eps_rel: 1e-1 # extra precision on any single integration. mproton: 0.938272046 # proton mass from 2015 update of PDG review diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index a4e8325679..59ddbcde30 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -109,7 +109,7 @@ def __init__(self, photons, **kwargs): def register_photon(self, xgrid): """Compute the photon array and set the layer to be rebuilt""" if self._photons_generator: - self._pdf_ph = self._photons_generator.compute(xgrid) + self._pdf_ph = self._photons_generator(xgrid) self.built = False def call(self, pdfs, ph_replica): diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 590dd4ce3b..4074174c0d 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -25,17 +25,11 @@ class Photon: def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard + self.fiatlux_runcard["qed_running"] = "QrefQED" in self.theory self.replicas_id = replicas_id self.q_in = 100 self.q_in2 = self.q_in**2 - # parameters for the alphaem running - self.alpha_em_ref = self.theory["alphaqed"] - self.qref = self.theory["Qref"] - - self.set_betas() - self.set_thresholds_alpha_em() - # structure functions self.qcd_pdfs = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") @@ -60,12 +54,13 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): # we have a dict but fiatlux wants a yaml file # TODO : once that fiatlux will allow dictionaries # pass directly self.fiatlux_runcard + alpha = Alpha(self.theory) for id in replicas_id: - self.lux[id].PlugAlphaQED(self.alpha_em, self.qref) + self.lux[id].PlugAlphaQED(alpha.alpha_em, alpha.qref) self.lux[id].InsertInelasticSplitQ( [ - self.thresh_b, - self.thresh_t if self.theory["MaxNfPdf"] == 6 else 1e100, + self.theory["kbThr"] * self.theory["mb"], + self.theory["ktThr"] * self.theory["mt"] if self.theory["MaxNfPdf"] == 6 else 1e100, ] ) self.lux[id].PlugStructureFunctions(f2[id].fxq, fl[id].fxq, f2lo[id].fxq) @@ -75,123 +70,6 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.produce_interpolators() - def alpha_em(self, q): - r""" - Compute the value of alpha_em. - - Parameters - ---------- - q: float - value in which the coupling is computed - - Returns - ------- - alpha_em: float - electromagnetic coupling - """ - if q < self.thresh_c: - nf = 3 - elif q < self.thresh_b: - nf = 4 - elif q < self.thresh_t: - nf = 5 - else: - nf = 6 - return self.alpha_em_nlo(q, self.alpha_thresh[nf], self.thresh[nf], nf) - - def alpha_em_nlo(self, q, alpha_ref, qref, nf): - """ - Compute the alpha_em running for FFS at NLO. - - Parameters - ---------- - q : float - target scale - a_ref : float - reference value of a = alpha_em/(4*pi) - qref: float - reference scale - nf: int - number of flavors - - Returns - ------- - as_NLO : float - target value of a - """ - lmu = 2 * np.log(q / qref) - den = 1.0 + self.beta0[nf] * alpha_ref * lmu - alpha_LO = alpha_ref / den - alpha_NLO = alpha_LO * (1 - self.b1[nf] * alpha_LO * np.log(den)) - return alpha_NLO - - def set_thresholds_alpha_em(self): - """ - Compute and store the couplings at thresholds to speed up the calling - to alpha_em inside fiatlux: - when q is in a certain range (e.g. thresh_c < q < thresh_b) and qref in a different one - (e.g. thresh_b < q < thresh_t) we need to evolve from qref to thresh_b with nf=5 and then - from thresh_b to q with nf=4. Given that the value of alpha at thresh_b is always the same - we can avoid computing the first step storing the values of alpha in the threshold points. - It is done for qref in a generic range (not necessarly qref=91.2). - - """ - self.thresh_c = self.theory["kcThr"] * self.theory["mc"] - self.thresh_b = self.theory["kbThr"] * self.theory["mb"] - self.thresh_t = self.theory["ktThr"] * self.theory["mt"] - if self.theory["MaxNfAs"] <= 5: - self.thresh_t = np.inf - if self.theory["MaxNfAs"] <= 4: - self.thresh_b = np.inf - if self.theory["MaxNfAs"] <= 3: - self.thresh_c = np.inf - - thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] - # determine nfref - if self.qref < self.thresh_c: - nfref = 3 - elif self.qref < self.thresh_b: - nfref = 4 - elif self.qref < self.thresh_t: - nfref = 5 - else: - nfref = 6 - thresh_list.insert(nfref - 3, self.qref) - - self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} - - self.alpha_thresh = {nfref: self.alpha_em_ref} - - # determine the values of alpha in the threshold points, depending on the value of qref - for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): - self.alpha_thresh[nf] = self.alpha_em_nlo( - self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1 - ) - - for nf in reversed(range(3, nfref)): - self.alpha_thresh[nf] = self.alpha_em_nlo( - self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1 - ) - - def set_betas(self): - """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" - nl = 3 - nc = 3 - eu2 = 4.0 / 9 - ed2 = 1.0 / 9 - self.beta0 = {} - self.b1 = {} - for nf in range(3, 6 + 1): - nu = nf // 2 - nd = nf - nu - self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / (4 * np.pi) - self.b1[nf] = ( - -4.0 - * (nl + nc * (nu * eu2**2 + nd * ed2**2)) - / self.beta0[nf] - / (4 * np.pi) ** 2 - ) - def compute_photon_array(self, id): r""" Compute the photon PDF for every point in the grid xgrid. @@ -204,7 +82,7 @@ def compute_photon_array(self, id): Returns ------- compute_photon_array: numpy.array - photon PDF at the scale 1 GeV + photon PDF at the fitting scale Q0 """ # Compute photon PDF log.info(f"Computing photon") @@ -255,7 +133,7 @@ def produce_interpolators(self): for photon_array in self.photons_array ] - def compute(self, xgrid): + def __call__(self, xgrid): """ Compute the photon interpolating the values of self.photon_array. @@ -310,3 +188,140 @@ def generate_errors(self, replica_id): # or use directly replica_nnseed? def replica_luxseed(replica, luxseed): return replica_nnseed(replica, luxseed) + + +class Alpha : + def __init__(self, theory): + # parameters for the alphaem running + self.theory = theory + self.alpha_em_ref = theory["alphaqed"] + self.theory["QrefQED"] = 91.2 + self.qref = self.theory.get("QrefQED") + + if self.qref: + self.set_betas() + self.set_thresholds_alpha_em() + self.alpha_em = self.running_alpha_em + else : + self.qref = self.theory["Qref"] + self.alpha_em = self.fixed_alpha_em + + def running_alpha_em(self, q): + r""" + Compute the value of alpha_em. + + Parameters + ---------- + q: float + value in which the coupling is computed + + Returns + ------- + alpha_em: float + electromagnetic coupling + """ + if q < self.thresh_c: + nf = 3 + elif q < self.thresh_b: + nf = 4 + elif q < self.thresh_t: + nf = 5 + else: + nf = 6 + return self.alpha_em_nlo(q, self.alpha_thresh[nf], self.thresh[nf], nf) + + def fixed_alpha_em(self, q): + return self.alpha_em_ref + + def alpha_em_nlo(self, q, alpha_ref, qref, nf): + """ + Compute the alpha_em running for FFS at NLO. + + Parameters + ---------- + q : float + target scale + a_ref : float + reference value of a = alpha_em/(4*pi) + qref: float + reference scale + nf: int + number of flavors + + Returns + ------- + as_NLO : float + target value of a + """ + lmu = 2 * np.log(q / qref) + den = 1.0 + self.beta0[nf] * alpha_ref * lmu + alpha_LO = alpha_ref / den + alpha_NLO = alpha_LO * (1 - self.b1[nf] * alpha_LO * np.log(den)) + return alpha_NLO + + def set_thresholds_alpha_em(self): + """ + Compute and store the couplings at thresholds to speed up the calling + to alpha_em inside fiatlux: + when q is in a certain range (e.g. thresh_c < q < thresh_b) and qref in a different one + (e.g. thresh_b < q < thresh_t) we need to evolve from qref to thresh_b with nf=5 and then + from thresh_b to q with nf=4. Given that the value of alpha at thresh_b is always the same + we can avoid computing the first step storing the values of alpha in the threshold points. + It is done for qref in a generic range (not necessarly qref=91.2). + + """ + self.thresh_c = self.theory["kcThr"] * self.theory["mc"] + self.thresh_b = self.theory["kbThr"] * self.theory["mb"] + self.thresh_t = self.theory["ktThr"] * self.theory["mt"] + if self.theory["MaxNfAs"] <= 5: + self.thresh_t = np.inf + if self.theory["MaxNfAs"] <= 4: + self.thresh_b = np.inf + if self.theory["MaxNfAs"] <= 3: + self.thresh_c = np.inf + + thresh_list = [self.thresh_c, self.thresh_b, self.thresh_t] + # determine nfref + if self.qref < self.thresh_c: + nfref = 3 + elif self.qref < self.thresh_b: + nfref = 4 + elif self.qref < self.thresh_t: + nfref = 5 + else: + nfref = 6 + thresh_list.insert(nfref - 3, self.qref) + + self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} + + self.alpha_thresh = {nfref: self.alpha_em_ref} + + # determine the values of alpha in the threshold points, depending on the value of qref + for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): + self.alpha_thresh[nf] = self.alpha_em_nlo( + self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1 + ) + + for nf in reversed(range(3, nfref)): + self.alpha_thresh[nf] = self.alpha_em_nlo( + self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1 + ) + + def set_betas(self): + """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" + nl = 3 + nc = 3 + eu2 = 4.0 / 9 + ed2 = 1.0 / 9 + self.beta0 = {} + self.b1 = {} + for nf in range(3, 6 + 1): + nu = nf // 2 + nd = nf - nu + self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / (4 * np.pi) + self.b1[nf] = ( + -4.0 + * (nl + nc * (nu * eu2**2 + nd * ed2**2)) + / self.beta0[nf] + / (4 * np.pi) ** 2 + ) \ No newline at end of file diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index a69d1188e5..36019a5cf2 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -97,7 +97,7 @@ def fxq(self, x, q): Returns ------- - F2_LO : float + F_2^{LO} : float Structure function F2 at LO """ # at LO we use ZM-VFS From 1e3303b19b65247feac0277ad18fdf075356bcdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 13:07:45 +0200 Subject: [PATCH 157/204] Add module constants --- validphys2/src/validphys/n3fit_data.py | 6 + validphys2/src/validphys/photon/compute.py | 121 +++++++++--------- .../validphys/photon/structure_functions.py | 6 +- 3 files changed, 66 insertions(+), 67 deletions(-) diff --git a/validphys2/src/validphys/n3fit_data.py b/validphys2/src/validphys/n3fit_data.py index 75e5efdc12..2019aadc88 100644 --- a/validphys2/src/validphys/n3fit_data.py +++ b/validphys2/src/validphys/n3fit_data.py @@ -52,6 +52,12 @@ def replica_mcseed(replica, mcseed, genrep): res = np.random.randint(0, pow(2, 31)) return res +def replica_luxseed(replica, luxseed): + """Generate the ``luxseed`` for a ``replica``. + Identical to replica_nnseed but used for a different purpose. + """ + return replica_nnseed(replica, luxseed) + class _TrMasks(TupleComp): """Class holding the training validation mask for a group of datasets diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 4074174c0d..d7339f32ae 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -10,10 +10,11 @@ from scipy.integrate import trapezoid from scipy.interpolate import interp1d from validphys.lhapdfset import LHAPDFSet -from validphys.n3fit_data import replica_nnseed +from validphys.n3fit_data import replica_luxseed from n3fit.io.writer import XGRID +from . import constants from . import structure_functions as sf log = logging.getLogger(__name__) @@ -60,7 +61,9 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.lux[id].InsertInelasticSplitQ( [ self.theory["kbThr"] * self.theory["mb"], - self.theory["ktThr"] * self.theory["mt"] if self.theory["MaxNfPdf"] == 6 else 1e100, + self.theory["ktThr"] * self.theory["mt"] + if self.theory["MaxNfPdf"] == 6 + else 1e100, ] ) self.lux[id].PlugStructureFunctions(f2[id].fxq, fl[id].fxq, f2lo[id].fxq) @@ -68,7 +71,11 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() - self.produce_interpolators() + self.photons_array = [self.compute_photon_array(id) for id in self.replicas_id] + self.interpolator = [ + interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") + for photon_array in self.photons_array + ] def compute_photon_array(self, id): r""" @@ -87,52 +94,41 @@ def compute_photon_array(self, id): # Compute photon PDF log.info(f"Computing photon") start_time = time.perf_counter() - photon_100GeV = np.array( + photon_qin = np.array( [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] ) log.info(f"Computation time: {time.perf_counter() - start_time}") - photon_100GeV += self.generate_errors(id) - photon_100GeV /= self.xgrid + photon_qin += self.generate_errors(id) + photon_qin /= self.xgrid # TODO : the different x points could be even computed in parallel # Load eko and reshape it with EKO.read(self.path_to_eko_photon) as eko: - # If we make sure that the grid of the precomputed EKO is the same of - # self.xgrid then we don't need to reshape - # TODO : move the reshape inside vp-setupfit - # xgrid_reshape(eko, targetgrid = XGrid(self.xgrid), inputgrid = XGrid(self.xgrid)) + # TODO : if the eko has not the correct grid we have to reshape it + # it has to be done inside vp-setupfit # construct PDFs - pdfs = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) + pdfs_init = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) for j, pid in enumerate(eko.rotations.inputpids): if pid == 22: - pdfs[j] = photon_100GeV + pdfs_init[j] = photon_qin ph_id = j if pid not in self.qcd_pdfs.flavors: continue - pdfs[j] = np.array( + pdfs_init[j] = np.array( [self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x for x in self.xgrid] ) # Apply EKO to PDFs q2 = eko.mu2grid[0] with eko.operator(q2) as elem: - pdf_final = np.einsum("ajbk,bk", elem.operator, pdfs) - # error_final = np.einsum("ajbk,bk", elem.error, pdfs) + pdfs_final = np.einsum("ajbk,bk", elem.operator, pdfs_init) - photon_Q0 = pdf_final[ph_id] + photon_Q0 = pdfs_final[ph_id] # we want x * gamma(x) return self.xgrid * photon_Q0 - def produce_interpolators(self): - """Produce the interpolation functions to be called in compute.""" - self.photons_array = [self.compute_photon_array(id) for id in self.replicas_id] - self.interpolator = [ - interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") - for photon_array in self.photons_array - ] - def __call__(self, xgrid): """ Compute the photon interpolating the values of self.photon_array. @@ -155,7 +151,8 @@ def __call__(self, xgrid): def integrate(self): """Compute the integral of the photon on the x range.""" return [ - trapezoid(self.photons_array[id], self.xgrid) for id in range(len(self.replicas_id)) + trapezoid(self.photons_array[id], self.xgrid) + for id in range(len(self.replicas_id)) ] def generate_error_matrix(self): @@ -184,29 +181,17 @@ def generate_errors(self, replica_id): return errors -# TODO : move it in n3fit_data.py with the others replica_seed generators? -# or use directly replica_nnseed? -def replica_luxseed(replica, luxseed): - return replica_nnseed(replica, luxseed) - - -class Alpha : +class Alpha: def __init__(self, theory): # parameters for the alphaem running self.theory = theory self.alpha_em_ref = theory["alphaqed"] - self.theory["QrefQED"] = 91.2 - self.qref = self.theory.get("QrefQED") - - if self.qref: - self.set_betas() - self.set_thresholds_alpha_em() - self.alpha_em = self.running_alpha_em - else : - self.qref = self.theory["Qref"] - self.alpha_em = self.fixed_alpha_em - - def running_alpha_em(self, q): + self.qref = self.theory.get("QrefQED", theory["Qref"]) + + self.beta0, self.b1 = self.set_betas() + self.thresh, self.alpha_thresh = self.set_thresholds_alpha_em() + + def alpha_em(self, q): r""" Compute the value of alpha_em. @@ -229,9 +214,6 @@ def running_alpha_em(self, q): else: nf = 6 return self.alpha_em_nlo(q, self.alpha_thresh[nf], self.thresh[nf], nf) - - def fixed_alpha_em(self, q): - return self.alpha_em_ref def alpha_em_nlo(self, q, alpha_ref, qref, nf): """ @@ -292,36 +274,47 @@ def set_thresholds_alpha_em(self): nfref = 6 thresh_list.insert(nfref - 3, self.qref) - self.thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} + thresh = { + nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1) + } - self.alpha_thresh = {nfref: self.alpha_em_ref} + alpha_thresh = {nfref: self.alpha_em_ref} # determine the values of alpha in the threshold points, depending on the value of qref for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): - self.alpha_thresh[nf] = self.alpha_em_nlo( - self.thresh[nf], self.alpha_thresh[nf - 1], self.thresh[nf - 1], nf - 1 + alpha_thresh[nf] = self.alpha_em_nlo( + thresh[nf], alpha_thresh[nf - 1], thresh[nf - 1], nf - 1 ) for nf in reversed(range(3, nfref)): - self.alpha_thresh[nf] = self.alpha_em_nlo( - self.thresh[nf], self.alpha_thresh[nf + 1], self.thresh[nf + 1], nf + 1 + alpha_thresh[nf] = self.alpha_em_nlo( + thresh[nf], alpha_thresh[nf + 1], thresh[nf + 1], nf + 1 ) + return thresh, alpha_thresh + def set_betas(self): """Compute and store beta0 / 4pi and b1 = (beta1/beta0)/4pi as a function of nf.""" - nl = 3 - nc = 3 - eu2 = 4.0 / 9 - ed2 = 1.0 / 9 - self.beta0 = {} - self.b1 = {} + beta0 = {} + b1 = {} for nf in range(3, 6 + 1): nu = nf // 2 nd = nf - nu - self.beta0[nf] = (-4.0 / 3 * (nl + nc * (nu * eu2 + nd * ed2))) / (4 * np.pi) - self.b1[nf] = ( + beta0[nf] = ( + -4.0 + / 3 + * ( + constants.NL + + constants.NC * (nu * constants.EU2 + nd * constants.ED2) + ) + ) / (4 * np.pi) + b1[nf] = ( -4.0 - * (nl + nc * (nu * eu2**2 + nd * ed2**2)) - / self.beta0[nf] + * ( + constants.NL + + constants.NC * (nu * constants.EU2**2 + nd * constants.ED2**2) + ) + / beta0[nf] / (4 * np.pi) ** 2 - ) \ No newline at end of file + ) + return beta0, b1 diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 36019a5cf2..bff47d1f33 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -4,6 +4,8 @@ import pineappl from scipy.interpolate import RectBivariateSpline +from . import constants + class StructureFunction(ABC): """Abstract class for the DIS structure functions""" @@ -80,9 +82,7 @@ def __init__(self, pdfs, theory): self.thresh_b = np.inf if theory["MaxNfPdf"] <= 3: self.thresh_c = np.inf - eu2 = 4.0 / 9 - ed2 = 1.0 / 9 - self.eq2 = [ed2, eu2, ed2, eu2, ed2, eu2] # d u s c b t + self.eq2 = [constants.ED2, constants.EU2] * 3 # d u s c b t def fxq(self, x, q): r""" From 6934ba5ec55017ee0868f72cbf0d3ef1a67fa4af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Thu, 13 Apr 2023 13:13:47 +0200 Subject: [PATCH 158/204] Update validphys2/src/validphys/photon/compute.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/photon/compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index d7339f32ae..e6050026e7 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -232,7 +232,7 @@ def alpha_em_nlo(self, q, alpha_ref, qref, nf): Returns ------- - as_NLO : float + alpha_em at NLO : float target value of a """ lmu = 2 * np.log(q / qref) From 00be444c849bfdf120494631f3a10f6de982d43a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 14:28:58 +0200 Subject: [PATCH 159/204] Rename replicas_id -> replicas --- n3fit/src/n3fit/model_trainer.py | 6 +-- n3fit/src/n3fit/performfit.py | 2 +- validphys2/src/validphys/photon/compute.py | 43 +++++++++++----------- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 647562fc8c..9e22b35d84 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -102,7 +102,7 @@ def __init__( parallel_models=1, theoryid=None, fiatlux_runcard=None, - replicas_id=None, + replicas=None, ): """ Parameters @@ -163,7 +163,7 @@ def __init__( self._parallel_models = parallel_models self.theoryid = theoryid self.fiatlux_runcard = fiatlux_runcard - self.replicas_id = replicas_id + self.replicas = replicas # Initialise internal variables which define behaviour if debug: @@ -889,7 +889,7 @@ def hyperparametrizable(self, params): photons=Photon( theoryid=self.theoryid, fiatlux_runcard=self.fiatlux_runcard, - replicas_id=self.replicas_id, + replicas=self.replicas, ) else: photons=None diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 4aba5964c7..6c9a9223f3 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -200,7 +200,7 @@ def performfit( parallel_models=n_models, theoryid=theoryid, fiatlux_runcard=fiatlux, - replicas_id=replica_idxs, + replicas=replica_idxs, ) # This is just to give a descriptive name to the fit function diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index d7339f32ae..6144d8aa34 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -19,17 +19,18 @@ log = logging.getLogger(__name__) +Q_IN = 100 +EXTRA_SET = "LUXqed17_plus_PDF4LHC15_nnlo_100" + class Photon: """Photon class computing the photon array with the LuxQED approach.""" - def __init__(self, theoryid, fiatlux_runcard, replicas_id): + def __init__(self, theoryid, fiatlux_runcard, replicas): self.theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard self.fiatlux_runcard["qed_running"] = "QrefQED" in self.theory - self.replicas_id = replicas_id - self.q_in = 100 - self.q_in2 = self.q_in**2 + self.replicas = replicas # structure functions self.qcd_pdfs = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") @@ -44,21 +45,21 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): f2 = {} fl = {} f2lo = {} - for id in replicas_id: - f2[id] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[id]) - fl[id] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[id]) - f2lo[id] = sf.F2LO(self.qcd_pdfs.members[id], self.theory) + for replica in replicas: + f2[replica] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[replica]) + fl[replica] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[replica]) + f2lo[replica] = sf.F2LO(self.qcd_pdfs.members[replica], self.theory) with tempfile.NamedTemporaryFile(mode="w") as tmp: with tmp.file as tmp_file: tmp_file.write(yaml.dump(self.fiatlux_runcard)) - self.lux[id] = fiatlux.FiatLux(tmp_file.name) + self.lux[replica] = fiatlux.FiatLux(tmp_file.name) # we have a dict but fiatlux wants a yaml file # TODO : once that fiatlux will allow dictionaries # pass directly self.fiatlux_runcard alpha = Alpha(self.theory) - for id in replicas_id: - self.lux[id].PlugAlphaQED(alpha.alpha_em, alpha.qref) - self.lux[id].InsertInelasticSplitQ( + for replica in replicas: + self.lux[replica].PlugAlphaQED(alpha.alpha_em, alpha.qref) + self.lux[replica].InsertInelasticSplitQ( [ self.theory["kbThr"] * self.theory["mb"], self.theory["ktThr"] * self.theory["mt"] @@ -66,24 +67,24 @@ def __init__(self, theoryid, fiatlux_runcard, replicas_id): else 1e100, ] ) - self.lux[id].PlugStructureFunctions(f2[id].fxq, fl[id].fxq, f2lo[id].fxq) + self.lux[replica].PlugStructureFunctions(f2[replica].fxq, fl[replica].fxq, f2lo[replica].fxq) self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() - self.photons_array = [self.compute_photon_array(id) for id in self.replicas_id] + self.photons_array = [self.compute_photon_array(id) for id in self.replicas] self.interpolator = [ interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") for photon_array in self.photons_array ] - def compute_photon_array(self, id): + def compute_photon_array(self, replica): r""" Compute the photon PDF for every point in the grid xgrid. Parameters ---------- - id: int + replica: int replica id Returns @@ -95,10 +96,10 @@ def compute_photon_array(self, id): log.info(f"Computing photon") start_time = time.perf_counter() photon_qin = np.array( - [self.lux[id].EvaluatePhoton(x, self.q_in2).total for x in self.xgrid] + [self.lux[replica].EvaluatePhoton(x, Q_IN**2).total for x in self.xgrid] ) log.info(f"Computation time: {time.perf_counter() - start_time}") - photon_qin += self.generate_errors(id) + photon_qin += self.generate_errors(replica) photon_qin /= self.xgrid # TODO : the different x points could be even computed in parallel @@ -116,7 +117,7 @@ def compute_photon_array(self, id): if pid not in self.qcd_pdfs.flavors: continue pdfs_init[j] = np.array( - [self.qcd_pdfs.xfxQ(x, self.q_in, id, pid) / x for x in self.xgrid] + [self.qcd_pdfs.xfxQ(x, Q_IN, replica, pid) / x for x in self.xgrid] ) # Apply EKO to PDFs @@ -159,8 +160,8 @@ def generate_error_matrix(self): """generate error matrix to be used for the additional errors.""" if not self.fiatlux_runcard["additional_errors"]: return None - extra_set = LHAPDFSet("LUXqed17_plus_PDF4LHC15_nnlo_100", "replicas") - qs = [self.q_in] * len(self.xgrid) + extra_set = LHAPDFSet(EXTRA_SET, "replicas") + qs = [Q_IN] * len(self.xgrid) res_central = np.array(extra_set.central_member.xfxQ(22, self.xgrid, qs)) res = [] for idx_member in range(101, 107 + 1): From 75fa5907a1e0f11a4d5b919395fe0bab21385257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 14:43:43 +0200 Subject: [PATCH 160/204] Remove comment --- validphys2/src/validphys/photon/compute.py | 2 +- validphys2/src/validphys/photon/structure_functions.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 78574a66bc..915bd48d32 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -72,7 +72,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): self.xgrid = XGRID self.error_matrix = self.generate_error_matrix() - self.photons_array = [self.compute_photon_array(id) for id in self.replicas] + self.photons_array = [self.compute_photon_array(replica) for replica in self.replicas] self.interpolator = [ interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") for photon_array in self.photons_array diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index bff47d1f33..f365f9275d 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -43,8 +43,6 @@ def __init__(self, path_to_fktable, pdfs): pdgid = int(pdfs.set().get_entry("Particle")) predictions = self.fktable.convolute_with_one(pdgid, pdfs.xfxQ2) - # here we require that the (x,Q2) couples that we passed - # to pinefarm is a rectangular matrix grid2D = predictions.reshape(len(x), len(q2)) self.interpolator = RectBivariateSpline(x, q2, grid2D) From cc74266fe571fdc5a73a8442c6039326dbeb44ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 14:45:05 +0200 Subject: [PATCH 161/204] add constants --- validphys2/src/validphys/photon/constants.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 validphys2/src/validphys/photon/constants.py diff --git a/validphys2/src/validphys/photon/constants.py b/validphys2/src/validphys/photon/constants.py new file mode 100644 index 0000000000..d820fb12a9 --- /dev/null +++ b/validphys2/src/validphys/photon/constants.py @@ -0,0 +1,4 @@ +EU2 = 4.0 / 9 +ED2 = 1.0 / 9 +NL = 3 +NC = 3 From d3a6921aec4a5c0565a5d9120c10da1e8b4ec4e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 16:13:50 +0200 Subject: [PATCH 162/204] Use @property decorator for photon class --- n3fit/src/n3fit/msr.py | 2 +- validphys2/src/validphys/photon/compute.py | 32 ++++++++++++------- .../validphys/photon/structure_functions.py | 3 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index f924e66982..b43ec40bfb 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -75,7 +75,7 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): # 3.1 If a photon is given, compute the photon component of the MSR photons_c = None if photons: - photons_c = photons.integrate() + photons_c = photons.integral # 4. Now create the normalization by selecting the right integrations normalizer = MSR_Normalization(mode=mode, photons_contribution=photons_c) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 915bd48d32..f605423332 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -46,8 +46,12 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): fl = {} f2lo = {} for replica in replicas: - f2[replica] = sf.InterpStructureFunction(path_to_F2, self.qcd_pdfs.members[replica]) - fl[replica] = sf.InterpStructureFunction(path_to_FL, self.qcd_pdfs.members[replica]) + f2[replica] = sf.InterpStructureFunction( + path_to_F2, self.qcd_pdfs.members[replica] + ) + fl[replica] = sf.InterpStructureFunction( + path_to_FL, self.qcd_pdfs.members[replica] + ) f2lo[replica] = sf.F2LO(self.qcd_pdfs.members[replica], self.theory) with tempfile.NamedTemporaryFile(mode="w") as tmp: with tmp.file as tmp_file: @@ -67,12 +71,15 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): else 1e100, ] ) - self.lux[replica].PlugStructureFunctions(f2[replica].fxq, fl[replica].fxq, f2lo[replica].fxq) + self.lux[replica].PlugStructureFunctions( + f2[replica].fxq, fl[replica].fxq, f2lo[replica].fxq + ) self.xgrid = XGRID - self.error_matrix = self.generate_error_matrix() - self.photons_array = [self.compute_photon_array(replica) for replica in self.replicas] + self.photons_array = [ + self.compute_photon_array(replica) for replica in self.replicas + ] self.interpolator = [ interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") for photon_array in self.photons_array @@ -100,6 +107,7 @@ def compute_photon_array(self, replica): ) log.info(f"Computation time: {time.perf_counter() - start_time}") photon_qin += self.generate_errors(replica) + # fiatlux computes x * gamma(x) photon_qin /= self.xgrid # TODO : the different x points could be even computed in parallel @@ -146,17 +154,19 @@ def __call__(self, xgrid): """ return [ self.interpolator[id](xgrid[0, :, 0])[np.newaxis, :, np.newaxis] - for id in range(len(self.replicas_id)) + for id in range(len(self.replicas)) ] - def integrate(self): + @property + def integral(self): """Compute the integral of the photon on the x range.""" return [ trapezoid(self.photons_array[id], self.xgrid) - for id in range(len(self.replicas_id)) + for id in range(len(self.replicas)) ] - def generate_error_matrix(self): + @property + def error_matrix(self): """generate error matrix to be used for the additional errors.""" if not self.fiatlux_runcard["additional_errors"]: return None @@ -170,12 +180,12 @@ def generate_error_matrix(self): # first index must be x, while second one must be replica index return np.stack(res, axis=1) - def generate_errors(self, replica_id): + def generate_errors(self, replica): """generate LUX additional errors.""" log.info(f"Generating photon additional errors") if self.error_matrix is None: return np.zeros_like(self.xgrid) - seed = replica_luxseed(replica_id, self.fiatlux_runcard["luxseed"]) + seed = replica_luxseed(replica, self.fiatlux_runcard["luxseed"]) rng = np.random.default_rng(seed=seed) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) errors = u @ (s * rng.normal(size=7)) diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index f365f9275d..97996a73b8 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -41,8 +41,7 @@ def __init__(self, path_to_fktable, pdfs): x = np.unique(self.fktable.bin_left(1)) q2 = np.unique(self.fktable.bin_left(0)) - pdgid = int(pdfs.set().get_entry("Particle")) - predictions = self.fktable.convolute_with_one(pdgid, pdfs.xfxQ2) + predictions = self.fktable.convolute_with_one(2212, pdfs.xfxQ2) grid2D = predictions.reshape(len(x), len(q2)) self.interpolator = RectBivariateSpline(x, q2, grid2D) From 8da70599dd920b18f3a29efc79c8dc4b36d5bddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 18:16:41 +0200 Subject: [PATCH 163/204] Rename alpha_em_nlo --- validphys2/src/validphys/photon/compute.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index f605423332..2cea1b215b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -182,9 +182,9 @@ def error_matrix(self): def generate_errors(self, replica): """generate LUX additional errors.""" - log.info(f"Generating photon additional errors") if self.error_matrix is None: return np.zeros_like(self.xgrid) + log.info(f"Generating photon additional errors") seed = replica_luxseed(replica, self.fiatlux_runcard["luxseed"]) rng = np.random.default_rng(seed=seed) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) @@ -224,11 +224,11 @@ def alpha_em(self, q): nf = 5 else: nf = 6 - return self.alpha_em_nlo(q, self.alpha_thresh[nf], self.thresh[nf], nf) + return self.alpha_em_fixed_flavor(q, self.alpha_thresh[nf], self.thresh[nf], nf) - def alpha_em_nlo(self, q, alpha_ref, qref, nf): + def alpha_em_fixed_flavor(self, q, alpha_ref, qref, nf): """ - Compute the alpha_em running for FFS at NLO. + Compute the alpha_em running for nf fixed at NLO. Parameters ---------- @@ -293,12 +293,12 @@ def set_thresholds_alpha_em(self): # determine the values of alpha in the threshold points, depending on the value of qref for nf in range(nfref + 1, self.theory["MaxNfAs"] + 1): - alpha_thresh[nf] = self.alpha_em_nlo( + alpha_thresh[nf] = self.alpha_em_fixed_flavor( thresh[nf], alpha_thresh[nf - 1], thresh[nf - 1], nf - 1 ) for nf in reversed(range(3, nfref)): - alpha_thresh[nf] = self.alpha_em_nlo( + alpha_thresh[nf] = self.alpha_em_fixed_flavor( thresh[nf], alpha_thresh[nf + 1], thresh[nf + 1], nf + 1 ) From 905dc8d220a5517d53ab6c546622473e510456e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 18:25:52 +0200 Subject: [PATCH 164/204] Rename qcd_pdfs -> luxpdfset --- validphys2/src/validphys/photon/compute.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 2cea1b215b..4b98d2aad7 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -33,7 +33,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): self.replicas = replicas # structure functions - self.qcd_pdfs = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") + self.luxpdfset = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") # TODO : maybe find a different name for fiatlux_dis_F2 path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" @@ -47,12 +47,12 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): f2lo = {} for replica in replicas: f2[replica] = sf.InterpStructureFunction( - path_to_F2, self.qcd_pdfs.members[replica] + path_to_F2, self.luxpdfset.members[replica] ) fl[replica] = sf.InterpStructureFunction( - path_to_FL, self.qcd_pdfs.members[replica] + path_to_FL, self.luxpdfset.members[replica] ) - f2lo[replica] = sf.F2LO(self.qcd_pdfs.members[replica], self.theory) + f2lo[replica] = sf.F2LO(self.luxpdfset.members[replica], self.theory) with tempfile.NamedTemporaryFile(mode="w") as tmp: with tmp.file as tmp_file: tmp_file.write(yaml.dump(self.fiatlux_runcard)) @@ -122,10 +122,10 @@ def compute_photon_array(self, replica): if pid == 22: pdfs_init[j] = photon_qin ph_id = j - if pid not in self.qcd_pdfs.flavors: + if pid not in self.luxpdfset.flavors: continue pdfs_init[j] = np.array( - [self.qcd_pdfs.xfxQ(x, Q_IN, replica, pid) / x for x in self.xgrid] + [self.luxpdfset.xfxQ(x, Q_IN, replica, pid) / x for x in self.xgrid] ) # Apply EKO to PDFs From 1f8232d0fc41006790487076f85c22cf711e06de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 13 Apr 2023 21:52:41 +0200 Subject: [PATCH 165/204] Add reference in generate_errors --- validphys2/src/validphys/photon/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 4b98d2aad7..5fa7365575 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -1,7 +1,6 @@ """Script that calls fiatlux to add the photon PDF.""" import logging import tempfile -import time import fiatlux import numpy as np @@ -101,11 +100,9 @@ def compute_photon_array(self, replica): """ # Compute photon PDF log.info(f"Computing photon") - start_time = time.perf_counter() photon_qin = np.array( [self.lux[replica].EvaluatePhoton(x, Q_IN**2).total for x in self.xgrid] ) - log.info(f"Computation time: {time.perf_counter() - start_time}") photon_qin += self.generate_errors(replica) # fiatlux computes x * gamma(x) photon_qin /= self.xgrid @@ -167,7 +164,7 @@ def integral(self): @property def error_matrix(self): - """generate error matrix to be used for the additional errors.""" + """Generate error matrix to be used for the additional errors.""" if not self.fiatlux_runcard["additional_errors"]: return None extra_set = LHAPDFSet(EXTRA_SET, "replicas") @@ -181,7 +178,10 @@ def error_matrix(self): return np.stack(res, axis=1) def generate_errors(self, replica): - """generate LUX additional errors.""" + """ + Generate LUX additional errors according to the procedure + described in sec. 2.5 of https://arxiv.org/pdf/1712.07053.pdf + """ if self.error_matrix is None: return np.zeros_like(self.xgrid) log.info(f"Generating photon additional errors") From 8de6bac5dbe5d8ba38bff1716cd36318f460542a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 14 Apr 2023 11:22:18 +0200 Subject: [PATCH 166/204] Change Alpha class --- validphys2/src/validphys/photon/compute.py | 26 ++++++++++++++----- .../validphys/tests/photon/test_compute.py | 3 +-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 5fa7365575..0b301015c7 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -164,7 +164,7 @@ def integral(self): @property def error_matrix(self): - """Generate error matrix to be used for the additional errors.""" + """Generate error matrix to be used in generate_errors.""" if not self.fiatlux_runcard["additional_errors"]: return None extra_set = LHAPDFSet(EXTRA_SET, "replicas") @@ -194,15 +194,29 @@ def generate_errors(self, replica): class Alpha: def __init__(self, theory): - # parameters for the alphaem running self.theory = theory self.alpha_em_ref = theory["alphaqed"] - self.qref = self.theory.get("QrefQED", theory["Qref"]) + self.qref = self.theory.get("QrefQED") + # if QrefQED is specified in the runcard then alpha_em + # is running, otherwise it's fixed + + if self.qref: + self.beta0, self.b1 = self.set_betas() + self.thresh, self.alpha_thresh = self.set_thresholds_alpha_em() + self.alpha_em = self.running_alpha_em + else: + self.qref = 0.0 + # Putting qref to 0. for compatibility with fiatlux + self.alpha_em = self.fixed_alpha_em - self.beta0, self.b1 = self.set_betas() - self.thresh, self.alpha_thresh = self.set_thresholds_alpha_em() + def fixed_alpha_em(self, q): + """ + Return the fixed alpha_em. + Putting the q dependence only for compatibility with fiatlux. + """ + return self.alpha_em_ref - def alpha_em(self, q): + def running_alpha_em(self, q): r""" Compute the value of alpha_em. diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index ec464ad164..698de0c799 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -46,7 +46,7 @@ def __init__(self, runcard): self.f2 = None self.fl = None self.f2lo = None - self.res = photon(0, 0, 0) + self.res = photon(0., 0., 0.) def PlugAlphaQED(self, alphaem, qref): self.alphaem = alphaem @@ -90,7 +90,6 @@ def test_parameters_init(monkeypatch): monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) - monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) From fcb766bf84e4a72c0aedcf955265f13f991f2d6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 14 Apr 2023 15:12:31 +0200 Subject: [PATCH 167/204] Check if luxset is installed or not --- n3fit/runcards/examples/Basic_runcard_qed.yml | 2 +- n3fit/src/n3fit/checks.py | 6 +-- n3fit/src/n3fit/scripts/vp_setupfit.py | 3 ++ validphys2/src/validphys/config.py | 5 ++ validphys2/src/validphys/filters.py | 6 +++ validphys2/src/validphys/photon/compute.py | 54 +++++++++---------- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index 9841ce29c1..eda7c1724c 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -198,7 +198,7 @@ fiatlux: # general verbose: off - pdf_name: NNPDF40_nnlo_as_01180_1000 + luxset: NNPDF40_nnlo_as_01180_1000 additional_errors: true # should be set to true only for the last iteration luxseed: 1234567890 \ No newline at end of file diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 57213be98f..2cdeb760b8 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -398,8 +398,8 @@ def check_deprecated_options(fitting): @make_argcheck def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): if fiatlux : - pdf_name = fiatlux["pdf_name"] - pdfs_ids = PDF(pdf_name).get_members() - 1 # get_members counts also replica0 + luxset = fiatlux["luxset"] + pdfs_ids = PDF(luxset).get_members() - 1 # get_members counts also replica0 max_id = max(replicas) if max_id > pdfs_ids : for replica_id in replicas: @@ -407,6 +407,6 @@ def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): os.rmdir(replica_path / f"replica_{replica_id}") raise CheckError( f"Cannot generate a photon replica with id larger than the number of replicas of the PDFs set " - + pdf_name + f": replica id={max_id}, replicas of " + pdf_name + f"={pdfs_ids}" + + luxset + f": replica id={max_id}, replicas of " + luxset + f"={pdfs_ids}" +"\nRemoving replica output folders" ) diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index 19722802af..3800b964aa 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -157,6 +157,9 @@ def from_yaml(cls, o, *args, **kwargs): if file_content.get('theorycovmatconfig') is not None: SETUPFIT_FIXED_CONFIG['actions_'].append( 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat') + if file_content.get('fiatlux') is not None: + SETUPFIT_FIXED_CONFIG['actions_'].append( + 'fiatlux check_luxset') for k,v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) file_content.update(SETUPFIT_FIXED_CONFIG) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index f82781536c..47af0548cc 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -1039,6 +1039,11 @@ def produce_t0set( raise ConfigError("Setting use_t0 requires specifying a valid t0pdfset") return t0pdfset return None + + def parse_luxset(self, name): + """PDF set used to generate the photon with fiatlux.""" + return self.parse_pdf(name) + def parse_fakepdf(self, name): """PDF set used to generate the fake data in a closure test.""" diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index 2532ca2c00..ac16651971 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -276,6 +276,12 @@ def check_t0pdfset(t0pdfset): log.info(f'{t0pdfset} T0 checked.') +def check_luxset(luxset): + """Lux pdf check""" + luxset.load() + log.info(f'{luxset} Lux pdf checked.') + + def check_positivity(posdatasets): """Verify positive datasets are ready for the fit.""" log.info('Verifying positivity tables:') diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 0b301015c7..5cba81babe 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -26,13 +26,13 @@ class Photon: """Photon class computing the photon array with the LuxQED approach.""" def __init__(self, theoryid, fiatlux_runcard, replicas): - self.theory = theoryid.get_description() + theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard - self.fiatlux_runcard["qed_running"] = "QrefQED" in self.theory + self.fiatlux_runcard["qed_running"] = "QrefQED" in theory self.replicas = replicas # structure functions - self.luxpdfset = LHAPDFSet(fiatlux_runcard["pdf_name"], "replicas") + self.luxpdfset = LHAPDFSet(fiatlux_runcard["luxset"], "replicas") # TODO : maybe find a different name for fiatlux_dis_F2 path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" @@ -51,7 +51,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): fl[replica] = sf.InterpStructureFunction( path_to_FL, self.luxpdfset.members[replica] ) - f2lo[replica] = sf.F2LO(self.luxpdfset.members[replica], self.theory) + f2lo[replica] = sf.F2LO(self.luxpdfset.members[replica], theory) with tempfile.NamedTemporaryFile(mode="w") as tmp: with tmp.file as tmp_file: tmp_file.write(yaml.dump(self.fiatlux_runcard)) @@ -59,14 +59,14 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): # we have a dict but fiatlux wants a yaml file # TODO : once that fiatlux will allow dictionaries # pass directly self.fiatlux_runcard - alpha = Alpha(self.theory) + alpha = Alpha(theory) for replica in replicas: self.lux[replica].PlugAlphaQED(alpha.alpha_em, alpha.qref) self.lux[replica].InsertInelasticSplitQ( [ - self.theory["kbThr"] * self.theory["mb"], - self.theory["ktThr"] * self.theory["mt"] - if self.theory["MaxNfPdf"] == 6 + theory["kbThr"] * theory["mb"], + theory["ktThr"] * theory["mt"] + if theory["MaxNfPdf"] == 6 else 1e100, ] ) @@ -74,14 +74,16 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): f2[replica].fxq, fl[replica].fxq, f2lo[replica].fxq ) - self.xgrid = XGRID - - self.photons_array = [ + photons_array = [ self.compute_photon_array(replica) for replica in self.replicas ] self.interpolator = [ - interp1d(self.xgrid, photon_array, fill_value=0.0, kind="cubic") - for photon_array in self.photons_array + interp1d(XGRID, photon_array, fill_value=0.0, kind="cubic") + for photon_array in photons_array + ] + self.integral = [ + trapezoid(photons_array[id], XGRID) + for id in range(len(self.replicas)) ] def compute_photon_array(self, replica): @@ -101,11 +103,11 @@ def compute_photon_array(self, replica): # Compute photon PDF log.info(f"Computing photon") photon_qin = np.array( - [self.lux[replica].EvaluatePhoton(x, Q_IN**2).total for x in self.xgrid] + [self.lux[replica].EvaluatePhoton(x, Q_IN**2).total for x in XGRID] ) photon_qin += self.generate_errors(replica) # fiatlux computes x * gamma(x) - photon_qin /= self.xgrid + photon_qin /= XGRID # TODO : the different x points could be even computed in parallel # Load eko and reshape it @@ -114,7 +116,7 @@ def compute_photon_array(self, replica): # it has to be done inside vp-setupfit # construct PDFs - pdfs_init = np.zeros((len(eko.rotations.inputpids), len(self.xgrid))) + pdfs_init = np.zeros((len(eko.rotations.inputpids), len(XGRID))) for j, pid in enumerate(eko.rotations.inputpids): if pid == 22: pdfs_init[j] = photon_qin @@ -122,7 +124,7 @@ def compute_photon_array(self, replica): if pid not in self.luxpdfset.flavors: continue pdfs_init[j] = np.array( - [self.luxpdfset.xfxQ(x, Q_IN, replica, pid) / x for x in self.xgrid] + [self.luxpdfset.xfxQ(x, Q_IN, replica, pid) / x for x in XGRID] ) # Apply EKO to PDFs @@ -133,7 +135,7 @@ def compute_photon_array(self, replica): photon_Q0 = pdfs_final[ph_id] # we want x * gamma(x) - return self.xgrid * photon_Q0 + return XGRID * photon_Q0 def __call__(self, xgrid): """ @@ -154,25 +156,17 @@ def __call__(self, xgrid): for id in range(len(self.replicas)) ] - @property - def integral(self): - """Compute the integral of the photon on the x range.""" - return [ - trapezoid(self.photons_array[id], self.xgrid) - for id in range(len(self.replicas)) - ] - @property def error_matrix(self): """Generate error matrix to be used in generate_errors.""" if not self.fiatlux_runcard["additional_errors"]: return None extra_set = LHAPDFSet(EXTRA_SET, "replicas") - qs = [Q_IN] * len(self.xgrid) - res_central = np.array(extra_set.central_member.xfxQ(22, self.xgrid, qs)) + qs = [Q_IN] * len(XGRID) + res_central = np.array(extra_set.central_member.xfxQ(22, XGRID, qs)) res = [] for idx_member in range(101, 107 + 1): - tmp = np.array(extra_set.members[idx_member].xfxQ(22, self.xgrid, qs)) + tmp = np.array(extra_set.members[idx_member].xfxQ(22, XGRID, qs)) res.append(tmp - res_central) # first index must be x, while second one must be replica index return np.stack(res, axis=1) @@ -183,7 +177,7 @@ def generate_errors(self, replica): described in sec. 2.5 of https://arxiv.org/pdf/1712.07053.pdf """ if self.error_matrix is None: - return np.zeros_like(self.xgrid) + return np.zeros_like(XGRID) log.info(f"Generating photon additional errors") seed = replica_luxseed(replica, self.fiatlux_runcard["luxseed"]) rng = np.random.default_rng(seed=seed) From bbc6d293ec2c8942b7fd38b9790358dc48305275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 19 Apr 2023 11:35:50 +0200 Subject: [PATCH 168/204] Fix bug in photon/compute --- n3fit/src/n3fit/layers/msr_normalization.py | 2 +- validphys2/src/validphys/photon/compute.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index ea6f511c0a..a684ccd688 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -42,7 +42,7 @@ def __init__(self, output_dim=14, mode="ALL", photons_contribution=None, **kwarg def call(self, pdf_integrated, ph_replica): """Imposes the valence and momentum sum rules: - A_g = (1-sigma)/g + A_g = (1-sigma-photon)/g A_v = A_v24 = A_v35 = 3/V A_v3 = 1/V_3 A_v8 = 3/V_8 diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 5cba81babe..fe3e78ef26 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -121,11 +121,12 @@ def compute_photon_array(self, replica): if pid == 22: pdfs_init[j] = photon_qin ph_id = j - if pid not in self.luxpdfset.flavors: - continue - pdfs_init[j] = np.array( - [self.luxpdfset.xfxQ(x, Q_IN, replica, pid) / x for x in XGRID] - ) + else : + if pid not in self.luxpdfset.flavors: + continue + pdfs_init[j] = np.array( + [self.luxpdfset.xfxQ(x, Q_IN, replica, pid) / x for x in XGRID] + ) # Apply EKO to PDFs q2 = eko.mu2grid[0] From 0381481e6fb182f727b21ce0d0bce77b5657abb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 19 Apr 2023 16:48:31 +0200 Subject: [PATCH 169/204] Remove parse_luxset --- n3fit/src/n3fit/scripts/vp_setupfit.py | 3 --- validphys2/src/validphys/config.py | 4 ---- validphys2/src/validphys/filters.py | 7 ------- 3 files changed, 14 deletions(-) diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index 3800b964aa..19722802af 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -157,9 +157,6 @@ def from_yaml(cls, o, *args, **kwargs): if file_content.get('theorycovmatconfig') is not None: SETUPFIT_FIXED_CONFIG['actions_'].append( 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat') - if file_content.get('fiatlux') is not None: - SETUPFIT_FIXED_CONFIG['actions_'].append( - 'fiatlux check_luxset') for k,v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) file_content.update(SETUPFIT_FIXED_CONFIG) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index 47af0548cc..116e91cd98 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -1040,10 +1040,6 @@ def produce_t0set( return t0pdfset return None - def parse_luxset(self, name): - """PDF set used to generate the photon with fiatlux.""" - return self.parse_pdf(name) - def parse_fakepdf(self, name): """PDF set used to generate the fake data in a closure test.""" diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index ac16651971..ba6a7f8c91 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -275,13 +275,6 @@ def check_t0pdfset(t0pdfset): t0pdfset.load() log.info(f'{t0pdfset} T0 checked.') - -def check_luxset(luxset): - """Lux pdf check""" - luxset.load() - log.info(f'{luxset} Lux pdf checked.') - - def check_positivity(posdatasets): """Verify positive datasets are ready for the fit.""" log.info('Verifying positivity tables:') From c446156cb41c900793f1acb2ecaf35f74e3764a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 2 May 2023 14:26:24 +0200 Subject: [PATCH 170/204] Fix tests and add basis LUX --- validphys2/src/validphys/pdfbases.py | 30 ++++++++ validphys2/src/validphys/photon/compute.py | 1 + .../validphys/tests/photon/test_compute.py | 69 ++++++++----------- 3 files changed, 60 insertions(+), 40 deletions(-) diff --git a/validphys2/src/validphys/pdfbases.py b/validphys2/src/validphys/pdfbases.py index aac0f01ef1..15b2e1b94d 100644 --- a/validphys2/src/validphys/pdfbases.py +++ b/validphys2/src/validphys/pdfbases.py @@ -523,8 +523,38 @@ def f_(transform_func): default_elements=(r'\Sigma', 'V', 'T3', 'V3', 'T8', 'V8', 'T15', 'gluon', ) ) +lux = LinearBasis.from_mapping({ + r'\Sigma' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1 ,'b':1, 'bbar': 1, 't': 1, 'tbar': 1}, + 'V' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c': 1, 'cbar':-1 ,'b':1, 'bbar':-1, 't': 1, 'tbar':-1}, + + 'T3' : {'u': 1, 'ubar': 1, 'd':-1, 'dbar':-1}, + 'V3' : {'u': 1, 'ubar':-1, 'd':-1, 'dbar': 1}, + + 'T8' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's':-2, 'sbar':-2}, + 'V8' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's':-2, 'sbar':+2}, + + 'T15' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c':-3, 'cbar':-3}, + 'V15' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c':-3, 'cbar':+3}, + + + 'T24' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1, 'b':-4, 'bbar':-4}, + 'V24' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c': 1, 'cbar':-1, 'b':-4, 'bbar':+4}, + + 'T35' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1, 'b': 1, 'bbar': 1, 't':-5, 'tbar':-5}, + 'V35' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c': 1, 'cbar':-1, 'b': 1, 'bbar':-1, 't':-5, 'tbar':+5}, + + 'g' : {'g':1}, + 'photon' : {'photon':1}, + }, + aliases = {'gluon':'g', 'singlet': r'\Sigma', 'sng': r'\Sigma', 'sigma': r'\Sigma', + 'v': 'V', 'v3': 'V3', 'v8': 'V8', 't3': 'T3', 't8': 'T8', 't15': 'T15', 'v15': 'V15',}, + default_elements=(r'\Sigma', 'V', 'T3', 'V3', 'T8', 'V8', 'T15', 'gluon', 'photon') +) + EVOL = evolution +LUX = lux + CCBAR_ASYMM = copy.deepcopy(evolution) CCBAR_ASYMM.default_elements = (r'\Sigma', 'V', 'T3', 'V3', 'T8', 'V8', 'T15', 'gluon', 'V15') diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index fe3e78ef26..a37c71a487 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -29,6 +29,7 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): theory = theoryid.get_description() self.fiatlux_runcard = fiatlux_runcard self.fiatlux_runcard["qed_running"] = "QrefQED" in theory + # TODO: QrefQED is going to be removed from the runcard self.replicas = replicas # structure functions diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 698de0c799..154bfe5521 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -4,7 +4,7 @@ import fiatlux import numpy as np from validphys.photon import structure_functions -from validphys.photon.compute import Photon +from validphys.photon.compute import Photon, Alpha from ..conftest import PDF @@ -17,6 +17,7 @@ def get_description(self): return { "alphaqed": 0.01, "Qref": 91.2, + "QrefQED": 91.2, "mc": 1.3, "mb": 4.92, "mt": 172.0, @@ -29,7 +30,7 @@ def get_description(self): fiatlux_runcard = { - "pdf_name": PDF, + "luxset": PDF, "additional_errors": False, } @@ -88,30 +89,23 @@ def test_parameters_init(monkeypatch): structure_functions, "InterpStructureFunction", FakeStructureFunction ) monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) - monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) + monkeypatch.setattr(Photon, "compute_photon_array", lambda *args: np.zeros(196)) photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) + alpha = Alpha(FakeTheory().get_description()) - np.testing.assert_equal(photon.replicas_id, [1, 2, 3]) + np.testing.assert_equal(photon.replicas, [1, 2, 3]) np.testing.assert_equal(photon.fiatlux_runcard, fiatlux_runcard) - np.testing.assert_almost_equal(photon.q_in2, 1e4) np.testing.assert_almost_equal( - photon.alpha_em_ref, FakeTheory().get_description()["alphaqed"] + alpha.alpha_em_ref, FakeTheory().get_description()["alphaqed"] ) -def test_masses_init(monkeypatch): - monkeypatch.setattr( - structure_functions, "InterpStructureFunction", FakeStructureFunction - ) - monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) - - monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) - monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) - photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) - np.testing.assert_equal(photon.thresh_t, np.inf) - np.testing.assert_almost_equal(photon.thresh_b, 4.92) - np.testing.assert_almost_equal(photon.thresh_c, 1.3) +def test_masses_init(): + alpha = Alpha(FakeTheory().get_description()) + np.testing.assert_equal(alpha.thresh_t, np.inf) + np.testing.assert_almost_equal(alpha.thresh_b, 4.92) + np.testing.assert_almost_equal(alpha.thresh_c, 1.3) def test_set_thresholds_alpha_em(monkeypatch): monkeypatch.setattr( @@ -120,31 +114,26 @@ def test_set_thresholds_alpha_em(monkeypatch): monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) - monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) - photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) - np.testing.assert_almost_equal(photon.thresh[5], 91.2) - np.testing.assert_almost_equal(photon.thresh[4], 4.92) - np.testing.assert_almost_equal(photon.thresh[3], 1.3) - np.testing.assert_almost_equal(photon.alpha_thresh[5], 0.01) + monkeypatch.setattr(Photon, "compute_photon_array", lambda *args: np.zeros(196)) + + alpha = Alpha(FakeTheory().get_description()) + + np.testing.assert_almost_equal(alpha.thresh[5], 91.2) + np.testing.assert_almost_equal(alpha.thresh[4], 4.92) + np.testing.assert_almost_equal(alpha.thresh[3], 1.3) + np.testing.assert_almost_equal(alpha.alpha_thresh[5], 0.01) np.testing.assert_almost_equal( - photon.alpha_thresh[4], photon.alpha_em_nlo(4.92, 0.01, 91.2, 5) + alpha.alpha_thresh[4], alpha.alpha_em_fixed_flavor(4.92, 0.01, 91.2, 5) ) np.testing.assert_almost_equal( - photon.alpha_thresh[3], - photon.alpha_em_nlo(1.3, photon.alpha_thresh[4], 4.92, 4), + alpha.alpha_thresh[3], + alpha.alpha_em_fixed_flavor(1.3, alpha.alpha_thresh[4], 4.92, 4), ) - np.testing.assert_equal(len(photon.alpha_thresh), 3) - np.testing.assert_equal(len(photon.thresh), 3) + np.testing.assert_equal(len(alpha.alpha_thresh), 3) + np.testing.assert_equal(len(alpha.thresh), 3) -def test_betas(monkeypatch): - monkeypatch.setattr( - structure_functions, "InterpStructureFunction", FakeStructureFunction - ) - monkeypatch.setattr(structure_functions, "F2LO", FakeF2LO) - - monkeypatch.setattr(fiatlux, "FiatLux", FakeFiatlux) - monkeypatch.setattr(Photon, "produce_interpolators", lambda *args: None) - photon = Photon(FakeTheory(), fiatlux_runcard, [1, 2, 3]) +def test_betas(): + alpha = Alpha(FakeTheory().get_description()) vec_beta0 = [ -0.5305164769729844, -0.6719875374991137, @@ -158,5 +147,5 @@ def test_betas(monkeypatch): 0.1458920311675707, ] for nf in range(3, 6 + 1): - np.testing.assert_allclose(photon.beta0[nf], vec_beta0[nf - 3], rtol=1e-7) - np.testing.assert_allclose(photon.b1[nf], vec_b1[nf - 3], rtol=1e-7) + np.testing.assert_allclose(alpha.beta0[nf], vec_beta0[nf - 3], rtol=1e-7) + np.testing.assert_allclose(alpha.b1[nf], vec_b1[nf - 3], rtol=1e-7) From 2e5cf4dd59cddf7094e21222eae48691392dd1f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 2 May 2023 15:27:21 +0200 Subject: [PATCH 171/204] Fix test_layers.py --- n3fit/src/n3fit/tests/test_layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/tests/test_layers.py b/n3fit/src/n3fit/tests/test_layers.py index c3a78f2dbb..b8cbb844a0 100644 --- a/n3fit/src/n3fit/tests/test_layers.py +++ b/n3fit/src/n3fit/tests/test_layers.py @@ -269,7 +269,7 @@ def test_addphoton_init(): np.testing.assert_equal(addphoton._pdf_ph, None) class FakePhoton(): - def compute(self, xgrid): + def __call__(self, xgrid): return [np.exp(-xgrid)] def test_compute_photon(): From 3fda19240fa80db967e5024393c0576df7269ea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 4 May 2023 18:22:20 +0200 Subject: [PATCH 172/204] Parse luxset --- n3fit/runcards/examples/Basic_runcard_qed.yml | 34 +------------ n3fit/src/n3fit/checks.py | 2 +- n3fit/src/n3fit/model_trainer.py | 12 ++--- n3fit/src/n3fit/performfit.py | 6 +-- n3fit/src/n3fit/scripts/vp_setupfit.py | 3 ++ validphys2/src/validphys/config.py | 4 ++ validphys2/src/validphys/filters.py | 5 ++ validphys2/src/validphys/photon/compute.py | 49 ++++++++++++++++--- 8 files changed, 64 insertions(+), 51 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index eda7c1724c..f840ca2bd6 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -89,7 +89,7 @@ dataset_inputs: ############################################################ datacuts: - t0pdfset: NNPDF40_nnlo_as_01180_1000 # PDF set to generate t0 covmat + t0pdfset: NNPDF40_nnlo_as_01180 # PDF set to generate t0 covmat q2min: 3.49 # Q2 minimum w2min: 12.5 # W2 minimum @@ -166,38 +166,6 @@ debug: True maxcores: 8 fiatlux: - apfel: off - - q2_max: 1e8 # the maximum allowed Q2. - eps_base: 1e-5 # precision on final integration of double integral. - eps_rel: 1e-1 # extra precision on any single integration. - mproton: 0.938272046 # proton mass from 2015 update of PDG review - mum_proton: 2.792847356 # proton magnetic moment, from - # http://pdglive.lbl.gov/DataBlock.action?node=S016MM which itself - # gets it from arXiv:1203.5425 (CODATA) - - # the elastic param type, options: - # dipole - # A1_world_spline - # A1_world_pol_spline - elastic_param: A1_world_pol_spline - elastic_electric_rescale: 1 - elastic_magnetic_rescale: 1 - - # the inelastic param type, options: - inelastic_param: LHAPDF_Hermes_ALLM_CLAS # Hermes_ALLM_CLAS, LHAPDF_Hermes_ALLM_CLAS - rescale_r_twist4: 0 - rescale_r: 1 - allm_limits: 0 - rescale_non_resonance: 1 - rescale_resonance: 1 - use_mu2_as_upper_limit: off - q2min_inel_override: 0.0 - q2max_inel_override: 1E300 - lhapdf_transition_q2: 9 - - # general - verbose: off luxset: NNPDF40_nnlo_as_01180_1000 additional_errors: true # should be set to true only for the last iteration luxseed: 1234567890 diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 2cdeb760b8..0b87a9af50 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -399,7 +399,7 @@ def check_deprecated_options(fitting): def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): if fiatlux : luxset = fiatlux["luxset"] - pdfs_ids = PDF(luxset).get_members() - 1 # get_members counts also replica0 + pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 max_id = max(replicas) if max_id > pdfs_ids : for replica_id in replicas: diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 9e22b35d84..b68c448fe4 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -101,7 +101,7 @@ def __init__( sum_rules=None, parallel_models=1, theoryid=None, - fiatlux_runcard=None, + lux_params=None, replicas=None, ): """ @@ -137,8 +137,8 @@ def __init__( number of models to fit in parallel theoryid : validphys.core.TheoryIDSpec object contining info for generating the photon - fiatlux_runcard : dict - dictionary containing the fiatlux runcard + lux_params : dict + dictionary containing the params needed from LuxQED replicas_id : list list with the replicas ids to be fitted """ @@ -162,7 +162,7 @@ def __init__( self._scaler = None self._parallel_models = parallel_models self.theoryid = theoryid - self.fiatlux_runcard = fiatlux_runcard + self.lux_params = lux_params self.replicas = replicas # Initialise internal variables which define behaviour @@ -885,10 +885,10 @@ def hyperparametrizable(self, params): # Generate the grid in x, note this is the same for all partitions xinput = self._xgrid_generation() # Initialize all photon classes for the different replicas: - if self.fiatlux_runcard: + if self.lux_params: photons=Photon( theoryid=self.theoryid, - fiatlux_runcard=self.fiatlux_runcard, + lux_params=self.lux_params, replicas=self.replicas, ) else: diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 6c9a9223f3..bc34b95372 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -86,8 +86,8 @@ def performfit( Theory which is used to generate theory predictions from model during fit. Object also contains some metadata on the theory settings. - fiatlux : dict - dictionary containing the fiatlux runcard + lux_params : dict + dictionary containing the params needed from LuxQED basis: list[dict] preprocessing information for each flavour to be fitted. fitbasis: str @@ -199,7 +199,7 @@ def performfit( sum_rules=sum_rules, parallel_models=n_models, theoryid=theoryid, - fiatlux_runcard=fiatlux, + lux_params=fiatlux, replicas=replica_idxs, ) diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index 19722802af..3800b964aa 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -157,6 +157,9 @@ def from_yaml(cls, o, *args, **kwargs): if file_content.get('theorycovmatconfig') is not None: SETUPFIT_FIXED_CONFIG['actions_'].append( 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat') + if file_content.get('fiatlux') is not None: + SETUPFIT_FIXED_CONFIG['actions_'].append( + 'fiatlux check_luxset') for k,v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) file_content.update(SETUPFIT_FIXED_CONFIG) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index 116e91cd98..c9c062f1b9 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -1040,6 +1040,10 @@ def produce_t0set( return t0pdfset return None + def parse_luxset(self, name): + """PDF set used to generate the photon with fiatlux.""" + return self.parse_pdf(name) + def parse_fakepdf(self, name): """PDF set used to generate the fake data in a closure test.""" diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index ba6a7f8c91..63bee3e4a9 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -275,6 +275,11 @@ def check_t0pdfset(t0pdfset): t0pdfset.load() log.info(f'{t0pdfset} T0 checked.') +def check_luxset(luxset): + """Lux pdf check""" + luxset.load() + log.info(f'{luxset} Lux pdf checked.') + def check_positivity(posdatasets): """Verify positive datasets are ready for the fit.""" log.info('Verifying positivity tables:') diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index a37c71a487..87e9a193ca 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -20,20 +20,53 @@ Q_IN = 100 EXTRA_SET = "LUXqed17_plus_PDF4LHC15_nnlo_100" +FIATLUX_DEFAULT = { + "apfel" : False, + "q2_max" : 1e8, # the maximum allowed Q2. + "eps_base" : 1e-5, # precision on final integration of double integral. + "eps_rel" : 1e-1, # extra precision on any single integration. + "mum_proton" : 2.792847356, # proton magnetic moment, from + # http://pdglive.lbl.gov/DataBlock.action?node=S016MM which itself + # gets it from arXiv:1203.5425 (CODATA) + + # the elastic param type, options: + # dipole + # A1_world_spline + # A1_world_pol_spline + "elastic_param" : "A1_world_pol_spline", + "elastic_electric_rescale": 1, + "elastic_magnetic_rescale": 1, + # the inelastic param type, options: + "inelastic_param" : "LHAPDF_Hermes_ALLM_CLAS", # Hermes_ALLM_CLAS, LHAPDF_Hermes_ALLM_CLAS + "rescale_r_twist4" : 0, + "rescale_r" : 1, + "allm_limits" : 0, + "rescale_non_resonance" : 1, + "rescale_resonance" : 1, + "use_mu2_as_upper_limit" : False, + "q2min_inel_override" : 0.0, + "q2max_inel_override" : 1E300, + "lhapdf_transition_q2" : 9, + # general + "verbose" : False, +} class Photon: """Photon class computing the photon array with the LuxQED approach.""" - def __init__(self, theoryid, fiatlux_runcard, replicas): + def __init__(self, theoryid, lux_params, replicas): theory = theoryid.get_description() - self.fiatlux_runcard = fiatlux_runcard - self.fiatlux_runcard["qed_running"] = "QrefQED" in theory + fiatlux_runcard = FIATLUX_DEFAULT + fiatlux_runcard["qed_running"] = "QrefQED" in theory # TODO: QrefQED is going to be removed from the runcard + fiatlux_runcard["mproton"] = theory["MP"] self.replicas = replicas # structure functions - self.luxpdfset = LHAPDFSet(fiatlux_runcard["luxset"], "replicas") + self.luxpdfset = lux_params["luxset"].load() + self.additional_errors = lux_params["additional_errors"] + self.luxseed = lux_params["luxseed"] # TODO : maybe find a different name for fiatlux_dis_F2 path_to_F2 = theoryid.path / "fastkernel/fiatlux_dis_F2.pineappl.lz4" @@ -55,11 +88,11 @@ def __init__(self, theoryid, fiatlux_runcard, replicas): f2lo[replica] = sf.F2LO(self.luxpdfset.members[replica], theory) with tempfile.NamedTemporaryFile(mode="w") as tmp: with tmp.file as tmp_file: - tmp_file.write(yaml.dump(self.fiatlux_runcard)) + tmp_file.write(yaml.dump(fiatlux_runcard)) self.lux[replica] = fiatlux.FiatLux(tmp_file.name) # we have a dict but fiatlux wants a yaml file # TODO : once that fiatlux will allow dictionaries - # pass directly self.fiatlux_runcard + # pass directly fiatlux_runcard alpha = Alpha(theory) for replica in replicas: self.lux[replica].PlugAlphaQED(alpha.alpha_em, alpha.qref) @@ -161,7 +194,7 @@ def __call__(self, xgrid): @property def error_matrix(self): """Generate error matrix to be used in generate_errors.""" - if not self.fiatlux_runcard["additional_errors"]: + if not self.additional_errors: return None extra_set = LHAPDFSet(EXTRA_SET, "replicas") qs = [Q_IN] * len(XGRID) @@ -181,7 +214,7 @@ def generate_errors(self, replica): if self.error_matrix is None: return np.zeros_like(XGRID) log.info(f"Generating photon additional errors") - seed = replica_luxseed(replica, self.fiatlux_runcard["luxseed"]) + seed = replica_luxseed(replica, self.luxseed) rng = np.random.default_rng(seed=seed) u, s, _ = np.linalg.svd(self.error_matrix, full_matrices=False) errors = u @ (s * rng.normal(size=7)) From 4b2ea77705c82be03aa8019d12a85c301b4c6508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 4 May 2023 18:35:40 +0200 Subject: [PATCH 173/204] Fix test_compute --- validphys2/src/validphys/tests/photon/test_compute.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 154bfe5521..73ce223507 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -5,6 +5,7 @@ import numpy as np from validphys.photon import structure_functions from validphys.photon.compute import Photon, Alpha +from validphys.core import PDF as PDFset from ..conftest import PDF @@ -26,12 +27,14 @@ def get_description(self): "ktThr": 1.0, "MaxNfAs": 5, "MaxNfPdf": 5, + "MP": 0.938 } fiatlux_runcard = { - "luxset": PDF, + "luxset": PDFset(PDF), "additional_errors": False, + "luxseed": 123456789 } photon = namedtuple("photon", ["total", "elastic", "inelastic"]) @@ -96,7 +99,9 @@ def test_parameters_init(monkeypatch): alpha = Alpha(FakeTheory().get_description()) np.testing.assert_equal(photon.replicas, [1, 2, 3]) - np.testing.assert_equal(photon.fiatlux_runcard, fiatlux_runcard) + np.testing.assert_equal(photon.luxpdfset._name, fiatlux_runcard["luxset"].name) + np.testing.assert_equal(photon.additional_errors, fiatlux_runcard["additional_errors"]) + np.testing.assert_equal(photon.luxseed, fiatlux_runcard["luxseed"]) np.testing.assert_almost_equal( alpha.alpha_em_ref, FakeTheory().get_description()["alphaqed"] ) From c25c51d50616762b09de4e4b8f6fe1bf8de5d661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 5 May 2023 12:40:07 +0200 Subject: [PATCH 174/204] Parse additional_errors --- n3fit/runcards/examples/Basic_runcard_qed.yml | 2 +- n3fit/src/n3fit/scripts/vp_setupfit.py | 3 +++ validphys2/src/validphys/config.py | 6 ++++++ validphys2/src/validphys/filters.py | 5 +++++ validphys2/src/validphys/photon/compute.py | 4 +--- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index f840ca2bd6..5d02bf3db3 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -166,7 +166,7 @@ debug: True maxcores: 8 fiatlux: - luxset: NNPDF40_nnlo_as_01180_1000 + luxset: NNPDF40_nnlo_as_01180 additional_errors: true # should be set to true only for the last iteration luxseed: 1234567890 \ No newline at end of file diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index 3800b964aa..d6a4f4b509 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -160,6 +160,9 @@ def from_yaml(cls, o, *args, **kwargs): if file_content.get('fiatlux') is not None: SETUPFIT_FIXED_CONFIG['actions_'].append( 'fiatlux check_luxset') + if file_content.get('fiatlux')["additional_errors"]: + SETUPFIT_FIXED_CONFIG['actions_'].append( + 'fiatlux check_additional_errors') for k,v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) file_content.update(SETUPFIT_FIXED_CONFIG) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index c9c062f1b9..873b7245a8 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -1044,6 +1044,12 @@ def parse_luxset(self, name): """PDF set used to generate the photon with fiatlux.""" return self.parse_pdf(name) + def parse_additional_errors(self, bool): + """PDF set used to generate the photon additional errors.""" + if bool: + return self.parse_pdf("LUXqed17_plus_PDF4LHC15_nnlo_100") + else: + return False def parse_fakepdf(self, name): """PDF set used to generate the fake data in a closure test.""" diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index 63bee3e4a9..a1d6bf685d 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -280,6 +280,11 @@ def check_luxset(luxset): luxset.load() log.info(f'{luxset} Lux pdf checked.') +def check_additional_errors(additional_errors): + """Lux additional errors pdf check""" + additional_errors.load() + log.info(f'{additional_errors} Lux additional errors pdf checked.') + def check_positivity(posdatasets): """Verify positive datasets are ready for the fit.""" log.info('Verifying positivity tables:') diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 87e9a193ca..ea5d792fd2 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -8,7 +8,6 @@ from eko.io import EKO from scipy.integrate import trapezoid from scipy.interpolate import interp1d -from validphys.lhapdfset import LHAPDFSet from validphys.n3fit_data import replica_luxseed from n3fit.io.writer import XGRID @@ -19,7 +18,6 @@ log = logging.getLogger(__name__) Q_IN = 100 -EXTRA_SET = "LUXqed17_plus_PDF4LHC15_nnlo_100" FIATLUX_DEFAULT = { "apfel" : False, "q2_max" : 1e8, # the maximum allowed Q2. @@ -196,7 +194,7 @@ def error_matrix(self): """Generate error matrix to be used in generate_errors.""" if not self.additional_errors: return None - extra_set = LHAPDFSet(EXTRA_SET, "replicas") + extra_set = self.additional_errors.load() qs = [Q_IN] * len(XGRID) res_central = np.array(extra_set.central_member.xfxQ(22, XGRID, qs)) res = [] From 9fae2eec96955aa7b6553140e1019ce44f4dbf77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 5 May 2023 14:33:18 +0200 Subject: [PATCH 175/204] Change calling of alphaem running --- validphys2/src/validphys/photon/compute.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index ea5d792fd2..b7b416cbd2 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -32,8 +32,8 @@ # A1_world_spline # A1_world_pol_spline "elastic_param" : "A1_world_pol_spline", - "elastic_electric_rescale": 1, - "elastic_magnetic_rescale": 1, + "elastic_electric_rescale" : 1, + "elastic_magnetic_rescale" : 1, # the inelastic param type, options: "inelastic_param" : "LHAPDF_Hermes_ALLM_CLAS", # Hermes_ALLM_CLAS, LHAPDF_Hermes_ALLM_CLAS "rescale_r_twist4" : 0, @@ -56,8 +56,10 @@ class Photon: def __init__(self, theoryid, lux_params, replicas): theory = theoryid.get_description() fiatlux_runcard = FIATLUX_DEFAULT - fiatlux_runcard["qed_running"] = "QrefQED" in theory - # TODO: QrefQED is going to be removed from the runcard + fiatlux_runcard["qed_running"] = np.isclose(theory["Qedref"], theory["Qref"]) + # TODO: for the time being, we trigger alphaem running if Qedref=Qref. + # This is going to be changed in favor of a bool em_running + # in the runcard fiatlux_runcard["mproton"] = theory["MP"] self.replicas = replicas From eeb068312ee47b745c3e0201dce2fa006e207d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 5 May 2023 15:49:05 +0200 Subject: [PATCH 176/204] Fix dumping of fiatlux runcard --- n3fit/runcards/examples/Basic_runcard_qed.yml | 2 +- nnpdfcpp/data/theory.db | Bin 102400 -> 102400 bytes validphys2/src/validphys/photon/compute.py | 27 ++++-------------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index 5d02bf3db3..aff16fadec 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -95,7 +95,7 @@ datacuts: ############################################################ theory: - theoryid: 522 # database id + theoryid: 523 # database id ############################################################ trvlseed: 1551864071 diff --git a/nnpdfcpp/data/theory.db b/nnpdfcpp/data/theory.db index f3c94c9d3ef9e3d0ec07ce67f0037376f4e79eb2..affda346bca15c18297b7beb68b6f77211de724e 100644 GIT binary patch delta 162 zcmZozz}B#UZGto-^F$eEM&^wPOXQg)IdnF&E9fz?>TtMl*laBP#LB_W%*4dR&BWbo zqOsjXgYiPR6gQJU1K%3nR30_1XPomm;@D-GOPTyxAF(XjSZK(^(xlYFJ-xD-QEhsG z79;m`?gUEy Date: Fri, 5 May 2023 15:50:01 +0200 Subject: [PATCH 177/204] Fix key in test --- validphys2/src/validphys/tests/photon/test_compute.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 73ce223507..8928a4c28a 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -18,7 +18,7 @@ def get_description(self): return { "alphaqed": 0.01, "Qref": 91.2, - "QrefQED": 91.2, + "Qedref": 91.2, "mc": 1.3, "mb": 4.92, "mt": 172.0, From d10dc73025413e62fb1730ed7db44e8c4de03bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Fri, 5 May 2023 17:37:46 +0200 Subject: [PATCH 178/204] Fix theory 523 in theory.db --- nnpdfcpp/data/theory.db | Bin 102400 -> 102400 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/nnpdfcpp/data/theory.db b/nnpdfcpp/data/theory.db index affda346bca15c18297b7beb68b6f77211de724e..1f953e2459c1a85332b22111d2482d6dfe3ba942 100644 GIT binary patch delta 36 scmZozz}B#UZGto-%S0JxMwZ5etqF`SP3-k|gxeZEcyC|b#2Cs80NjHNtN;K2 delta 36 scmZozz}B#UZGto-^F$eEM&`zZtqF`SP3(1Bk}ti!9KL;Z6JsbZ0OR}(cK`qY From f987049416e2481fb6ab1083f4bde6e6562319a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 10 May 2023 12:15:15 +0200 Subject: [PATCH 179/204] Update validphys2/src/validphys/pdfbases.py Co-authored-by: Roy Stegeman --- validphys2/src/validphys/pdfbases.py | 31 ++-------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/validphys2/src/validphys/pdfbases.py b/validphys2/src/validphys/pdfbases.py index 15b2e1b94d..91c7004338 100644 --- a/validphys2/src/validphys/pdfbases.py +++ b/validphys2/src/validphys/pdfbases.py @@ -523,37 +523,10 @@ def f_(transform_func): default_elements=(r'\Sigma', 'V', 'T3', 'V3', 'T8', 'V8', 'T15', 'gluon', ) ) -lux = LinearBasis.from_mapping({ - r'\Sigma' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1 ,'b':1, 'bbar': 1, 't': 1, 'tbar': 1}, - 'V' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c': 1, 'cbar':-1 ,'b':1, 'bbar':-1, 't': 1, 'tbar':-1}, - - 'T3' : {'u': 1, 'ubar': 1, 'd':-1, 'dbar':-1}, - 'V3' : {'u': 1, 'ubar':-1, 'd':-1, 'dbar': 1}, - - 'T8' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's':-2, 'sbar':-2}, - 'V8' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's':-2, 'sbar':+2}, - - 'T15' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c':-3, 'cbar':-3}, - 'V15' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c':-3, 'cbar':+3}, - - - 'T24' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1, 'b':-4, 'bbar':-4}, - 'V24' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c': 1, 'cbar':-1, 'b':-4, 'bbar':+4}, - - 'T35' : {'u': 1, 'ubar': 1, 'd': 1, 'dbar': 1, 's': 1, 'sbar': 1, 'c': 1, 'cbar': 1, 'b': 1, 'bbar': 1, 't':-5, 'tbar':-5}, - 'V35' : {'u': 1, 'ubar':-1, 'd': 1, 'dbar':-1, 's': 1, 'sbar':-1, 'c': 1, 'cbar':-1, 'b': 1, 'bbar':-1, 't':-5, 'tbar':+5}, - - 'g' : {'g':1}, - 'photon' : {'photon':1}, - }, - aliases = {'gluon':'g', 'singlet': r'\Sigma', 'sng': r'\Sigma', 'sigma': r'\Sigma', - 'v': 'V', 'v3': 'V3', 'v8': 'V8', 't3': 'T3', 't8': 'T8', 't15': 'T15', 'v15': 'V15',}, - default_elements=(r'\Sigma', 'V', 'T3', 'V3', 'T8', 'V8', 'T15', 'gluon', 'photon') -) - EVOL = evolution -LUX = lux +LUX = copy.deepcopy(evolution) +LUX.default_elements = (r'\Sigma', 'V', 'T3', 'V3', 'T8', 'V8', 'T15', 'gluon', 'photon') CCBAR_ASYMM = copy.deepcopy(evolution) CCBAR_ASYMM.default_elements = (r'\Sigma', 'V', 'T3', 'V3', 'T8', 'V8', 'T15', 'gluon', 'V15') From 217c598eb1a2061e003f36d38edfd4fc8ac5e825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 10 May 2023 12:18:53 +0200 Subject: [PATCH 180/204] Add empty line to runcard qed --- n3fit/runcards/examples/Basic_runcard_qed.yml | 1 - validphys2/src/validphys/photon/compute.py | 48 +++++++++---------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index aff16fadec..d07e9d6cfb 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -169,4 +169,3 @@ fiatlux: luxset: NNPDF40_nnlo_as_01180 additional_errors: true # should be set to true only for the last iteration luxseed: 1234567890 - \ No newline at end of file diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index a693d03296..5675ada069 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -19,34 +19,33 @@ Q_IN = 100 FIATLUX_DEFAULT = { - "apfel" : False, - "q2_max" : 1e8, # the maximum allowed Q2. - "eps_base" : 1e-5, # precision on final integration of double integral. - "eps_rel" : 1e-1, # extra precision on any single integration. - "mum_proton" : 2.792847356, # proton magnetic moment, from + "apfel": False, + "q2_max": 1e8, # the maximum allowed Q2. + "eps_base": 1e-5, # precision on final integration of double integral. + "eps_rel": 1e-1, # extra precision on any single integration. + "mum_proton": 2.792847356, # proton magnetic moment, from # http://pdglive.lbl.gov/DataBlock.action?node=S016MM which itself # gets it from arXiv:1203.5425 (CODATA) - # the elastic param type, options: # dipole # A1_world_spline # A1_world_pol_spline - "elastic_param" : "A1_world_pol_spline", - "elastic_electric_rescale" : 1, - "elastic_magnetic_rescale" : 1, + "elastic_param": "A1_world_pol_spline", + "elastic_electric_rescale": 1, + "elastic_magnetic_rescale": 1, # the inelastic param type, options: - "inelastic_param" : "LHAPDF_Hermes_ALLM_CLAS", # Hermes_ALLM_CLAS, LHAPDF_Hermes_ALLM_CLAS - "rescale_r_twist4" : 0, - "rescale_r" : 1, - "allm_limits" : 0, - "rescale_non_resonance" : 1, - "rescale_resonance" : 1, - "use_mu2_as_upper_limit" : False, - "q2min_inel_override" : 0.0, - "q2max_inel_override" : 1E300, - "lhapdf_transition_q2" : 9, + "inelastic_param": "LHAPDF_Hermes_ALLM_CLAS", # Hermes_ALLM_CLAS, LHAPDF_Hermes_ALLM_CLAS + "rescale_r_twist4": 0, + "rescale_r": 1, + "allm_limits": 0, + "rescale_non_resonance": 1, + "rescale_resonance": 1, + "use_mu2_as_upper_limit": False, + "q2min_inel_override": 0.0, + "q2max_inel_override": 1e300, + "lhapdf_transition_q2": 9, # general - "verbose" : False, + "verbose": False, } @@ -56,7 +55,9 @@ class Photon: def __init__(self, theoryid, lux_params, replicas): theory = theoryid.get_description() fiatlux_runcard = FIATLUX_DEFAULT - fiatlux_runcard["qed_running"] = bool(np.isclose(theory["Qedref"], theory["Qref"])) + fiatlux_runcard["qed_running"] = bool( + np.isclose(theory["Qedref"], theory["Qref"]) + ) # cast explicitly from np.bool_ to bool otherwise problems in dumping it # TODO: for the time being, we trigger alphaem running if Qedref=Qref. # This is going to be changed in favor of a bool em_running @@ -117,8 +118,7 @@ def __init__(self, theoryid, lux_params, replicas): for photon_array in photons_array ] self.integral = [ - trapezoid(photons_array[id], XGRID) - for id in range(len(self.replicas)) + trapezoid(photons_array[id], XGRID) for id in range(len(self.replicas)) ] def compute_photon_array(self, replica): @@ -156,7 +156,7 @@ def compute_photon_array(self, replica): if pid == 22: pdfs_init[j] = photon_qin ph_id = j - else : + else: if pid not in self.luxpdfset.flavors: continue pdfs_init[j] = np.array( From 54b365cd31d804fe1d274777f5a854dc03ae352a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 16 May 2023 12:10:05 +0200 Subject: [PATCH 181/204] Fix some stuff --- conda-recipe/meta.yaml | 2 +- n3fit/src/n3fit/checks.py | 2 +- validphys2/src/validphys/photon/compute.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 35e97c3a63..c2c3028999 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -51,7 +51,7 @@ requirements: - sphinxcontrib-bibtex - curio >=1.0 - pineappl >=0.5.8 - - eko >=0.12.0 + - eko >=0.12.2 - banana-hep >=0.6.8 - fiatlux diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 0b87a9af50..1d3477051b 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -397,7 +397,7 @@ def check_deprecated_options(fitting): @make_argcheck def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): - if fiatlux : + if fiatlux is not None: luxset = fiatlux["luxset"] pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 max_id = max(replicas) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 5675ada069..e4bb450101 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -262,8 +262,8 @@ def alpha_em_fixed_flavor(self, q, alpha_ref, qref, nf): ---------- q : float target scale - a_ref : float - reference value of a = alpha_em/(4*pi) + alph_aem_ref : float + reference value of alpha_em qref: float reference scale nf: int From 23c360059f1be7e0495026c46efe449516a1c351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 16 May 2023 12:11:53 +0200 Subject: [PATCH 182/204] Update n3fit/src/n3fit/checks.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 1d3477051b..93162d5007 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -401,7 +401,7 @@ def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): luxset = fiatlux["luxset"] pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 max_id = max(replicas) - if max_id > pdfs_ids : + if max_id > pdfs_ids: for replica_id in replicas: # At this point it should be always empty os.rmdir(replica_path / f"replica_{replica_id}") From f1a2ce112f2bf04b47551269af10c197e3657607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 16 May 2023 12:17:19 +0200 Subject: [PATCH 183/204] Do not remove folder for id>max_id --- n3fit/src/n3fit/checks.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 1d3477051b..0f31d86f12 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -402,11 +402,7 @@ def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 max_id = max(replicas) if max_id > pdfs_ids : - for replica_id in replicas: - # At this point it should be always empty - os.rmdir(replica_path / f"replica_{replica_id}") raise CheckError( f"Cannot generate a photon replica with id larger than the number of replicas of the PDFs set " - + luxset + f": replica id={max_id}, replicas of " + luxset + f"={pdfs_ids}" - +"\nRemoving replica output folders" + + luxset.name + f":\nreplica id={max_id}, replicas of " + luxset.name + f"={pdfs_ids}" ) From a1d129e83c24fd6e7e2d91b45d230bc5862fd6b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 16 May 2023 12:25:23 +0200 Subject: [PATCH 184/204] Fix else : -> else: --- n3fit/src/n3fit/layers/msr_normalization.py | 2 +- n3fit/src/n3fit/scripts/n3fit_exec.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index a684ccd688..991d92718f 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -55,7 +55,7 @@ def call(self, pdf_integrated, ph_replica): if self._photons: photon_integral = self._photons[ph_replica] - else : + else: photon_integral = 0. if self._msr_enabled: diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index a4a67578f7..9b80cb5308 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -150,7 +150,7 @@ def from_yaml(cls, o, *args, **kwargs): if (thconfig:=file_content.get('fiatlux')): N3FIT_FIXED_CONFIG['fiatlux']=thconfig - else : + else: N3FIT_FIXED_CONFIG['fiatlux']=None #Theorycovmat flags and defaults N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False From a40474ae928dfc2ba31304197e550510365305f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 16 May 2023 12:25:56 +0200 Subject: [PATCH 185/204] Update n3fit/src/n3fit/layers/rotations.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/layers/rotations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 59ddbcde30..6db811ec11 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -91,6 +91,7 @@ def call(self, pdf_raw): # Concatenating destroys the batch index so we have to regenerate it return op.batchit(ret) + class AddPhoton(MetaLayer): """ Changes the value of the photon component of the PDF to non-zero. From 53e3fb0ccdc37a4906bc4affe94a08ec3fe6efe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 16 May 2023 12:26:34 +0200 Subject: [PATCH 186/204] Update n3fit/src/n3fit/model_trainer.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/model_trainer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index b68c448fe4..fb165aa111 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -135,11 +135,11 @@ def __init__( whether sum rules should be enabled (All, MSR, VSR, False) parallel_models: int number of models to fit in parallel - theoryid : validphys.core.TheoryIDSpec + theoryid: validphys.core.TheoryIDSpec object contining info for generating the photon - lux_params : dict + lux_params: dict dictionary containing the params needed from LuxQED - replicas_id : list + replicas_id: list list with the replicas ids to be fitted """ # Save all input information From 38efd5e6fb3667e9ffefdda53cd9967316c53030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Tue, 16 May 2023 12:28:41 +0200 Subject: [PATCH 187/204] Update n3fit/src/n3fit/performfit.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/n3fit/performfit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index bc34b95372..03c36aa1b1 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -86,7 +86,7 @@ def performfit( Theory which is used to generate theory predictions from model during fit. Object also contains some metadata on the theory settings. - lux_params : dict + fiatlux: dict dictionary containing the params needed from LuxQED basis: list[dict] preprocessing information for each flavour to be fitted. From 4e325003cd4d500a8f140e128480012ac1bd6fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 16 May 2023 12:54:41 +0200 Subject: [PATCH 188/204] Restructure for loop in compute --- n3fit/runcards/examples/Basic_runcard_qed.yml | 2 +- validphys2/src/validphys/photon/compute.py | 55 ++++++++----------- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/n3fit/runcards/examples/Basic_runcard_qed.yml b/n3fit/runcards/examples/Basic_runcard_qed.yml index d07e9d6cfb..309f5901d5 100644 --- a/n3fit/runcards/examples/Basic_runcard_qed.yml +++ b/n3fit/runcards/examples/Basic_runcard_qed.yml @@ -95,7 +95,7 @@ datacuts: ############################################################ theory: - theoryid: 523 # database id + theoryid: 522 # database id ############################################################ trvlseed: 1551864071 diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index e4bb450101..7f38658a89 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -77,49 +77,40 @@ def __init__(self, theoryid, lux_params, replicas): # set fiatlux self.lux = {} - f2 = {} - fl = {} - f2lo = {} + + alpha = Alpha(theory) + mb_thr = theory["kbThr"] * theory["mb"] + mt_thr = theory["ktThr"] * theory["mt"] if theory["MaxNfPdf"] == 6 else 1e100 + + self.interpolator = [] + self.integral = [] + for replica in replicas: - f2[replica] = sf.InterpStructureFunction( - path_to_F2, self.luxpdfset.members[replica] - ) - fl[replica] = sf.InterpStructureFunction( - path_to_FL, self.luxpdfset.members[replica] - ) - f2lo[replica] = sf.F2LO(self.luxpdfset.members[replica], theory) + f2 = sf.InterpStructureFunction(path_to_F2, self.luxpdfset.members[replica]) + fl = sf.InterpStructureFunction(path_to_FL, self.luxpdfset.members[replica]) + f2lo = sf.F2LO(self.luxpdfset.members[replica], theory) with tempfile.NamedTemporaryFile(mode="w") as tmp: with tmp.file as tmp_file: tmp_file.write(yaml.dump(fiatlux_runcard)) self.lux[replica] = fiatlux.FiatLux(tmp_file.name) - # we have a dict but fiatlux wants a yaml file - # TODO : once that fiatlux will allow dictionaries - # pass directly fiatlux_runcard - alpha = Alpha(theory) - for replica in replicas: + # we have a dict but fiatlux wants a yaml file + # TODO : once that fiatlux will allow dictionaries + # pass directly fiatlux_runcard + self.lux[replica].PlugAlphaQED(alpha.alpha_em, alpha.qref) self.lux[replica].InsertInelasticSplitQ( [ - theory["kbThr"] * theory["mb"], - theory["ktThr"] * theory["mt"] - if theory["MaxNfPdf"] == 6 - else 1e100, + mb_thr, + mt_thr, ] ) - self.lux[replica].PlugStructureFunctions( - f2[replica].fxq, fl[replica].fxq, f2lo[replica].fxq - ) + self.lux[replica].PlugStructureFunctions(f2.fxq, fl.fxq, f2lo.fxq) - photons_array = [ - self.compute_photon_array(replica) for replica in self.replicas - ] - self.interpolator = [ - interp1d(XGRID, photon_array, fill_value=0.0, kind="cubic") - for photon_array in photons_array - ] - self.integral = [ - trapezoid(photons_array[id], XGRID) for id in range(len(self.replicas)) - ] + photon_array = self.compute_photon_array(replica) + self.interpolator.append( + interp1d(XGRID, photon_array, fill_value=0.0, kind="cubic") + ) + self.integral.append(trapezoid(photon_array, XGRID)) def compute_photon_array(self, replica): r""" From 74e85f07645293ec9bc3787c436d11fbc1df8555 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 16 May 2023 14:22:25 +0200 Subject: [PATCH 189/204] Change constants.NC -> NC etc --- validphys2/src/validphys/photon/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 7f38658a89..177b58a3cf 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -12,7 +12,7 @@ from n3fit.io.writer import XGRID -from . import constants +from .constants import NC, NL, ED2, EU2 from . import structure_functions as sf log = logging.getLogger(__name__) @@ -334,15 +334,15 @@ def set_betas(self): -4.0 / 3 * ( - constants.NL - + constants.NC * (nu * constants.EU2 + nd * constants.ED2) + NL + + NC * (nu * EU2 + nd * ED2) ) ) / (4 * np.pi) b1[nf] = ( -4.0 * ( - constants.NL - + constants.NC * (nu * constants.EU2**2 + nd * constants.ED2**2) + NL + + NC * (nu * EU2**2 + nd * ED2**2) ) / beta0[nf] / (4 * np.pi) ** 2 From 5d9b9e8da6805abd18910cf09314b29a564728a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Tue, 16 May 2023 14:23:48 +0200 Subject: [PATCH 190/204] CAll black and isort on the last commit --- validphys2/src/validphys/photon/compute.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 177b58a3cf..b022396631 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -12,8 +12,8 @@ from n3fit.io.writer import XGRID -from .constants import NC, NL, ED2, EU2 from . import structure_functions as sf +from .constants import ED2, EU2, NC, NL log = logging.getLogger(__name__) @@ -330,20 +330,10 @@ def set_betas(self): for nf in range(3, 6 + 1): nu = nf // 2 nd = nf - nu - beta0[nf] = ( - -4.0 - / 3 - * ( - NL - + NC * (nu * EU2 + nd * ED2) - ) - ) / (4 * np.pi) + beta0[nf] = (-4.0 / 3 * (NL + NC * (nu * EU2 + nd * ED2))) / (4 * np.pi) b1[nf] = ( -4.0 - * ( - NL - + NC * (nu * EU2**2 + nd * ED2**2) - ) + * (NL + NC * (nu * EU2**2 + nd * ED2**2)) / beta0[nf] / (4 * np.pi) ** 2 ) From 19c188b724b92b4ec0e37d7ad834558254543335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 17 May 2023 14:55:00 +0200 Subject: [PATCH 191/204] Update n3fit/src/evolven3fit_new/evolve.py Co-authored-by: Juan M. Cruz-Martinez --- n3fit/src/evolven3fit_new/evolve.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/n3fit/src/evolven3fit_new/evolve.py b/n3fit/src/evolven3fit_new/evolve.py index f25b8b6cf8..d0a6710cf0 100644 --- a/n3fit/src/evolven3fit_new/evolve.py +++ b/n3fit/src/evolven3fit_new/evolve.py @@ -105,7 +105,9 @@ def evolve_fit( info["XMin"] = float(x_grid[0]) info["XMax"] = float(x_grid[-1]) with eko.EKO.read(eko_path) as eko_op: - info["AlphaS_Qs"] = eko_op.mu2grid.tolist() + if eko.__version__ >= "0.13": + raise ModuleNotFoundError("Please, fix evolven3fit np.sqrt(Q) hack") + info["AlphaS_Qs"] = np.sqrt(info["AlphaS_Qs"]).tolist() dump_info_file(usr_path, info) for replica in initial_PDFs_dict.keys(): evolved_block = evolve_exportgrid(initial_PDFs_dict[replica], eko_op, x_grid, qed) From 3c56b7b04fde49fc28744bc4c1d95f2456bb2cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 15:00:30 +0200 Subject: [PATCH 192/204] Add docstring to parse_additional_errors --- validphys2/src/validphys/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index 873b7245a8..c71c2a8354 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -1045,7 +1045,12 @@ def parse_luxset(self, name): return self.parse_pdf(name) def parse_additional_errors(self, bool): - """PDF set used to generate the photon additional errors.""" + """PDF set used to generate the photon additional errors: + they are constructed using the replicas 101-107 of the PDF set + LUXqed17_plus_PDF4LHC15_nnlo_100 (that are obtained varying some + parameters of the LuxQED approach) in the way described + in sec. 2.5 of https://arxiv.org/pdf/1712.07053.pdf + """ if bool: return self.parse_pdf("LUXqed17_plus_PDF4LHC15_nnlo_100") else: From 3eafc70923f0cdb6bae1a712f0eb8fadaa83e8d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 15:23:38 +0200 Subject: [PATCH 193/204] Read q2_max from fktables without hardcoding it --- validphys2/src/validphys/photon/compute.py | 5 ++++- validphys2/src/validphys/photon/structure_functions.py | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index b022396631..5fd5814d98 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -20,7 +20,6 @@ Q_IN = 100 FIATLUX_DEFAULT = { "apfel": False, - "q2_max": 1e8, # the maximum allowed Q2. "eps_base": 1e-5, # precision on final integration of double integral. "eps_rel": 1e-1, # extra precision on any single integration. "mum_proton": 2.792847356, # proton magnetic moment, from @@ -88,6 +87,10 @@ def __init__(self, theoryid, lux_params, replicas): for replica in replicas: f2 = sf.InterpStructureFunction(path_to_F2, self.luxpdfset.members[replica]) fl = sf.InterpStructureFunction(path_to_FL, self.luxpdfset.members[replica]) + if not np.isclose(f2.q2_max, fl.q2_max) : + log.error("FKtables for fiatlux_dis_F2 and fiatlux_dis_FL have two different q2_max") + + fiatlux_runcard["q2_max"] = float(f2.q2_max) f2lo = sf.F2LO(self.luxpdfset.members[replica], theory) with tempfile.NamedTemporaryFile(mode="w") as tmp: with tmp.file as tmp_file: diff --git a/validphys2/src/validphys/photon/structure_functions.py b/validphys2/src/validphys/photon/structure_functions.py index 97996a73b8..fe17e32ad8 100644 --- a/validphys2/src/validphys/photon/structure_functions.py +++ b/validphys2/src/validphys/photon/structure_functions.py @@ -41,6 +41,8 @@ def __init__(self, path_to_fktable, pdfs): x = np.unique(self.fktable.bin_left(1)) q2 = np.unique(self.fktable.bin_left(0)) + self.q2_max = max(q2) + predictions = self.fktable.convolute_with_one(2212, pdfs.xfxQ2) grid2D = predictions.reshape(len(x), len(q2)) From c4984dc4aab5eed6f9f7277186db11c03840b103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 15:46:51 +0200 Subject: [PATCH 194/204] Add qed runcard documentation --- doc/sphinx/source/n3fit/runcard_detailed.rst | 29 +++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/sphinx/source/n3fit/runcard_detailed.rst b/doc/sphinx/source/n3fit/runcard_detailed.rst index 180de72667..0ee697f508 100644 --- a/doc/sphinx/source/n3fit/runcard_detailed.rst +++ b/doc/sphinx/source/n3fit/runcard_detailed.rst @@ -419,4 +419,31 @@ It is however possible to disable them by setting to false the ``sum_rules`` fla It is also possible to impose just the valence or the momentum sum rules by using the -``VSR`` or ``MSR`` flags, respectively (``True`` is equal to ``All``). \ No newline at end of file +``VSR`` or ``MSR`` flags, respectively (``True`` is equal to ``All``). + + +QED fit +^^^^^^^ + +It is possible to perform a QED fit adding the key `fiatlux` to the runcard. In this way +a photon PDF will be generated using the `FiatLux` public library that implements the `LuxQED` +approach. The parameters to be added are the following: + +.. code-block:: yaml + + fiatlux: + luxset: NNPDF40_nnlo_as_01180 + additional_errors: true + luxseed: 1234567890 + +`luxset` is the PDF set used to generate the photon PDF with `FiatLux`. The code wil generate as +much photon replicas as the number of replicas contained in the `luxset`. Therefore, if the user +tries to generate a replica with ID higher than the maximum ID of the `luxset`, the code will +raise an error. Moreover, being the `LuxQED` approach an iterated prcedure, and given that some replicas +do not pass the `postfit` selection criteria, the user should make sure that the number of replicas in +the `luxset` is high enough such that in the final iteration there will be a number of replicas +higher than the final replicas desired. +`additional_errors` is the parameter that switches on and off the additional errors of the `LuxQED` approach, +while `luxseed` is the seed used to generate such errors. + +Whenever the photon PDF is generated, it will remain constant during the fit and will enter in the `MSR`. From fbfd77118d60306d4e0ea84b832088eafb8cb867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 16:01:56 +0200 Subject: [PATCH 195/204] Call balck and isort to all changed files --- n3fit/src/n3fit/checks.py | 111 ++++++++--- n3fit/src/n3fit/layers/__init__.py | 11 +- n3fit/src/n3fit/layers/msr_normalization.py | 6 +- n3fit/src/n3fit/layers/rotations.py | 14 +- n3fit/src/n3fit/model_gen.py | 25 ++- n3fit/src/n3fit/model_trainer.py | 103 ++++++++--- n3fit/src/n3fit/msr.py | 54 +++--- n3fit/src/n3fit/performfit.py | 195 ++++++++++---------- n3fit/src/n3fit/scripts/n3fit_exec.py | 125 ++++++++----- n3fit/src/n3fit/scripts/vp_setupfit.py | 139 +++++++------- n3fit/src/n3fit/vpinterface.py | 15 +- validphys2/src/validphys/config.py | 183 +++++++++--------- validphys2/src/validphys/filters.py | 94 +++++----- validphys2/src/validphys/n3fit_data.py | 35 ++-- validphys2/src/validphys/photon/compute.py | 6 +- 15 files changed, 655 insertions(+), 461 deletions(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 4573926822..73e912b693 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -1,13 +1,15 @@ """ This module contains checks to be perform by n3fit on the input """ -import os import logging import numbers +import os + import numpy as np -from reportengine.checks import make_argcheck, CheckError +from reportengine.checks import CheckError, make_argcheck from validphys.core import PDF from validphys.pdfbases import check_basis + from n3fit.hyper_optimization import penalties as penalties_module from n3fit.hyper_optimization import rewards as rewards_module @@ -53,7 +55,9 @@ def check_consistent_layers(parameters): npl = len(parameters["nodes_per_layer"]) apl = len(parameters["activation_per_layer"]) if npl != apl: - raise CheckError(f"Number of layers ({npl}) does not match activation functions: {apl}") + raise CheckError( + f"Number of layers ({npl}) does not match activation functions: {apl}" + ) def check_stopping(parameters): @@ -131,12 +135,16 @@ def check_lagrange_multipliers(parameters, key): return multiplier = lagrange_dict.get("multiplier") if multiplier is not None and multiplier < 0: - log.warning("The %s multiplier is below 0, it will produce a negative loss", key) + log.warning( + "The %s multiplier is below 0, it will produce a negative loss", key + ) elif multiplier is not None and multiplier == 0: log.warning("The %s multiplier is 0 so it won't contribute to the loss", key) threshold = lagrange_dict.get("threshold") if threshold is not None and not _is_floatable(threshold): - raise CheckError(f"The {key}::threshold must be a number, received: {threshold}") + raise CheckError( + f"The {key}::threshold must be a number, received: {threshold}" + ) def check_model_file(save, load): @@ -150,9 +158,13 @@ def check_model_file(save, load): if not isinstance(load, str): raise CheckError(f"Model file to load: {load} not understood, str expected") if not os.path.isfile(load): - raise CheckError(f"Model file to load: {load} can not be opened, does it exist?") + raise CheckError( + f"Model file to load: {load} can not be opened, does it exist?" + ) if not os.access(load, os.R_OK): - raise CheckError(f"Model file to load: {load} cannot be read, permission denied") + raise CheckError( + f"Model file to load: {load} cannot be read, permission denied" + ) if os.stat(load).st_size == 0: raise CheckError(f"Model file {load} seems to be empty") @@ -193,7 +205,9 @@ def check_hyperopt_architecture(architecture): min_u = architecture.get("min_units", 1) # Set a minimum number of units in case none is defined to check later if the maximum is sane if min_u <= 0: - raise CheckError(f"All layers must have at least 1 unit, got min_units: {min_u}") + raise CheckError( + f"All layers must have at least 1 unit, got min_units: {min_u}" + ) max_u = architecture.get("max_units") if max_u is not None and max_u < min_u: raise CheckError( @@ -210,9 +224,13 @@ def check_hyperopt_positivity(positivity_dict): max_mul = positivity_dict.get("max_multiplier") if max_mul is not None or min_mul is not None: if max_mul is None: - raise CheckError("Need to set a maximum positivity multiplier if the minimum is set") + raise CheckError( + "Need to set a maximum positivity multiplier if the minimum is set" + ) if min_mul is not None and max_mul <= min_mul: - raise CheckError("The minimum multiplier cannot be greater than the maximum") + raise CheckError( + "The minimum multiplier cannot be greater than the maximum" + ) min_ini = positivity_dict.get("min_initial") max_ini = positivity_dict.get("max_initial") if max_ini is not None or min_ini is not None: @@ -221,7 +239,9 @@ def check_hyperopt_positivity(positivity_dict): "Need to set both the max_initial and the min_initial positivity values" ) if max_ini <= min_ini: - raise CheckError("The minimum initial value cannot be greater than the maximum") + raise CheckError( + "The minimum initial value cannot be greater than the maximum" + ) def check_kfold_options(kfold): @@ -246,9 +266,13 @@ def check_kfold_options(kfold): # Check specific errors for specific targets if loss_target == "fit_future_tests": if len(partitions) == 1: - raise CheckError("Cannot use target 'fit_future_tests' with just one partition") + raise CheckError( + "Cannot use target 'fit_future_tests' with just one partition" + ) if partitions[-1]["datasets"]: - log.warning("Last partition in future test is not empty, some datasets will be ignored") + log.warning( + "Last partition in future test is not empty, some datasets will be ignored" + ) def check_correct_partitions(kfold, data): @@ -260,7 +284,9 @@ def check_correct_partitions(kfold, data): fold_sets = partition["datasets"] for dset in fold_sets: if dset not in datasets: - raise CheckError(f"The k-fold defined dataset {dset} is not part of the fit") + raise CheckError( + f"The k-fold defined dataset {dset} is not part of the fit" + ) def check_hyperopt_stopping(stopping_dict): @@ -273,14 +299,20 @@ def check_hyperopt_stopping(stopping_dict): if min_ep is None or max_ep is None: raise CheckError("Need to set both the max_epochs and the min_epochs") if min_ep < 1: - raise CheckError(f"Can't run for less than 1 epoch: selected min_ep = {min_ep}") + raise CheckError( + f"Can't run for less than 1 epoch: selected min_ep = {min_ep}" + ) if max_ep <= min_ep: - raise CheckError(f"min_epochs cannot be greater than max_epochs: ({min_ep} > {max_ep})") + raise CheckError( + f"min_epochs cannot be greater than max_epochs: ({min_ep} > {max_ep})" + ) min_pat = stopping_dict.get("min_patience") max_pat = stopping_dict.get("max_patience") if min_pat is not None or max_pat is not None: if min_pat is not None and min_pat < 0.0: - raise CheckError(f"min_patience cannot be less than 0.0: selected min_pat = {min_pat}") + raise CheckError( + f"min_patience cannot be less than 0.0: selected min_pat = {min_pat}" + ) if max_pat is not None: if max_pat > 1.0: raise CheckError( @@ -300,7 +332,9 @@ def wrapper_hyperopt(hyperopt, hyperscan_config, kfold, data): if not hyperopt: return if hyperscan_config is None: - raise CheckError("Can't perform hyperoptimization without the hyperscan_config key") + raise CheckError( + "Can't perform hyperoptimization without the hyperscan_config key" + ) if kfold is None: raise CheckError("Can't perform hyperoptimization without folds") check_hyperopt_stopping(hyperscan_config.get("stopping")) @@ -317,7 +351,9 @@ def check_sumrules(sum_rules): accepted_options = ["ALL", "MSR", "VSR"] if sum_rules.upper() in accepted_options: return - raise CheckError(f"The only accepted options for the sum rules are: {accepted_options}") + raise CheckError( + f"The only accepted options for the sum rules are: {accepted_options}" + ) # Checks on the physics @@ -346,11 +382,17 @@ def check_consistent_basis(sum_rules, fitbasis, basis, theoryid): # Check that the basis given in the runcard is one of those defined in validphys.pdfbases basis = check_basis(fitbasis, flavs)["basis"] # Now check that basis and theory id are consistent - has_c = basis.has_element("c") or basis.has_element("T15") or basis.has_element("cp") + has_c = ( + basis.has_element("c") or basis.has_element("T15") or basis.has_element("cp") + ) if theoryid.get_description()["IC"] and not has_c: - raise CheckError(f"{theoryid} (intrinsic charm) is incompatible with basis {fitbasis}") + raise CheckError( + f"{theoryid} (intrinsic charm) is incompatible with basis {fitbasis}" + ) if not theoryid.get_description()["IC"] and has_c: - raise CheckError(f"{theoryid} (perturbative charm) is incompatible with basis {fitbasis}") + raise CheckError( + f"{theoryid} (perturbative charm) is incompatible with basis {fitbasis}" + ) @make_argcheck @@ -366,7 +408,9 @@ def check_consistent_parallel(parameters, parallel_models, same_trvl_per_replica " masks, please set `same_trvl_per_replica` to True in the runcard" ) if parameters.get("layer_type") != "dense": - raise CheckError("Parallelization has only been tested with layer_type=='dense'") + raise CheckError( + "Parallelization has only been tested with layer_type=='dense'" + ) @make_argcheck @@ -382,24 +426,37 @@ def can_run_multiple_replicas(replicas, parallel_models): @make_argcheck def check_deprecated_options(fitting): """Checks whether the runcard is using deprecated options""" - options_outside = ["trvlseed", "nnseed", "mcseed", "save", "load", "genrep", "parameters"] + options_outside = [ + "trvlseed", + "nnseed", + "mcseed", + "save", + "load", + "genrep", + "parameters", + ] for option in options_outside: if option in fitting: raise CheckError( f"The key '{option}' should be top-level key and not part of the 'fitting' namespace" ) if "epochs" in fitting: - raise CheckError("The key 'epoch' should only appear as part of the 'parameters' namespace") + raise CheckError( + "The key 'epoch' should only appear as part of the 'parameters' namespace" + ) nnfit_options = ["seed", "rnalgo", "fitmethod", "nmutants", "paramtype", "nnodes"] for option in nnfit_options: if option in fitting: - log.warning("'fitting::%s' is an nnfit-only key, it will be ignored", option) + log.warning( + "'fitting::%s' is an nnfit-only key, it will be ignored", option + ) + @make_argcheck def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): if fiatlux is not None: luxset = fiatlux["luxset"] - pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 + pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 max_id = max(replicas) if max_id > pdfs_ids: raise CheckError( diff --git a/n3fit/src/n3fit/layers/__init__.py b/n3fit/src/n3fit/layers/__init__.py index d7d288bf9b..a9168bda13 100644 --- a/n3fit/src/n3fit/layers/__init__.py +++ b/n3fit/src/n3fit/layers/__init__.py @@ -1,8 +1,9 @@ -from .preprocessing import Preprocessing -from .rotations import FkRotation, FlavourToEvolution, ObsRotation, AddPhoton -from .x_operations import xIntegrator, xDivide -from .msr_normalization import MSR_Normalization +from n3fit.backends import MetaLayer + from .DIS import DIS from .DY import DY from .mask import Mask -from n3fit.backends import MetaLayer +from .msr_normalization import MSR_Normalization +from .preprocessing import Preprocessing +from .rotations import AddPhoton, FkRotation, FlavourToEvolution, ObsRotation +from .x_operations import xDivide, xIntegrator diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 991d92718f..8306853b19 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -56,10 +56,12 @@ def call(self, pdf_integrated, ph_replica): if self._photons: photon_integral = self._photons[ph_replica] else: - photon_integral = 0. + photon_integral = 0.0 if self._msr_enabled: - n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - photon_integral) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) + n_ag = [ + (1.0 - y[GLUON_IDX[0][0] - 1] - photon_integral) / y[GLUON_IDX[0][0]] + ] * len(GLUON_IDX) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 6db811ec11..a543462d6b 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -2,9 +2,10 @@ This module includes rotation layers """ import numpy as np +from validphys import pdfbases + from n3fit.backends import MetaLayer from n3fit.backends import operations as op -from validphys import pdfbases class Rotation(MetaLayer): @@ -28,7 +29,7 @@ def __init__(self, rotation_matrix, axes=1, **kwargs): super().__init__(**kwargs) def is_identity(self): - """ Returns true if the rotation is an identity """ + """Returns true if the rotation is an identity""" # check whether it is a mxm matrix if self.rotation_matrix.shape[0] == self.rotation_matrix.shape[1]: # check whether it is the identity @@ -41,12 +42,15 @@ def call(self, x_raw): class FlavourToEvolution(Rotation): """ - Rotates from the flavour basis to - the evolution basis. + Rotates from the flavour basis to + the evolution basis. """ def __init__( - self, flav_info, fitbasis, **kwargs, + self, + flav_info, + fitbasis, + **kwargs, ): rotation_matrix = pdfbases.fitbasis_to_NN31IC(flav_info, fitbasis) super().__init__(rotation_matrix, axes=1, **kwargs) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index d360b43ea3..55a08debee 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -10,16 +10,17 @@ """ from dataclasses import dataclass + import numpy as np -from n3fit.msr import msr_impose -from n3fit.layers import DIS, DY, ObsRotation, losses -from n3fit.layers import Preprocessing, FkRotation, FlavourToEvolution, AddPhoton -from n3fit.layers.observable import is_unique -from n3fit.backends import MetaModel, Input +from n3fit.backends import (Input, Lambda, MetaLayer, MetaModel, + base_layer_selector) from n3fit.backends import operations as op -from n3fit.backends import MetaLayer, Lambda -from n3fit.backends import base_layer_selector, regularizer_selector +from n3fit.backends import regularizer_selector +from n3fit.layers import (DIS, DY, AddPhoton, FkRotation, FlavourToEvolution, + ObsRotation, Preprocessing, losses) +from n3fit.layers.observable import is_unique +from n3fit.msr import msr_impose @dataclass @@ -351,7 +352,9 @@ def generate_dense_per_flavour_network( initializers = [] for _ in range(basis_size): # select the initializer and move the seed - initializers.append(MetaLayer.select_initializer(initializer_name, seed=current_seed)) + initializers.append( + MetaLayer.select_initializer(initializer_name, seed=current_seed) + ) current_seed += 1 # set the arguments that will define the layer @@ -496,7 +499,7 @@ def pdfNN_layer_generator( will be a (1, None, 2) tensor where dim [:,:,0] is scaled parallel_models: int How many models should be trained in parallel - photon: :py:class:`validphys.photon.compute.Photon` + photon: :py:class:`validphys.photon.compute.Photon` If given, gives the AddPhoton layer a function to compute a photon which will be added at the index 0 of the 14-size FK basis This same function will also be used to compute the MSR component for the photon @@ -573,7 +576,9 @@ def pdfNN_layer_generator( # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photons=photons) + sumrule_layer, integrator_input = msr_impose( + mode=impose_sumrule, scaler=scaler, photons=photons + ) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index fb165aa111..3f62f9b131 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -11,16 +11,18 @@ import logging from collections import namedtuple from itertools import zip_longest + import numpy as np from scipy.interpolate import PchipInterpolator +from validphys.photon.compute import Photon + +import n3fit.hyper_optimization.penalties +import n3fit.hyper_optimization.rewards from n3fit import model_gen -from n3fit.backends import MetaModel, clear_backend_state, callbacks +from n3fit.backends import MetaModel, callbacks, clear_backend_state from n3fit.backends import operations as op from n3fit.stopping import Stopping from n3fit.vpinterface import N3PDF -import n3fit.hyper_optimization.penalties -import n3fit.hyper_optimization.rewards -from validphys.photon.compute import Photon log = logging.getLogger(__name__) @@ -37,6 +39,7 @@ # See ModelTrainer::_xgrid_generation for the definition of each field and how they are generated InputInfo = namedtuple("InputInfo", ["input", "split", "idx"]) + def _pdf_injection(pdf_layers, observables, masks): """ Takes as input a list of PDF layers each corresponding to one observable (also given as a list) @@ -192,7 +195,9 @@ def __init__( hyper_loss = kfold_parameters.get("target", None) if hyper_loss is None: hyper_loss = "average" - log.warning("No minimization target selected, defaulting to '%s'", hyper_loss) + log.warning( + "No minimization target selected, defaulting to '%s'", hyper_loss + ) log.info("Using '%s' as the target for hyperoptimization", hyper_loss) self._hyper_loss = getattr(n3fit.hyper_optimization.rewards, hyper_loss) @@ -468,7 +473,9 @@ def _model_generation(self, xinput, pdf_models, partition, partition_idx): validation = MetaModel(full_model_input_dict, output_vl) # Or the positivity in the total chi2 - output_ex = _pdf_injection(exp_pdfs, self.experimental["output"], experimental_mask) + output_ex = _pdf_injection( + exp_pdfs, self.experimental["output"], experimental_mask + ) experimental = MetaModel(full_model_input_dict, output_ex) if self.print_summary: @@ -564,7 +571,9 @@ def _generate_observables( all_pos_initial, all_pos_multiplier, max_lambda, positivity_steps ) - pos_layer = model_gen.observable_generator(pos_dict, positivity_initial=pos_initial) + pos_layer = model_gen.observable_generator( + pos_dict, positivity_initial=pos_initial + ) # The input list is still common self.input_list.append(pos_layer["inputs"]) @@ -579,13 +588,18 @@ def _generate_observables( if self.integ_info is not None: for integ_dict in self.integ_info: if not self.mode_hyperopt: - log.info("Generating integrability penalty for %s", integ_dict["name"]) + log.info( + "Generating integrability penalty for %s", integ_dict["name"] + ) integrability_steps = int(epochs / PUSH_INTEGRABILITY_EACH) max_lambda = integ_dict["lambda"] integ_initial, integ_multiplier = _LM_initial_and_multiplier( - all_integ_initial, all_integ_multiplier, max_lambda, integrability_steps + all_integ_initial, + all_integ_multiplier, + max_lambda, + integrability_steps, ) integ_layer = model_gen.observable_generator( @@ -611,10 +625,15 @@ def _generate_observables( force_set_smallest = input_arr.min() > 1e-9 if force_set_smallest: new_xgrid = np.linspace( - start=1 / input_arr_size, stop=1.0, endpoint=False, num=input_arr_size + start=1 / input_arr_size, + stop=1.0, + endpoint=False, + num=input_arr_size, ) else: - new_xgrid = np.linspace(start=0, stop=1.0, endpoint=False, num=input_arr_size) + new_xgrid = np.linspace( + start=0, stop=1.0, endpoint=False, num=input_arr_size + ) # When mapping the FK xgrids onto our new grid, we need to consider degeneracies among # the x-values in the FK grids @@ -634,7 +653,9 @@ def _generate_observables( # Select the indices of the points that will be used by the interpolator onein = map_from_complete.size / (int(interpolation_points) - 1) - selected_points = [round(i * onein - 1) for i in range(1, int(interpolation_points))] + selected_points = [ + round(i * onein - 1) for i in range(1, int(interpolation_points)) + ] if selected_points[0] != 0: selected_points = [0] + selected_points map_from = map_from_complete[selected_points] @@ -645,7 +666,8 @@ def _generate_observables( scaler = PchipInterpolator(map_from, map_to) except ValueError: raise ValueError( - "interpolation_points is larger than the number of unique " "input x-values" + "interpolation_points is larger than the number of unique " + "input x-values" ) self._scaler = lambda x: np.concatenate([scaler(np.log(x)), x], axis=-1) @@ -686,7 +708,7 @@ def _generate_pdf( dictionary of arguments for the regularizer seed: int seed for the NN - photons: :py:class:`validphys.photon.compute.Photon` + photons: :py:class:`validphys.photon.compute.Photon` function to compute the photon PDF see model_gen.pdfNN_layer_generator for more information @@ -722,7 +744,14 @@ def _prepare_reporting(self, partition): to select the bits necessary for reporting the chi2. Receives the chi2 partition data to see whether any dataset is to be left out """ - reported_keys = ["name", "count_chi2", "positivity", "integrability", "ndata", "ndata_vl"] + reported_keys = [ + "name", + "count_chi2", + "positivity", + "integrability", + "ndata", + "ndata_vl", + ] reporting_list = [] for exp_dict in self.all_info: reporting_dict = {k: exp_dict.get(k) for k in reported_keys} @@ -823,7 +852,10 @@ def evaluate(self, stopping_object): # training and the validation which are actually `chi2` and not part of the penalty train_chi2 = stopping_object.evaluate_training(self.training["model"]) val_chi2 = stopping_object.vl_chi2 - exp_chi2 = self.experimental["model"].compute_losses()["loss"] / self.experimental["ndata"] + exp_chi2 = ( + self.experimental["model"].compute_losses()["loss"] + / self.experimental["ndata"] + ) return train_chi2, val_chi2, exp_chi2 def hyperparametrizable(self, params): @@ -886,13 +918,13 @@ def hyperparametrizable(self, params): xinput = self._xgrid_generation() # Initialize all photon classes for the different replicas: if self.lux_params: - photons=Photon( + photons = Photon( theoryid=self.theoryid, lux_params=self.lux_params, replicas=self.replicas, ) else: - photons=None + photons = None ### Training loop for k, partition in enumerate(self.kpartitions): # Each partition of the kfolding needs to have its own separate model @@ -919,7 +951,6 @@ def hyperparametrizable(self, params): pl = m.get_layer("add_photon") pl.register_photon(xinput.input.tensor_content) - # Model generation joins all the different observable layers # together with pdf model generated above models = self._model_generation(xinput, pdf_models, partition, k) @@ -932,8 +963,12 @@ def hyperparametrizable(self, params): if k > 0: # Reset the positivity and integrability multipliers - pos_and_int = self.training["posdatasets"] + self.training["integdatasets"] - initial_values = self.training["posinitials"] + self.training["posinitials"] + pos_and_int = ( + self.training["posdatasets"] + self.training["integdatasets"] + ) + initial_values = ( + self.training["posinitials"] + self.training["posinitials"] + ) models["training"].reset_layer_weights_to(pos_and_int, initial_values) # Generate the list containing reporting info necessary for chi2 @@ -974,10 +1009,14 @@ def hyperparametrizable(self, params): validation_loss = np.mean(stopping_object.vl_chi2) # Compute experimental loss - exp_loss_raw = np.average(models["experimental"].compute_losses()["loss"]) + exp_loss_raw = np.average( + models["experimental"].compute_losses()["loss"] + ) # And divide by the number of active points in this fold # it would be nice to have a ndata_per_fold variable coming in the vp object... - ndata = np.sum([np.count_nonzero(i[k]) for i in self.experimental["folds"]]) + ndata = np.sum( + [np.count_nonzero(i[k]) for i in self.experimental["folds"]] + ) # If ndata == 0 then it's the opposite, all data is in! if ndata == 0: ndata = self.experimental["ndata"] @@ -985,12 +1024,18 @@ def hyperparametrizable(self, params): hyper_loss = experimental_loss if passed != self.pass_status: - log.info("Hyperparameter combination fail to find a good fit, breaking") + log.info( + "Hyperparameter combination fail to find a good fit, breaking" + ) # If the fit failed to fit, no need to add a penalty to the loss break for penalty in self.hyper_penalties: - hyper_loss += penalty(pdf_models=pdf_models, stopping_object=stopping_object) - log.info("Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed) + hyper_loss += penalty( + pdf_models=pdf_models, stopping_object=stopping_object + ) + log.info( + "Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed + ) # Now save all information from this fold l_hyper.append(hyper_loss) @@ -1039,5 +1084,9 @@ def hyperparametrizable(self, params): # In a normal run, the only information we need to output is the stopping object # (which contains metadata about the stopping) # and the pdf models (which are used to generate the PDF grids and compute arclengths) - dict_out = {"status": passed, "stopping_object": stopping_object, "pdf_models": pdf_models} + dict_out = { + "status": passed, + "stopping_object": stopping_object, + "pdf_models": pdf_models, + } return dict_out diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index b43ec40bfb..55f74a3dab 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -2,11 +2,11 @@ The constraint module include functions to impose the momentum sum rules on the PDFs """ import logging + import numpy as np -from n3fit.layers import xDivide, MSR_Normalization, xIntegrator from n3fit.backends import operations as op - +from n3fit.layers import MSR_Normalization, xDivide, xIntegrator log = logging.getLogger(__name__) @@ -36,28 +36,28 @@ def gen_integration_input(nx): return xgrid, weights_array -def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): +def msr_impose(nx=int(2e3), mode="All", scaler=None, photons=None): """ - This function receives: - Generates a function that applies a normalization layer to the fit. - - fit_layer: the 8-basis layer of PDF which we fit - The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) - - final_layer: the 14-basis which is fed to the fktable - and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). - It uses pdf_fit to compute the sum rule and returns a modified version of - the final_pdf layer with a normalisation by which the sum rule is imposed - - Parameters - ---------- - nx: int - number of points for the integration grid, default: 2000 - mode: str - what sum rules to compute (MSR, VSR or All), default: All - scaler: scaler - Function to apply to the input. If given the input to the model - will be a (1, None, 2) tensor where dim [:,:,0] is scaled - photon: :py:class:`validphys.photon.compute.Photon` - If given, gives the AddPhoton layer a function to compute the MSR component for the photon + This function receives: + Generates a function that applies a normalization layer to the fit. + - fit_layer: the 8-basis layer of PDF which we fit + The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) + - final_layer: the 14-basis which is fed to the fktable + and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). + It uses pdf_fit to compute the sum rule and returns a modified version of + the final_pdf layer with a normalisation by which the sum rule is imposed + + Parameters + ---------- + nx: int + number of points for the integration grid, default: 2000 + mode: str + what sum rules to compute (MSR, VSR or All), default: All + scaler: scaler + Function to apply to the input. If given the input to the model + will be a (1, None, 2) tensor where dim [:,:,0] is scaled + photon: :py:class:`validphys.photon.compute.Photon` + If given, gives the AddPhoton layer a function to compute the MSR component for the photon """ # 1. Generate the fake input which will be used to integrate @@ -86,14 +86,16 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): # and will return it appropiately normalized. def apply_normalization(layer_pdf, ph_replica): """ - layer_pdf: output of the PDF, unnormalized, ready for the fktable + layer_pdf: output of the PDF, unnormalized, ready for the fktable """ x_original = op.op_gather_keep_dims(xgrid_input, -1, axis=-1) - pdf_integrand = op.op_multiply([division_by_x(x_original), layer_pdf(xgrid_input)]) + pdf_integrand = op.op_multiply( + [division_by_x(x_original), layer_pdf(xgrid_input)] + ) normalization = normalizer(integrator(pdf_integrand), ph_replica) def ultimate_pdf(x): - return layer_pdf(x)*normalization + return layer_pdf(x) * normalization return ultimate_pdf diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 03c36aa1b1..c2334410d9 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -5,7 +5,9 @@ # Backend-independent imports import copy import logging + import numpy as np + import n3fit.checks from n3fit.vpinterface import N3PDF @@ -18,8 +20,8 @@ @n3fit.checks.check_fiatlux_pdfs_id def performfit( *, - n3fit_checks_action, # wrapper for all checks - replicas, # checks specific to performfit + n3fit_checks_action, # wrapper for all checks + replicas, # checks specific to performfit replicas_nnseed_fitting_data_dict, posdatasets_fitting_pos_dict, integdatasets_fitting_integ_dict, @@ -42,86 +44,86 @@ def performfit( parallel_models=False, ): """ - This action will (upon having read a validcard) process a full PDF fit - for a set of replicas. - - The input to this function is provided by validphys - and/or defined in the runcards or commandline arguments. - - This controller is provided with: - 1. Seeds generated using the replica number and the seeds defined in the runcard. - 2. Loaded datasets with replicas generated. - 2.1 Loaded positivity/integrability sets. - - The workflow of this controller is as follows: - 1. Generate a ModelTrainer object holding information to create the NN and perform a fit - (at this point no NN object has been generated) - 1.1 (if hyperopt) generates the hyperopt scanning dictionary - taking as a base the fitting dictionary and the runcard's hyperscanner dictionary - 2. Pass the dictionary of parameters to ModelTrainer - for the NN to be generated and the fit performed - 2.1 (if hyperopt) Loop over point 4 for `hyperopt` number of times - 3. Once the fit is finished, output the PDF grid and accompanying files - - Parameters - ---------- - genrep: bool - Whether or not to generate MC replicas. (Only used for checks) - data: validphys.core.DataGroupSpec - containing the datasets to be included in the fit. (Only used - for checks) - replicas_nnseed_fitting_data_dict: list[tuple] - list with element for each replica (typically just one) to be - fitted. Each element - is a tuple containing the replica number, nnseed and - ``fitted_data_dict`` containing all of the data, metadata - for each group of datasets which is to be fitted. - posdatasets_fitting_pos_dict: list[dict] - list of dictionaries containing all data and metadata for each - positivity dataset - integdatasets_fitting_integ_dict: list[dict] - list of dictionaries containing all data and metadata for each - integrability dataset - theoryid: validphys.core.TheoryIDSpec - Theory which is used to generate theory predictions from model - during fit. Object also contains some metadata on the theory - settings. - fiatlux: dict - dictionary containing the params needed from LuxQED - basis: list[dict] - preprocessing information for each flavour to be fitted. - fitbasis: str - Valid basis which the fit is to be ran in. Available bases can - be found in :py:mod:`validphys.pdfbases`. - sum_rules: bool - Whether to impose sum rules in fit. By default set to True - parameters: dict - Mapping containing parameters which define the network - architecture/fitting methodology. - replica_path: pathlib.Path - path to the output of this run - output_path: str - name of the fit - save: None, str - model file where weights will be saved, used in conjunction with - ``load``. - load: None, str - model file from which to load weights from. - hyperscanner: dict - dictionary containing the details of the hyperscanner - hyperopt: int - if given, number of hyperopt iterations to run - kfold_parameters: None, dict - dictionary with kfold settings used in hyperopt. - tensorboard: None, dict - mapping containing tensorboard settings if it is to be used. By - default it is None and tensorboard is not enabled. - debug: bool - activate some debug options - maxcores: int - maximum number of (logical) cores that the backend should be aware of - parallel_models: bool - whether to run models in parallel + This action will (upon having read a validcard) process a full PDF fit + for a set of replicas. + + The input to this function is provided by validphys + and/or defined in the runcards or commandline arguments. + + This controller is provided with: + 1. Seeds generated using the replica number and the seeds defined in the runcard. + 2. Loaded datasets with replicas generated. + 2.1 Loaded positivity/integrability sets. + + The workflow of this controller is as follows: + 1. Generate a ModelTrainer object holding information to create the NN and perform a fit + (at this point no NN object has been generated) + 1.1 (if hyperopt) generates the hyperopt scanning dictionary + taking as a base the fitting dictionary and the runcard's hyperscanner dictionary + 2. Pass the dictionary of parameters to ModelTrainer + for the NN to be generated and the fit performed + 2.1 (if hyperopt) Loop over point 4 for `hyperopt` number of times + 3. Once the fit is finished, output the PDF grid and accompanying files + + Parameters + ---------- + genrep: bool + Whether or not to generate MC replicas. (Only used for checks) + data: validphys.core.DataGroupSpec + containing the datasets to be included in the fit. (Only used + for checks) + replicas_nnseed_fitting_data_dict: list[tuple] + list with element for each replica (typically just one) to be + fitted. Each element + is a tuple containing the replica number, nnseed and + ``fitted_data_dict`` containing all of the data, metadata + for each group of datasets which is to be fitted. + posdatasets_fitting_pos_dict: list[dict] + list of dictionaries containing all data and metadata for each + positivity dataset + integdatasets_fitting_integ_dict: list[dict] + list of dictionaries containing all data and metadata for each + integrability dataset + theoryid: validphys.core.TheoryIDSpec + Theory which is used to generate theory predictions from model + during fit. Object also contains some metadata on the theory + settings. + fiatlux: dict + dictionary containing the params needed from LuxQED + basis: list[dict] + preprocessing information for each flavour to be fitted. + fitbasis: str + Valid basis which the fit is to be ran in. Available bases can + be found in :py:mod:`validphys.pdfbases`. + sum_rules: bool + Whether to impose sum rules in fit. By default set to True + parameters: dict + Mapping containing parameters which define the network + architecture/fitting methodology. + replica_path: pathlib.Path + path to the output of this run + output_path: str + name of the fit + save: None, str + model file where weights will be saved, used in conjunction with + ``load``. + load: None, str + model file from which to load weights from. + hyperscanner: dict + dictionary containing the details of the hyperscanner + hyperopt: int + if given, number of hyperopt iterations to run + kfold_parameters: None, dict + dictionary with kfold settings used in hyperopt. + tensorboard: None, dict + mapping containing tensorboard settings if it is to be used. By + default it is None and tensorboard is not enabled. + debug: bool + activate some debug options + maxcores: int + maximum number of (logical) cores that the backend should be aware of + parallel_models: bool + whether to run models in parallel """ from n3fit.backends import set_initial_state @@ -134,8 +136,8 @@ def performfit( # All potentially backend dependent imports should come inside the fit function # so they can eventually be set from the runcard - from n3fit.model_trainer import ModelTrainer from n3fit.io.writer import WriterWrapper + from n3fit.model_trainer import ModelTrainer # Note: there are three possible scenarios for the loop of replicas: # 1.- Only one replica is being run, in this case the loop is only evaluated once @@ -162,10 +164,12 @@ def performfit( training_data = [] validation_data = [] for i_rep in range(n_models): - training_data.append(replica_experiments[i_rep][i_exp]['expdata']) - validation_data.append(replica_experiments[i_rep][i_exp]['expdata_vl']) - all_experiments[i_exp]['expdata'] = np.concatenate(training_data, axis=0) - all_experiments[i_exp]['expdata_vl'] = np.concatenate(validation_data, axis=0) + training_data.append(replica_experiments[i_rep][i_exp]["expdata"]) + validation_data.append(replica_experiments[i_rep][i_exp]["expdata_vl"]) + all_experiments[i_exp]["expdata"] = np.concatenate(training_data, axis=0) + all_experiments[i_exp]["expdata_vl"] = np.concatenate( + validation_data, axis=0 + ) log.info( "Starting parallel fits from replica %d to %d", replicas[0], @@ -262,7 +266,9 @@ def performfit( log.info("Stopped at epoch=%d", stopping_object.stop_epoch) final_time = stopwatch.stop() - all_training_chi2, all_val_chi2, all_exp_chi2 = the_model_trainer.evaluate(stopping_object) + all_training_chi2, all_val_chi2, all_exp_chi2 = the_model_trainer.evaluate( + stopping_object + ) pdf_models = result["pdf_models"] for i, (replica_number, pdf_model) in enumerate(zip(replica_idxs, pdf_models)): @@ -292,13 +298,12 @@ def performfit( replica_path_set, output_path.name, training_chi2, val_chi2, exp_chi2 ) log.info( - "Best fit for replica #%d, chi2=%.3f (tr=%.3f, vl=%.3f)", - replica_number, - exp_chi2, - training_chi2, - val_chi2 - ) - + "Best fit for replica #%d, chi2=%.3f (tr=%.3f, vl=%.3f)", + replica_number, + exp_chi2, + training_chi2, + val_chi2, + ) # Save the weights to some file for the given replica if save: diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 9b80cb5308..71f4aada82 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -3,28 +3,23 @@ n3fit - performs fit using ml external frameworks """ -import sys +import argparse +import logging +import pathlib import re import shutil -import pathlib -import logging +import sys import warnings -import argparse -from validphys.app import App -from validphys.config import Environment, Config -from validphys.config import EnvironmentError_, ConfigError -from validphys.core import FitSpec from reportengine import colors from reportengine.compat import yaml from reportengine.namespaces import NSList +from validphys.app import App +from validphys.config import (Config, ConfigError, Environment, + EnvironmentError_) +from validphys.core import FitSpec - -N3FIT_FIXED_CONFIG = dict( - use_cuts = 'internal', - use_t0 = True, - actions_ = [] -) +N3FIT_FIXED_CONFIG = dict(use_cuts="internal", use_t0=True, actions_=[]) FIT_NAMESPACE = "datacuts::theory::fitting " CLOSURE_NAMESPACE = "datacuts::theory::closuretest::fitting " @@ -45,6 +40,7 @@ INPUT_FOLDER = "input" TAB_FOLDER = "tables" + class N3FitError(Exception): """Exception raised when n3fit cannot succeed and knows why""" @@ -64,12 +60,12 @@ def init_output(self): # check if results folder exists self.output_path = pathlib.Path(self.output_path).absolute() - if not (self.output_path/"nnfit").is_dir(): + if not (self.output_path / "nnfit").is_dir(): if not re.fullmatch(r"[\w.\-]+", self.output_path.name): raise N3FitError("Invalid output folder name. Must be alphanumeric.") try: self.output_path.mkdir(exist_ok=True) - (self.output_path /"nnfit").mkdir(exist_ok=True) + (self.output_path / "nnfit").mkdir(exist_ok=True) except OSError as e: raise EnvironmentError_(e) from e @@ -124,18 +120,23 @@ def from_yaml(cls, o, *args, **kwargs): except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError(f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'.") + raise ConfigError( + f"Expecting input runcard to be a mapping, " + f"not '{type(file_content)}'." + ) - if file_content.get('closuretest') is not None: + if file_content.get("closuretest") is not None: namespace = CLOSURE_NAMESPACE else: namespace = FIT_NAMESPACE - N3FIT_FIXED_CONFIG['actions_'].append(namespace + "performfit") + N3FIT_FIXED_CONFIG["actions_"].append(namespace + "performfit") if fps := file_content["fitting"].get("savepseudodata", True): if fps != True: - raise TypeError(f"fitting::savepseudodata is neither True nor False ({fps})") + raise TypeError( + f"fitting::savepseudodata is neither True nor False ({fps})" + ) if len(kwargs["environment"].replicas) != 1: raise ConfigError( "Cannot request that multiple replicas are fitted and that " @@ -146,27 +147,40 @@ def from_yaml(cls, o, *args, **kwargs): training_action = namespace + "training_pseudodata" validation_action = namespace + "validation_pseudodata" - N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) + N3FIT_FIXED_CONFIG["actions_"].extend((training_action, validation_action)) - if (thconfig:=file_content.get('fiatlux')): - N3FIT_FIXED_CONFIG['fiatlux']=thconfig + if thconfig := file_content.get("fiatlux"): + N3FIT_FIXED_CONFIG["fiatlux"] = thconfig else: - N3FIT_FIXED_CONFIG['fiatlux']=None - #Theorycovmat flags and defaults - N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False - N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = False - N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = False - if (thconfig:=file_content.get('theorycovmatconfig')) is not None: - N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = thconfig.get('use_thcovmat_in_fitting', True) - N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = thconfig.get('use_thcovmat_in_sampling', True) - if N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] or N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting']: - N3FIT_FIXED_CONFIG['theory_covmat_flag'] = True - N3FIT_FIXED_CONFIG['use_user_uncertainties'] = thconfig.get('use_user_uncertainties', False) - N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = thconfig.get('use_scalevar_uncertainties', True) - #Sampling flags - if (sam_t0:=file_content.get('sampling')) is not None: - N3FIT_FIXED_CONFIG['separate_multiplicative'] = sam_t0.get('separate_multiplicative', True) - #Fitting flag + N3FIT_FIXED_CONFIG["fiatlux"] = None + # Theorycovmat flags and defaults + N3FIT_FIXED_CONFIG["theory_covmat_flag"] = False + N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = False + N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = False + if (thconfig := file_content.get("theorycovmatconfig")) is not None: + N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = thconfig.get( + "use_thcovmat_in_fitting", True + ) + N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = thconfig.get( + "use_thcovmat_in_sampling", True + ) + if ( + N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] + or N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] + ): + N3FIT_FIXED_CONFIG["theory_covmat_flag"] = True + N3FIT_FIXED_CONFIG["use_user_uncertainties"] = thconfig.get( + "use_user_uncertainties", False + ) + N3FIT_FIXED_CONFIG["use_scalevar_uncertainties"] = thconfig.get( + "use_scalevar_uncertainties", True + ) + # Sampling flags + if (sam_t0 := file_content.get("sampling")) is not None: + N3FIT_FIXED_CONFIG["separate_multiplicative"] = sam_t0.get( + "separate_multiplicative", True + ) + # Fitting flag file_content.update(N3FIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -183,12 +197,13 @@ def parse_fakedata(self, fakedata: bool): """ if fakedata: log.warning("using filtered closure data") - if not (self.environment.output_path/'filter').is_dir(): + if not (self.environment.output_path / "filter").is_dir(): raise ConfigError( "Could not find filter result at " f"{self.environment.output_path/'filter'} " "to load commondata from. Did you run filter on the " - "runcard?") + "runcard?" + ) return fakedata def produce_use_fitcommondata(self, fakedata): @@ -198,8 +213,7 @@ def produce_use_fitcommondata(self, fakedata): return fakedata def produce_kfold_parameters(self, kfold=None, hyperopt=None): - """Return None even if there are kfolds in the runcard if the hyperopt flag is not active - """ + """Return None even if there are kfolds in the runcard if the hyperopt flag is not active""" if hyperopt is not None: return kfold return None @@ -237,18 +251,27 @@ def __init__(self): @property def argparser(self): parser = super().argparser - parser.add_argument("-o", "--output", help="Output folder and name of the fit", default=None) + parser.add_argument( + "-o", "--output", help="Output folder and name of the fit", default=None + ) def check_positive(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError("%s is an invalid positive int value." % value) + raise argparse.ArgumentTypeError( + "%s is an invalid positive int value." % value + ) return ivalue - parser.add_argument("--hyperopt", help="Enable hyperopt scan", default=None, type=int) + parser.add_argument( + "--hyperopt", help="Enable hyperopt scan", default=None, type=int + ) parser.add_argument("replica", help="MC replica number", type=check_positive) parser.add_argument( - "-r", "--replica_range", help="End of the range of replicas to compute", type=check_positive + "-r", + "--replica_range", + help="End of the range of replicas to compute", + type=check_positive, ) return parser @@ -260,7 +283,9 @@ def get_commandline_arguments(self, cmdline=None): def run(self): try: - self.environment.config_yml = pathlib.Path(self.args["config_yml"]).absolute() + self.environment.config_yml = pathlib.Path( + self.args["config_yml"] + ).absolute() replica = self.args["replica"] if self.args["replica_range"]: replicas = list(range(replica, self.args["replica_range"] + 1)) @@ -274,7 +299,9 @@ def run(self): sys.exit(1) except Exception as e: log.critical(f"Bug in n3fit ocurred. Please report it.") - print(colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr) + print( + colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr + ) sys.exit(1) diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index d6a4f4b509..24b39c8987 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -25,39 +25,40 @@ # top. -import sys +import hashlib +import logging +import pathlib import re import shutil -import pathlib -import logging -import hashlib +import sys import warnings -from validphys.config import Environment, Config, EnvironmentError_, ConfigError -from validphys.app import App -from reportengine.compat import yaml from reportengine import colors - +from reportengine.compat import yaml +from validphys.app import App +from validphys.config import (Config, ConfigError, Environment, + EnvironmentError_) SETUPFIT_FIXED_CONFIG = dict( actions_=[ - 'datacuts check_t0pdfset', - 'theory check_positivity', - ]) - -SETUPFIT_PROVIDERS = ['validphys.filters', - 'validphys.theorycovariance.construction', - 'validphys.results', - 'validphys.covmats', - 'n3fit.n3fit_checks_provider' + "datacuts check_t0pdfset", + "theory check_positivity", + ] +) + +SETUPFIT_PROVIDERS = [ + "validphys.filters", + "validphys.theorycovariance.construction", + "validphys.results", + "validphys.covmats", + "n3fit.n3fit_checks_provider", ] SETUPFIT_DEFAULTS = dict( - use_cuts = 'internal', + use_cuts="internal", ) - log = logging.getLogger(__name__) RUNCARD_COPY_FILENAME = "filter.yml" @@ -69,11 +70,13 @@ class SetupFitError(Exception): """Exception raised when setup-fit cannot succeed and knows why""" + pass class SetupFitEnvironment(Environment): """Container for information to be filled at run time""" + def init_output(self): # check file exists, is a file, has extension. if not self.config_yml.exists(): @@ -86,9 +89,11 @@ def init_output(self): self.output_path = pathlib.Path(self.output_path).absolute() if self.output_path.is_dir(): - log.warning(f"Output folder exists: {self.output_path} Overwriting contents") + log.warning( + f"Output folder exists: {self.output_path} Overwriting contents" + ) else: - if not re.fullmatch(r'[\w\-]+', self.output_path.name): + if not re.fullmatch(r"[\w\-]+", self.output_path.name): raise SetupFitError("Invalid output folder name. Must be alphanumeric.") try: self.output_path.mkdir() @@ -114,16 +119,18 @@ def init_output(self): def save_md5(self): """Generate md5 key from file""" output_filename = self.output_path / MD5_FILENAME - with open(self.config_yml, 'rb') as f: + with open(self.config_yml, "rb") as f: hash_md5 = hashlib.md5(f.read()).hexdigest() - with open(output_filename, 'w') as g: + with open(output_filename, "w") as g: g.write(hash_md5) log.info(f"md5 {hash_md5} stored in {output_filename}") @classmethod def ns_dump_description(cls): - return {'filter_path': "The filter output folder", - **super().ns_dump_description()} + return { + "filter_path": "The filter output folder", + **super().ns_dump_description(), + } class SetupFitConfig(Config): @@ -133,37 +140,41 @@ class SetupFitConfig(Config): def from_yaml(cls, o, *args, **kwargs): try: with warnings.catch_warnings(): - warnings.simplefilter('ignore', - yaml.error.MantissaNoDotYAML1_1Warning) - #We need to specify the older version 1.1 to support the - #older configuration files, which liked to use on/off for - #booleans. - #The floating point parsing yields warnings everywhere, which - #we suppress. - file_content = yaml.safe_load(o, version='1.1') + warnings.simplefilter("ignore", yaml.error.MantissaNoDotYAML1_1Warning) + # We need to specify the older version 1.1 to support the + # older configuration files, which liked to use on/off for + # booleans. + # The floating point parsing yields warnings everywhere, which + # we suppress. + file_content = yaml.safe_load(o, version="1.1") except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError(f"Expecting input runcard to be a mapping, " - f"not '{type(file_content)}'.") - - if file_content.get('closuretest') is not None: - filter_action = 'datacuts::closuretest::theory::fitting filter' - check_n3fit_action = 'datacuts::theory::closuretest::fitting n3fit_checks_action' + raise ConfigError( + f"Expecting input runcard to be a mapping, " + f"not '{type(file_content)}'." + ) + + if file_content.get("closuretest") is not None: + filter_action = "datacuts::closuretest::theory::fitting filter" + check_n3fit_action = ( + "datacuts::theory::closuretest::fitting n3fit_checks_action" + ) else: - filter_action = 'datacuts::theory::fitting filter' - check_n3fit_action = 'datacuts::theory::fitting n3fit_checks_action' - SETUPFIT_FIXED_CONFIG['actions_'] += [check_n3fit_action, filter_action] - if file_content.get('theorycovmatconfig') is not None: - SETUPFIT_FIXED_CONFIG['actions_'].append( - 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat') - if file_content.get('fiatlux') is not None: - SETUPFIT_FIXED_CONFIG['actions_'].append( - 'fiatlux check_luxset') - if file_content.get('fiatlux')["additional_errors"]: - SETUPFIT_FIXED_CONFIG['actions_'].append( - 'fiatlux check_additional_errors') - for k,v in SETUPFIT_DEFAULTS.items(): + filter_action = "datacuts::theory::fitting filter" + check_n3fit_action = "datacuts::theory::fitting n3fit_checks_action" + SETUPFIT_FIXED_CONFIG["actions_"] += [check_n3fit_action, filter_action] + if file_content.get("theorycovmatconfig") is not None: + SETUPFIT_FIXED_CONFIG["actions_"].append( + "datacuts::theory::theorycovmatconfig nnfit_theory_covmat" + ) + if file_content.get("fiatlux") is not None: + SETUPFIT_FIXED_CONFIG["actions_"].append("fiatlux check_luxset") + if file_content.get("fiatlux")["additional_errors"]: + SETUPFIT_FIXED_CONFIG["actions_"].append( + "fiatlux check_additional_errors" + ) + for k, v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) file_content.update(SETUPFIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -171,31 +182,35 @@ def from_yaml(cls, o, *args, **kwargs): class SetupFitApp(App): """The class which parsers and perform the filtering""" + environment_class = SetupFitEnvironment config_class = SetupFitConfig def __init__(self): - super(SetupFitApp, self).__init__(name='setup-fit', - providers=SETUPFIT_PROVIDERS) + super(SetupFitApp, self).__init__( + name="setup-fit", providers=SETUPFIT_PROVIDERS + ) @property def argparser(self): parser = super().argparser - parser.add_argument('-o','--output', - help="Output folder and name of the fit", - default=None) + parser.add_argument( + "-o", "--output", help="Output folder and name of the fit", default=None + ) return parser def get_commandline_arguments(self, cmdline=None): args = super().get_commandline_arguments(cmdline) - if args['output'] is None: - args['output'] = pathlib.Path(args['config_yml']).stem + if args["output"] is None: + args["output"] = pathlib.Path(args["config_yml"]).stem return args def run(self): try: # set folder output name - self.environment.config_yml = pathlib.Path(self.args['config_yml']).absolute() + self.environment.config_yml = pathlib.Path( + self.args["config_yml"] + ).absolute() # proceed with default run super().run() @@ -208,8 +223,8 @@ def run(self): except Exception as e: log.critical(f"Bug in setup-fit ocurred. Please report it.") print( - colors.color_exception(e.__class__, e, e.__traceback__), - file=sys.stderr) + colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr + ) sys.exit(1) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 6c76f73d87..637aa9021c 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -20,12 +20,13 @@ """ import logging from collections.abc import Iterable + import numpy as np import numpy.linalg as la +from validphys.arclength import arc_lengths, integrability_number from validphys.core import PDF, MCStats -from validphys.pdfbases import ALL_FLAVOURS, check_basis from validphys.lhapdfset import LHAPDFSet -from validphys.arclength import integrability_number, arc_lengths +from validphys.pdfbases import ALL_FLAVOURS, check_basis log = logging.getLogger(__name__) # Order of the evolution basis output from n3fit @@ -74,7 +75,9 @@ def xfxQ(self, x, Q, n, fl): """Return the value of the PDF member for the given value in x""" if Q != self._fitting_q: log.warning( - "Querying N3LHAPDFSet at a value of Q=%f different from %f", Q, self._fitting_q + "Querying N3LHAPDFSet at a value of Q=%f different from %f", + Q, + self._fitting_q, ) return self.grid_values([fl], [x]).squeeze()[n] @@ -83,7 +86,7 @@ def _register_photon(self, xgrid): for m in self._lhapdf_set: pl = m.get_layer_re("add_photon") # if pl is an empy list there's no photon - if not pl : + if not pl: continue pl[0].register_photon(xgrid) # Recompile the model if necessary @@ -120,7 +123,9 @@ def __call__(self, xarr, flavours=None, replica=None): if replica is None or replica == 0: # We need generate output values for all replicas - result = np.concatenate([m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0) + result = np.concatenate( + [m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0 + ) if replica == 0: # We want _only_ the central value result = np.mean(result, axis=0, keepdims=True) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index c71c2a8354..a1bdff9610 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -4,60 +4,37 @@ @author: Zahari Kassabov """ -import logging -import pathlib +import copy import functools +import glob import inspect +import logging import numbers -import copy -from importlib.resources import read_text, contents - +import pathlib from collections import ChainMap, defaultdict from collections.abc import Mapping, Sequence +from importlib.resources import contents, read_text import pandas as pd -import glob - -from reportengine import configparser +import validphys.scalevariations +from reportengine import configparser, report +from reportengine.compat import yaml +from reportengine.configparser import (ConfigError, _parse_func, element_of, + record_from_defaults) from reportengine.environment import Environment, EnvironmentError_ -from reportengine.configparser import ( - ConfigError, - element_of, - _parse_func, - record_from_defaults, -) from reportengine.helputils import get_parser_type from reportengine.namespaces import NSList -from reportengine import report -from reportengine.compat import yaml - -from validphys.core import ( - DataGroupSpec, - DataSetInput, - ExperimentInput, - CutsPolicy, - MatchedCuts, - SimilarCuts, - ThCovMatSpec, -) +from validphys.core import (CutsPolicy, DataGroupSpec, DataSetInput, + ExperimentInput, MatchedCuts, SimilarCuts, + ThCovMatSpec) from validphys.fitdata import fitted_replica_indexes, num_fitted_replicas -from validphys.loader import ( - Loader, - LoaderError, - LoadFailedError, - DataNotFoundError, - PDFNotFound, - FallbackLoader, - InconsistentMetaDataError, -) from validphys.gridvalues import LUMI_CHANNELS - +from validphys.loader import (DataNotFoundError, FallbackLoader, + InconsistentMetaDataError, Loader, LoaderError, + LoadFailedError, PDFNotFound) from validphys.paramfits.config import ParamfitsConfig - from validphys.plotoptions import get_info -import validphys.scalevariations - log = logging.getLogger(__name__) @@ -65,7 +42,13 @@ class Environment(Environment): """Container for information to be filled at run time""" def __init__( - self, *, this_folder=None, net=True, upload=False, dry=False, **kwargs, + self, + *, + this_folder=None, + net=True, + upload=False, + dry=False, + **kwargs, ): if this_folder: self.this_folder = pathlib.Path(this_folder) @@ -223,7 +206,7 @@ def produce_inclusive_use_scalevar_uncertainties( point_prescription: (str, None) = None, ): """Whether to use a scale variation uncertainty theory covmat. - Checks whether a point prescription is included in the runcard and if so + Checks whether a point prescription is included in the runcard and if so assumes scale uncertainties are to be used.""" if (not use_scalevar_uncertainties) and (point_prescription is not None): use_scalevar_uncertainties = True @@ -255,8 +238,7 @@ def produce_pdfreplicas(self, fitpdf): return NSList(replicas, nskey="replica") def produce_fitcontextwithcuts(self, fit, fitinputcontext): - """Like fitinputcontext but setting the cuts policy. - """ + """Like fitinputcontext but setting the cuts policy.""" theoryid = fitinputcontext["theoryid"] data_input = fitinputcontext["data_input"] @@ -336,8 +318,7 @@ def parse_hyperscan(self, hyperscan): ) from e def parse_hyperscan_config(self, hyperscan_config, hyperopt=None): - """Configuration of the hyperscan - """ + """Configuration of the hyperscan""" if "from_hyperscan" in hyperscan_config: hyperscan = self.parse_hyperscan(hyperscan_config["from_hyperscan"]) log.info( @@ -398,7 +379,7 @@ def produce_basisfromfit(self, fit): return {"basis": basis} def produce_fitpdfandbasis(self, fitpdf, basisfromfit): - """ Set the PDF and basis from the fit config. """ + """Set the PDF and basis from the fit config.""" return {**fitpdf, **basisfromfit} @element_of("dataset_inputs") @@ -520,7 +501,7 @@ def _produce_matched_cuts(self, commondata): return MatchedCuts(cut_list, ndata=ndata) def _produce_similarity_cuts(self, commondata): - """ Compute the intersection between two namespaces (similar to + """Compute the intersection between two namespaces (similar to `fromintersection`) but additionally require that the predictions computed for each dataset across the namespaces are *similar*, specifically that the ratio between the absolute difference in the @@ -559,7 +540,13 @@ def _produce_similarity_cuts(self, commondata): matched_cuts = self._produce_matched_cuts(commondata) inps = [] for i, ns in enumerate(nss): - with self.set_context(ns=self._curr_ns.new_child({**ns,})): + with self.set_context( + ns=self._curr_ns.new_child( + { + **ns, + } + ) + ): # TODO: find a way to not duplicate this and use a dict # instead of a linear search _, dins = self.parse_from_(None, "dataset_inputs", write=False) @@ -613,9 +600,9 @@ def produce_dataset( check_plotting: bool = False, ): """Dataset specification from the theory and CommonData. - Use the cuts from the fit, if provided. If check_plotting is set to - True, attempt to lod and check the PLOTTING files - (note this may cause a noticeable slowdown in general).""" + Use the cuts from the fit, if provided. If check_plotting is set to + True, attempt to lod and check the PLOTTING files + (note this may cause a noticeable slowdown in general).""" name = dataset_input.name sysnum = dataset_input.sys cfac = dataset_input.cfac @@ -652,8 +639,8 @@ def produce_dataset( @configparser.element_of("experiments") def parse_experiment(self, experiment: dict): """A set of datasets where correlated systematics are taken - into account. It is a mapping where the keys are the experiment - name 'experiment' and a list of datasets.""" + into account. It is a mapping where the keys are the experiment + name 'experiment' and a list of datasets.""" try: name, datasets = experiment["experiment"], experiment["datasets"] except KeyError as e: @@ -700,8 +687,8 @@ def produce_experiment_from_input( def produce_sep_mult(self, separate_multiplicative=None): """ - Specifies whether to separate the multiplicative errors in the - experimental covmat construction. The default is True. + Specifies whether to separate the multiplicative errors in the + experimental covmat construction. The default is True. """ if separate_multiplicative is False: return False @@ -761,7 +748,7 @@ def produce_loaded_theory_covmat( ): """ Loads the theory covmat from the correct file according to how it - was generated by vp-setupfit. + was generated by vp-setupfit. """ if theory_covmat_flag is False: return 0.0 @@ -918,7 +905,12 @@ def produce_matched_datasets_from_dataspecs(self, dataspecs): inner_spec_list = inres["dataspecs"] = [] for ispec, spec in enumerate(dataspecs): # Passing spec by referene - d = ChainMap({"dataset_input": all_names[ispec][k],}, spec) + d = ChainMap( + { + "dataset_input": all_names[ispec][k], + }, + spec, + ) inner_spec_list.append(d) res.append(inres) res.sort(key=lambda x: (x["process"], x["dataset_name"])) @@ -942,7 +934,12 @@ def produce_matched_positivity_from_dataspecs(self, dataspecs): l = inres["dataspecs"] = [] for ispec, spec in enumerate(dataspecs): # Passing spec by referene - d = ChainMap({"posdataset": all_names[ispec][k],}, spec) + d = ChainMap( + { + "posdataset": all_names[ispec][k], + }, + spec, + ) l.append(d) res.append(inres) res.sort(key=lambda x: (x["posdataset_name"])) @@ -1029,7 +1026,9 @@ def parse_use_t0(self, do_use_t0: bool): # TODO: Find a good name for this def produce_t0set( - self, t0pdfset=None, use_t0=False, + self, + t0pdfset=None, + use_t0=False, ): """Return the t0set if use_t0 is True and None otherwise. Raises an error if t0 is requested but no t0set is given. @@ -1039,11 +1038,11 @@ def produce_t0set( raise ConfigError("Setting use_t0 requires specifying a valid t0pdfset") return t0pdfset return None - + def parse_luxset(self, name): """PDF set used to generate the photon with fiatlux.""" return self.parse_pdf(name) - + def parse_additional_errors(self, bool): """PDF set used to generate the photon additional errors: they are constructed using the replicas 101-107 of the PDF set @@ -1061,15 +1060,17 @@ def parse_fakepdf(self, name): return self.parse_pdf(name) def _parse_lagrange_multiplier(self, kind, theoryid, setdict): - """ Lagrange multiplier constraints are mappings + """Lagrange multiplier constraints are mappings containing a `dataset` and a `maxlambda` argument which - defines the maximum value allowed for the multiplier """ + defines the maximum value allowed for the multiplier""" bad_msg = f"{kind} must be a mapping with a name ('dataset') and a float multiplier (maxlambda)" theoryno, _ = theoryid lambda_key = "maxlambda" - #BCH allow for old-style runcards with 'poslambda' instead of 'maxlambda' + # BCH allow for old-style runcards with 'poslambda' instead of 'maxlambda' if "poslambda" in setdict and "maxlambda" not in setdict: - log.warning("The `poslambda` argument has been deprecated in favour of `maxlambda`") + log.warning( + "The `poslambda` argument has been deprecated in favour of `maxlambda`" + ) lambda_key = "poslambda" try: name = setdict["dataset"] @@ -1185,21 +1186,20 @@ def produce_nnfit_theory_covmat( if inclusive_use_scalevar_uncertainties: if use_user_uncertainties: # Both scalevar and user uncertainties - from validphys.theorycovariance.construction import ( - total_theory_covmat_fitting, - ) + from validphys.theorycovariance.construction import \ + total_theory_covmat_fitting f = total_theory_covmat_fitting else: # Only scalevar uncertainties - from validphys.theorycovariance.construction import ( - theory_covmat_custom_fitting, - ) + from validphys.theorycovariance.construction import \ + theory_covmat_custom_fitting f = theory_covmat_custom_fitting elif use_user_uncertainties: # Only user uncertainties - from validphys.theorycovariance.construction import user_covmat_fitting + from validphys.theorycovariance.construction import \ + user_covmat_fitting f = user_covmat_fitting @@ -1280,7 +1280,7 @@ def parse_fitdeclaration(self, label: str): return label def produce_all_commondata(self): - """produces all commondata using the loader function """ + """produces all commondata using the loader function""" ds_names = self.loader.available_datasets ds_inputs = [self.parse_dataset_input({"dataset": ds}) for ds in ds_names] cd_out = [ @@ -1368,11 +1368,8 @@ def produce_rules( ): """Produce filter rules based on the user defined input and defaults.""" - from validphys.filters import ( - Rule, - RuleProcessingError, - default_filter_rules_input, - ) + from validphys.filters import (Rule, RuleProcessingError, + default_filter_rules_input) theory_parameters = theoryid.get_description() @@ -1484,7 +1481,10 @@ def produce_defaults( return filter_defaults def produce_data( - self, data_input, *, group_name="data", + self, + data_input, + *, + group_name="data", ): """A set of datasets where correlated systematics are taken into account @@ -1586,22 +1586,23 @@ def load_default_data_grouping(self, spec): """Load the default grouping of data""" # slightly superfluous, only one default at present but perhaps # somebody will want to add to this at some point e.g for th. uncertainties - allowed = { - "standard_report": "experiment", - "thcovmat_fit": "ALL" - } + allowed = {"standard_report": "experiment", "thcovmat_fit": "ALL"} return allowed[spec] def produce_processed_data_grouping( - self, use_thcovmat_in_fitting=False, use_thcovmat_in_sampling=False, data_grouping=None, data_grouping_recorded_spec_=None + self, + use_thcovmat_in_fitting=False, + use_thcovmat_in_sampling=False, + data_grouping=None, + data_grouping_recorded_spec_=None, ): """Process the data_grouping key from the runcard, or lockfile. If `data_grouping_recorded_spec_` is present then its value is taken, and the runcard is assumed to be a lockfile. If data_grouping is None, then, if either use_thcovmat_in_fitting or use_thcovmat_in_sampling - (or both) are true (which means that the fit is a thcovmat fit), group all the datasets - together, otherwise fall back to the default behaviour of grouping by + (or both) are true (which means that the fit is a thcovmat fit), group all the datasets + together, otherwise fall back to the default behaviour of grouping by experiment (called standard_report). Else, the user can specfiy their own grouping, for example metadata_process. @@ -1627,7 +1628,9 @@ def produce_processed_metadata_group( return metadata_group def produce_group_dataset_inputs_by_metadata( - self, data_input, processed_metadata_group, + self, + data_input, + processed_metadata_group, ): """Take the data and the processed_metadata_group key and attempt to group the data, returns a list where each element specifies the data_input @@ -1687,9 +1690,9 @@ def produce_group_dataset_inputs_by_process(self, data_input): def produce_scale_variation_theories(self, theoryid, point_prescription): """Produces a list of theoryids given a theoryid at central scales and a point - prescription. The options for the latter are '3 point', '5 point', '5bar point', '7 point' - and '9 point'. Note that these are defined in arXiv:1906.10698. This hard codes the - theories needed for each prescription to avoid user error.""" + prescription. The options for the latter are '3 point', '5 point', '5bar point', '7 point' + and '9 point'. Note that these are defined in arXiv:1906.10698. This hard codes the + theories needed for each prescription to avoid user error.""" pp = point_prescription th = theoryid.id diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index a1d6bf685d..9d837331e0 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -8,14 +8,12 @@ from importlib.resources import read_text import numpy as np - +import validphys.cuts from reportengine.checks import check, make_check from reportengine.compat import yaml -import validphys.cuts -from validphys.commondatawriter import ( - write_commondata_to_file, - write_systype_to_file, - ) +from validphys.commondatawriter import (write_commondata_to_file, + write_systype_to_file) + log = logging.getLogger(__name__) KIN_LABEL = { @@ -53,6 +51,7 @@ class BadPerturbativeOrder(ValueError): """Exception raised when the perturbative order string is not recognized.""" + class MissingRuleAttribute(RuleProcessingError, AttributeError): """Exception raised when a rule is missing required attributes.""" @@ -75,13 +74,14 @@ def default_filter_rules_input(): return yaml.safe_load(read_text(validphys.cuts, "filters.yaml")) - def check_nonnegative(var: str): """Ensure that `var` is positive""" + @make_check def run_check(ns, **kwargs): val = ns[var] check(val >= 0, f"'{var}' must be positive or equal zero, but it is {val!r}.") + return run_check @@ -95,7 +95,7 @@ def make_dataset_dir(path): def export_mask(path, mask): """Dump mask to file""" - np.savetxt(path, mask, fmt='%d') + np.savetxt(path, mask, fmt="%d") def filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed): @@ -105,7 +105,7 @@ def filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed): being shifted away from the underlying law. """ - log.info('Filtering closure-test data.') + log.info("Filtering closure-test data.") return _filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed) @@ -137,16 +137,17 @@ def filter_closure_data_by_experiment( def filter_real_data(filter_path, data): - """Filter real data, cutting any points which do not pass the filter rules. - """ - log.info('Filtering real data.') + """Filter real data, cutting any points which do not pass the filter rules.""" + log.info("Filtering real data.") return _filter_real_data(filter_path, data) def filter(filter_data): """Summarise filters applied to all datasets""" total_data, total_cut_data = np.atleast_2d(filter_data).sum(axis=0) - log.info(f'Summary: {total_cut_data}/{total_data} datapoints passed kinematic cuts.') + log.info( + f"Summary: {total_cut_data}/{total_data} datapoints passed kinematic cuts." + ) def _write_ds_cut_data(path, dataset): @@ -158,11 +159,13 @@ def _write_ds_cut_data(path, dataset): log.info("All {all_ndata} points in in {dataset.name} passed kinematic cuts.") else: filtered_dsndata = len(datamask) - log.info(f"{len(datamask)}/{all_dsndata} datapoints " - f"in {dataset.name} passed kinematic cuts.") + log.info( + f"{len(datamask)}/{all_dsndata} datapoints " + f"in {dataset.name} passed kinematic cuts." + ) # save to disk if datamask is not None: - export_mask(path / f'FKMASK_{dataset.name}.dat', datamask) + export_mask(path / f"FKMASK_{dataset.name}.dat", datamask) return all_dsndata, filtered_dsndata @@ -233,24 +236,24 @@ def _filter_closure_data( closure_data = level0_commondata_wc(data, fakepdf) for dataset in data.datasets: - #== print number of points passing cuts, make dataset directory and write FKMASK ==# + # == print number of points passing cuts, make dataset directory and write FKMASK ==# path = filter_path / dataset.name nfull, ncut = _write_ds_cut_data(path, dataset) make_dataset_dir(path / "systypes") total_data_points += nfull total_cut_data_points += ncut - + if fakenoise: - #======= Level 1 closure test =======# + # ======= Level 1 closure test =======# closure_data = make_level1_data( - data, - closure_data, - filterseed, - experiments_index, - ) + data, + closure_data, + filterseed, + experiments_index, + ) - #====== write commondata and systype files ======# + # ====== write commondata and systype files ======# if fakenoise: log.info("Writing Level1 data") else: @@ -259,10 +262,7 @@ def _filter_closure_data( for cd in closure_data: path_cd = filter_path / cd.setname / f"DATA_{cd.setname}.dat" path_sys = ( - filter_path - / cd.setname - / "systypes" - / f"SYSTYPE_{cd.setname}_DEFAULT.dat" + filter_path / cd.setname / "systypes" / f"SYSTYPE_{cd.setname}_DEFAULT.dat" ) write_commondata_to_file(commondata=cd, path=path_cd) write_systype_to_file(commondata=cd, path=path_sys) @@ -273,31 +273,36 @@ def _filter_closure_data( def check_t0pdfset(t0pdfset): """T0 pdf check""" t0pdfset.load() - log.info(f'{t0pdfset} T0 checked.') + log.info(f"{t0pdfset} T0 checked.") + def check_luxset(luxset): """Lux pdf check""" luxset.load() - log.info(f'{luxset} Lux pdf checked.') + log.info(f"{luxset} Lux pdf checked.") + def check_additional_errors(additional_errors): """Lux additional errors pdf check""" additional_errors.load() - log.info(f'{additional_errors} Lux additional errors pdf checked.') + log.info(f"{additional_errors} Lux additional errors pdf checked.") + def check_positivity(posdatasets): """Verify positive datasets are ready for the fit.""" - log.info('Verifying positivity tables:') + log.info("Verifying positivity tables:") for pos in posdatasets: pos.load_commondata() - log.info(f'{pos.name} checked.') + log.info(f"{pos.name} checked.") + def check_integrability(integdatasets): """Verify positive datasets are ready for the fit.""" - log.info('Verifying integrability tables:') + log.info("Verifying integrability tables:") for integ in integdatasets: integ.load_commondata() - log.info(f'{integ.name} checked.') + log.info(f"{integ.name} checked.") + class PerturbativeOrder: """Class that conveniently handles @@ -369,6 +374,7 @@ def __contains__(self, i): else: return i == self.numeric_pto + class Rule: """Rule object to be used to generate cuts mask. @@ -503,7 +509,12 @@ def _properties(self): """Attributes of the Rule class that are defining. Two Rules with identical ``_properties`` are considered equal. """ - return (self.rule_string, self.dataset, self.process_type, self.theory_params['ID']) + return ( + self.rule_string, + self.dataset, + self.process_type, + self.theory_params["ID"], + ) def __eq__(self, other): return self._properties == other._properties @@ -534,9 +545,7 @@ def __call__(self, dataset, idat): if k == "PTO" and hasattr(self, "PTO"): if v not in self.PTO: return None - elif hasattr(self, k) and ( - getattr(self, k) != v - ): + elif hasattr(self, k) and (getattr(self, k) != v): return None # Will return True if datapoint passes through the filter @@ -550,12 +559,12 @@ def __call__(self, dataset, idat): **ns, }, ) - except Exception as e: # pragma: no cover + except Exception as e: # pragma: no cover raise FatalRuleError( f"Error when applying rule {self.rule_string!r}: {e}" ) from e - def __repr__(self): # pragma: no cover + def __repr__(self): # pragma: no cover return self.rule_string def _make_kinematics_dict(self, dataset, idat) -> dict: @@ -572,6 +581,7 @@ def _make_point_namespace(self, dataset, idat) -> dict: ns[key] = eval(value, {**self.numpy_functions, **ns}) return ns + def get_cuts_for_dataset(commondata, rules) -> list: """Function to generate a list containing the index of all experimental points that passed kinematic diff --git a/validphys2/src/validphys/n3fit_data.py b/validphys2/src/validphys/n3fit_data.py index 2019aadc88..d0c2b6ec6f 100644 --- a/validphys2/src/validphys/n3fit_data.py +++ b/validphys2/src/validphys/n3fit_data.py @@ -5,20 +5,16 @@ :py:func:`n3fit.performfit.performfit`. """ import functools -from collections import defaultdict import hashlib import logging +from collections import defaultdict import numpy as np import pandas as pd - from reportengine import collect from reportengine.table import table - -from validphys.n3fit_data_utils import ( - validphys_group_extractor, -) from validphys.core import IntegrabilitySetSpec, TupleComp +from validphys.n3fit_data_utils import validphys_group_extractor log = logging.getLogger(__name__) @@ -52,6 +48,7 @@ def replica_mcseed(replica, mcseed, genrep): res = np.random.randint(0, pow(2, 31)) return res + def replica_luxseed(replica, luxseed): """Generate the ``luxseed`` for a ``replica``. Identical to replica_nnseed but used for a different purpose. @@ -97,7 +94,7 @@ def tr_masks(data, replica_trvlseed): ndata = len(cuts.load()) if cuts else dataset.commondata.ndata frac = dataset.frac # We do this so that a given dataset will always have the same number of points masked - trmax = int(ndata*frac) + trmax = int(ndata * frac) if trmax == 0: # If that number is 0, then get 1 point with probability frac trmax = int(rng.random() < frac) @@ -238,11 +235,13 @@ def fitting_data_dict( """ # TODO: Plug in the python data loading when available. Including but not # limited to: central values, ndata, replica generation, covmat construction - expdata_true = np.concatenate([d.central_values for d in dataset_inputs_loaded_cd_with_cuts]) + expdata_true = np.concatenate( + [d.central_values for d in dataset_inputs_loaded_cd_with_cuts] + ) expdata = make_replica tr_masks = tr_masks.masks - covmat = dataset_inputs_fitting_covmat # t0 covmat, or theory covmat or whatever was decided by the runcard + covmat = dataset_inputs_fitting_covmat # t0 covmat, or theory covmat or whatever was decided by the runcard inv_true = np.linalg.inv(covmat) fittable_datasets = fittable_datasets_masked @@ -319,7 +318,9 @@ def fitting_data_dict( return dict_out -exps_fitting_data_dict = collect("fitting_data_dict", ("group_dataset_inputs_by_metadata",)) +exps_fitting_data_dict = collect( + "fitting_data_dict", ("group_dataset_inputs_by_metadata",) +) def replica_nnseed_fitting_data_dict(replica, exps_fitting_data_dict, replica_nnseed): @@ -336,7 +337,9 @@ def replica_nnseed_fitting_data_dict(replica, exps_fitting_data_dict, replica_nn return (replica, exps_fitting_data_dict, replica_nnseed) -replicas_nnseed_fitting_data_dict = collect("replica_nnseed_fitting_data_dict", ("replicas",)) +replicas_nnseed_fitting_data_dict = collect( + "replica_nnseed_fitting_data_dict", ("replicas",) +) groups_replicas_indexed_make_replica = collect( "indexed_make_replica", ("group_dataset_inputs_by_experiment", "replicas") ) @@ -351,7 +354,7 @@ def pseudodata_table(groups_replicas_indexed_make_replica, replicas): Notes ----- Whilst running ``n3fit``, this action will only be called if - `fitting::savepseudodata` is `true` (as per the default setting) and + `fitting::savepseudodata` is `true` (as per the default setting) and replicas are fitted one at a time. The table can be found in the replica folder i.e. /nnfit/replica_*/ @@ -444,8 +447,12 @@ def replica_training_mask(exps_tr_masks, replica, experiments_index): [345 rows x 1 columns] """ - all_masks = np.concatenate([ds_mask for exp_masks in exps_tr_masks for ds_mask in exp_masks]) - return pd.DataFrame(all_masks, columns=[f"replica {replica}"], index=experiments_index) + all_masks = np.concatenate( + [ds_mask for exp_masks in exps_tr_masks for ds_mask in exp_masks] + ) + return pd.DataFrame( + all_masks, columns=[f"replica {replica}"], index=experiments_index + ) replicas_training_mask = collect("replica_training_mask", ("replicas",)) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 5fd5814d98..bca895386b 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -87,8 +87,10 @@ def __init__(self, theoryid, lux_params, replicas): for replica in replicas: f2 = sf.InterpStructureFunction(path_to_F2, self.luxpdfset.members[replica]) fl = sf.InterpStructureFunction(path_to_FL, self.luxpdfset.members[replica]) - if not np.isclose(f2.q2_max, fl.q2_max) : - log.error("FKtables for fiatlux_dis_F2 and fiatlux_dis_FL have two different q2_max") + if not np.isclose(f2.q2_max, fl.q2_max): + log.error( + "FKtables for fiatlux_dis_F2 and fiatlux_dis_FL have two different q2_max" + ) fiatlux_runcard["q2_max"] = float(f2.q2_max) f2lo = sf.F2LO(self.luxpdfset.members[replica], theory) From 09059e62ff2f2271de54a9c17604ec2ea646604e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 16:15:19 +0200 Subject: [PATCH 196/204] Revert "Call balck and isort to all changed files" This reverts commit fbfd77118d60306d4e0ea84b832088eafb8cb867. --- n3fit/src/n3fit/checks.py | 111 +++-------- n3fit/src/n3fit/layers/__init__.py | 11 +- n3fit/src/n3fit/layers/msr_normalization.py | 6 +- n3fit/src/n3fit/layers/rotations.py | 14 +- n3fit/src/n3fit/model_gen.py | 25 +-- n3fit/src/n3fit/model_trainer.py | 103 +++-------- n3fit/src/n3fit/msr.py | 54 +++--- n3fit/src/n3fit/performfit.py | 195 ++++++++++---------- n3fit/src/n3fit/scripts/n3fit_exec.py | 125 +++++-------- n3fit/src/n3fit/scripts/vp_setupfit.py | 139 +++++++------- n3fit/src/n3fit/vpinterface.py | 15 +- validphys2/src/validphys/config.py | 183 +++++++++--------- validphys2/src/validphys/filters.py | 94 +++++----- validphys2/src/validphys/n3fit_data.py | 35 ++-- validphys2/src/validphys/photon/compute.py | 6 +- 15 files changed, 461 insertions(+), 655 deletions(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 73e912b693..4573926822 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -1,15 +1,13 @@ """ This module contains checks to be perform by n3fit on the input """ +import os import logging import numbers -import os - import numpy as np -from reportengine.checks import CheckError, make_argcheck +from reportengine.checks import make_argcheck, CheckError from validphys.core import PDF from validphys.pdfbases import check_basis - from n3fit.hyper_optimization import penalties as penalties_module from n3fit.hyper_optimization import rewards as rewards_module @@ -55,9 +53,7 @@ def check_consistent_layers(parameters): npl = len(parameters["nodes_per_layer"]) apl = len(parameters["activation_per_layer"]) if npl != apl: - raise CheckError( - f"Number of layers ({npl}) does not match activation functions: {apl}" - ) + raise CheckError(f"Number of layers ({npl}) does not match activation functions: {apl}") def check_stopping(parameters): @@ -135,16 +131,12 @@ def check_lagrange_multipliers(parameters, key): return multiplier = lagrange_dict.get("multiplier") if multiplier is not None and multiplier < 0: - log.warning( - "The %s multiplier is below 0, it will produce a negative loss", key - ) + log.warning("The %s multiplier is below 0, it will produce a negative loss", key) elif multiplier is not None and multiplier == 0: log.warning("The %s multiplier is 0 so it won't contribute to the loss", key) threshold = lagrange_dict.get("threshold") if threshold is not None and not _is_floatable(threshold): - raise CheckError( - f"The {key}::threshold must be a number, received: {threshold}" - ) + raise CheckError(f"The {key}::threshold must be a number, received: {threshold}") def check_model_file(save, load): @@ -158,13 +150,9 @@ def check_model_file(save, load): if not isinstance(load, str): raise CheckError(f"Model file to load: {load} not understood, str expected") if not os.path.isfile(load): - raise CheckError( - f"Model file to load: {load} can not be opened, does it exist?" - ) + raise CheckError(f"Model file to load: {load} can not be opened, does it exist?") if not os.access(load, os.R_OK): - raise CheckError( - f"Model file to load: {load} cannot be read, permission denied" - ) + raise CheckError(f"Model file to load: {load} cannot be read, permission denied") if os.stat(load).st_size == 0: raise CheckError(f"Model file {load} seems to be empty") @@ -205,9 +193,7 @@ def check_hyperopt_architecture(architecture): min_u = architecture.get("min_units", 1) # Set a minimum number of units in case none is defined to check later if the maximum is sane if min_u <= 0: - raise CheckError( - f"All layers must have at least 1 unit, got min_units: {min_u}" - ) + raise CheckError(f"All layers must have at least 1 unit, got min_units: {min_u}") max_u = architecture.get("max_units") if max_u is not None and max_u < min_u: raise CheckError( @@ -224,13 +210,9 @@ def check_hyperopt_positivity(positivity_dict): max_mul = positivity_dict.get("max_multiplier") if max_mul is not None or min_mul is not None: if max_mul is None: - raise CheckError( - "Need to set a maximum positivity multiplier if the minimum is set" - ) + raise CheckError("Need to set a maximum positivity multiplier if the minimum is set") if min_mul is not None and max_mul <= min_mul: - raise CheckError( - "The minimum multiplier cannot be greater than the maximum" - ) + raise CheckError("The minimum multiplier cannot be greater than the maximum") min_ini = positivity_dict.get("min_initial") max_ini = positivity_dict.get("max_initial") if max_ini is not None or min_ini is not None: @@ -239,9 +221,7 @@ def check_hyperopt_positivity(positivity_dict): "Need to set both the max_initial and the min_initial positivity values" ) if max_ini <= min_ini: - raise CheckError( - "The minimum initial value cannot be greater than the maximum" - ) + raise CheckError("The minimum initial value cannot be greater than the maximum") def check_kfold_options(kfold): @@ -266,13 +246,9 @@ def check_kfold_options(kfold): # Check specific errors for specific targets if loss_target == "fit_future_tests": if len(partitions) == 1: - raise CheckError( - "Cannot use target 'fit_future_tests' with just one partition" - ) + raise CheckError("Cannot use target 'fit_future_tests' with just one partition") if partitions[-1]["datasets"]: - log.warning( - "Last partition in future test is not empty, some datasets will be ignored" - ) + log.warning("Last partition in future test is not empty, some datasets will be ignored") def check_correct_partitions(kfold, data): @@ -284,9 +260,7 @@ def check_correct_partitions(kfold, data): fold_sets = partition["datasets"] for dset in fold_sets: if dset not in datasets: - raise CheckError( - f"The k-fold defined dataset {dset} is not part of the fit" - ) + raise CheckError(f"The k-fold defined dataset {dset} is not part of the fit") def check_hyperopt_stopping(stopping_dict): @@ -299,20 +273,14 @@ def check_hyperopt_stopping(stopping_dict): if min_ep is None or max_ep is None: raise CheckError("Need to set both the max_epochs and the min_epochs") if min_ep < 1: - raise CheckError( - f"Can't run for less than 1 epoch: selected min_ep = {min_ep}" - ) + raise CheckError(f"Can't run for less than 1 epoch: selected min_ep = {min_ep}") if max_ep <= min_ep: - raise CheckError( - f"min_epochs cannot be greater than max_epochs: ({min_ep} > {max_ep})" - ) + raise CheckError(f"min_epochs cannot be greater than max_epochs: ({min_ep} > {max_ep})") min_pat = stopping_dict.get("min_patience") max_pat = stopping_dict.get("max_patience") if min_pat is not None or max_pat is not None: if min_pat is not None and min_pat < 0.0: - raise CheckError( - f"min_patience cannot be less than 0.0: selected min_pat = {min_pat}" - ) + raise CheckError(f"min_patience cannot be less than 0.0: selected min_pat = {min_pat}") if max_pat is not None: if max_pat > 1.0: raise CheckError( @@ -332,9 +300,7 @@ def wrapper_hyperopt(hyperopt, hyperscan_config, kfold, data): if not hyperopt: return if hyperscan_config is None: - raise CheckError( - "Can't perform hyperoptimization without the hyperscan_config key" - ) + raise CheckError("Can't perform hyperoptimization without the hyperscan_config key") if kfold is None: raise CheckError("Can't perform hyperoptimization without folds") check_hyperopt_stopping(hyperscan_config.get("stopping")) @@ -351,9 +317,7 @@ def check_sumrules(sum_rules): accepted_options = ["ALL", "MSR", "VSR"] if sum_rules.upper() in accepted_options: return - raise CheckError( - f"The only accepted options for the sum rules are: {accepted_options}" - ) + raise CheckError(f"The only accepted options for the sum rules are: {accepted_options}") # Checks on the physics @@ -382,17 +346,11 @@ def check_consistent_basis(sum_rules, fitbasis, basis, theoryid): # Check that the basis given in the runcard is one of those defined in validphys.pdfbases basis = check_basis(fitbasis, flavs)["basis"] # Now check that basis and theory id are consistent - has_c = ( - basis.has_element("c") or basis.has_element("T15") or basis.has_element("cp") - ) + has_c = basis.has_element("c") or basis.has_element("T15") or basis.has_element("cp") if theoryid.get_description()["IC"] and not has_c: - raise CheckError( - f"{theoryid} (intrinsic charm) is incompatible with basis {fitbasis}" - ) + raise CheckError(f"{theoryid} (intrinsic charm) is incompatible with basis {fitbasis}") if not theoryid.get_description()["IC"] and has_c: - raise CheckError( - f"{theoryid} (perturbative charm) is incompatible with basis {fitbasis}" - ) + raise CheckError(f"{theoryid} (perturbative charm) is incompatible with basis {fitbasis}") @make_argcheck @@ -408,9 +366,7 @@ def check_consistent_parallel(parameters, parallel_models, same_trvl_per_replica " masks, please set `same_trvl_per_replica` to True in the runcard" ) if parameters.get("layer_type") != "dense": - raise CheckError( - "Parallelization has only been tested with layer_type=='dense'" - ) + raise CheckError("Parallelization has only been tested with layer_type=='dense'") @make_argcheck @@ -426,37 +382,24 @@ def can_run_multiple_replicas(replicas, parallel_models): @make_argcheck def check_deprecated_options(fitting): """Checks whether the runcard is using deprecated options""" - options_outside = [ - "trvlseed", - "nnseed", - "mcseed", - "save", - "load", - "genrep", - "parameters", - ] + options_outside = ["trvlseed", "nnseed", "mcseed", "save", "load", "genrep", "parameters"] for option in options_outside: if option in fitting: raise CheckError( f"The key '{option}' should be top-level key and not part of the 'fitting' namespace" ) if "epochs" in fitting: - raise CheckError( - "The key 'epoch' should only appear as part of the 'parameters' namespace" - ) + raise CheckError("The key 'epoch' should only appear as part of the 'parameters' namespace") nnfit_options = ["seed", "rnalgo", "fitmethod", "nmutants", "paramtype", "nnodes"] for option in nnfit_options: if option in fitting: - log.warning( - "'fitting::%s' is an nnfit-only key, it will be ignored", option - ) - + log.warning("'fitting::%s' is an nnfit-only key, it will be ignored", option) @make_argcheck def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): if fiatlux is not None: luxset = fiatlux["luxset"] - pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 + pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 max_id = max(replicas) if max_id > pdfs_ids: raise CheckError( diff --git a/n3fit/src/n3fit/layers/__init__.py b/n3fit/src/n3fit/layers/__init__.py index a9168bda13..d7d288bf9b 100644 --- a/n3fit/src/n3fit/layers/__init__.py +++ b/n3fit/src/n3fit/layers/__init__.py @@ -1,9 +1,8 @@ -from n3fit.backends import MetaLayer - +from .preprocessing import Preprocessing +from .rotations import FkRotation, FlavourToEvolution, ObsRotation, AddPhoton +from .x_operations import xIntegrator, xDivide +from .msr_normalization import MSR_Normalization from .DIS import DIS from .DY import DY from .mask import Mask -from .msr_normalization import MSR_Normalization -from .preprocessing import Preprocessing -from .rotations import AddPhoton, FkRotation, FlavourToEvolution, ObsRotation -from .x_operations import xDivide, xIntegrator +from n3fit.backends import MetaLayer diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 8306853b19..991d92718f 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -56,12 +56,10 @@ def call(self, pdf_integrated, ph_replica): if self._photons: photon_integral = self._photons[ph_replica] else: - photon_integral = 0.0 + photon_integral = 0. if self._msr_enabled: - n_ag = [ - (1.0 - y[GLUON_IDX[0][0] - 1] - photon_integral) / y[GLUON_IDX[0][0]] - ] * len(GLUON_IDX) + n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - photon_integral) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index a543462d6b..6db811ec11 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -2,10 +2,9 @@ This module includes rotation layers """ import numpy as np -from validphys import pdfbases - from n3fit.backends import MetaLayer from n3fit.backends import operations as op +from validphys import pdfbases class Rotation(MetaLayer): @@ -29,7 +28,7 @@ def __init__(self, rotation_matrix, axes=1, **kwargs): super().__init__(**kwargs) def is_identity(self): - """Returns true if the rotation is an identity""" + """ Returns true if the rotation is an identity """ # check whether it is a mxm matrix if self.rotation_matrix.shape[0] == self.rotation_matrix.shape[1]: # check whether it is the identity @@ -42,15 +41,12 @@ def call(self, x_raw): class FlavourToEvolution(Rotation): """ - Rotates from the flavour basis to - the evolution basis. + Rotates from the flavour basis to + the evolution basis. """ def __init__( - self, - flav_info, - fitbasis, - **kwargs, + self, flav_info, fitbasis, **kwargs, ): rotation_matrix = pdfbases.fitbasis_to_NN31IC(flav_info, fitbasis) super().__init__(rotation_matrix, axes=1, **kwargs) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 55a08debee..d360b43ea3 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -10,17 +10,16 @@ """ from dataclasses import dataclass - import numpy as np +from n3fit.msr import msr_impose +from n3fit.layers import DIS, DY, ObsRotation, losses +from n3fit.layers import Preprocessing, FkRotation, FlavourToEvolution, AddPhoton +from n3fit.layers.observable import is_unique -from n3fit.backends import (Input, Lambda, MetaLayer, MetaModel, - base_layer_selector) +from n3fit.backends import MetaModel, Input from n3fit.backends import operations as op -from n3fit.backends import regularizer_selector -from n3fit.layers import (DIS, DY, AddPhoton, FkRotation, FlavourToEvolution, - ObsRotation, Preprocessing, losses) -from n3fit.layers.observable import is_unique -from n3fit.msr import msr_impose +from n3fit.backends import MetaLayer, Lambda +from n3fit.backends import base_layer_selector, regularizer_selector @dataclass @@ -352,9 +351,7 @@ def generate_dense_per_flavour_network( initializers = [] for _ in range(basis_size): # select the initializer and move the seed - initializers.append( - MetaLayer.select_initializer(initializer_name, seed=current_seed) - ) + initializers.append(MetaLayer.select_initializer(initializer_name, seed=current_seed)) current_seed += 1 # set the arguments that will define the layer @@ -499,7 +496,7 @@ def pdfNN_layer_generator( will be a (1, None, 2) tensor where dim [:,:,0] is scaled parallel_models: int How many models should be trained in parallel - photon: :py:class:`validphys.photon.compute.Photon` + photon: :py:class:`validphys.photon.compute.Photon` If given, gives the AddPhoton layer a function to compute a photon which will be added at the index 0 of the 14-size FK basis This same function will also be used to compute the MSR component for the photon @@ -576,9 +573,7 @@ def pdfNN_layer_generator( # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose( - mode=impose_sumrule, scaler=scaler, photons=photons - ) + sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photons=photons) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 3f62f9b131..fb165aa111 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -11,18 +11,16 @@ import logging from collections import namedtuple from itertools import zip_longest - import numpy as np from scipy.interpolate import PchipInterpolator -from validphys.photon.compute import Photon - -import n3fit.hyper_optimization.penalties -import n3fit.hyper_optimization.rewards from n3fit import model_gen -from n3fit.backends import MetaModel, callbacks, clear_backend_state +from n3fit.backends import MetaModel, clear_backend_state, callbacks from n3fit.backends import operations as op from n3fit.stopping import Stopping from n3fit.vpinterface import N3PDF +import n3fit.hyper_optimization.penalties +import n3fit.hyper_optimization.rewards +from validphys.photon.compute import Photon log = logging.getLogger(__name__) @@ -39,7 +37,6 @@ # See ModelTrainer::_xgrid_generation for the definition of each field and how they are generated InputInfo = namedtuple("InputInfo", ["input", "split", "idx"]) - def _pdf_injection(pdf_layers, observables, masks): """ Takes as input a list of PDF layers each corresponding to one observable (also given as a list) @@ -195,9 +192,7 @@ def __init__( hyper_loss = kfold_parameters.get("target", None) if hyper_loss is None: hyper_loss = "average" - log.warning( - "No minimization target selected, defaulting to '%s'", hyper_loss - ) + log.warning("No minimization target selected, defaulting to '%s'", hyper_loss) log.info("Using '%s' as the target for hyperoptimization", hyper_loss) self._hyper_loss = getattr(n3fit.hyper_optimization.rewards, hyper_loss) @@ -473,9 +468,7 @@ def _model_generation(self, xinput, pdf_models, partition, partition_idx): validation = MetaModel(full_model_input_dict, output_vl) # Or the positivity in the total chi2 - output_ex = _pdf_injection( - exp_pdfs, self.experimental["output"], experimental_mask - ) + output_ex = _pdf_injection(exp_pdfs, self.experimental["output"], experimental_mask) experimental = MetaModel(full_model_input_dict, output_ex) if self.print_summary: @@ -571,9 +564,7 @@ def _generate_observables( all_pos_initial, all_pos_multiplier, max_lambda, positivity_steps ) - pos_layer = model_gen.observable_generator( - pos_dict, positivity_initial=pos_initial - ) + pos_layer = model_gen.observable_generator(pos_dict, positivity_initial=pos_initial) # The input list is still common self.input_list.append(pos_layer["inputs"]) @@ -588,18 +579,13 @@ def _generate_observables( if self.integ_info is not None: for integ_dict in self.integ_info: if not self.mode_hyperopt: - log.info( - "Generating integrability penalty for %s", integ_dict["name"] - ) + log.info("Generating integrability penalty for %s", integ_dict["name"]) integrability_steps = int(epochs / PUSH_INTEGRABILITY_EACH) max_lambda = integ_dict["lambda"] integ_initial, integ_multiplier = _LM_initial_and_multiplier( - all_integ_initial, - all_integ_multiplier, - max_lambda, - integrability_steps, + all_integ_initial, all_integ_multiplier, max_lambda, integrability_steps ) integ_layer = model_gen.observable_generator( @@ -625,15 +611,10 @@ def _generate_observables( force_set_smallest = input_arr.min() > 1e-9 if force_set_smallest: new_xgrid = np.linspace( - start=1 / input_arr_size, - stop=1.0, - endpoint=False, - num=input_arr_size, + start=1 / input_arr_size, stop=1.0, endpoint=False, num=input_arr_size ) else: - new_xgrid = np.linspace( - start=0, stop=1.0, endpoint=False, num=input_arr_size - ) + new_xgrid = np.linspace(start=0, stop=1.0, endpoint=False, num=input_arr_size) # When mapping the FK xgrids onto our new grid, we need to consider degeneracies among # the x-values in the FK grids @@ -653,9 +634,7 @@ def _generate_observables( # Select the indices of the points that will be used by the interpolator onein = map_from_complete.size / (int(interpolation_points) - 1) - selected_points = [ - round(i * onein - 1) for i in range(1, int(interpolation_points)) - ] + selected_points = [round(i * onein - 1) for i in range(1, int(interpolation_points))] if selected_points[0] != 0: selected_points = [0] + selected_points map_from = map_from_complete[selected_points] @@ -666,8 +645,7 @@ def _generate_observables( scaler = PchipInterpolator(map_from, map_to) except ValueError: raise ValueError( - "interpolation_points is larger than the number of unique " - "input x-values" + "interpolation_points is larger than the number of unique " "input x-values" ) self._scaler = lambda x: np.concatenate([scaler(np.log(x)), x], axis=-1) @@ -708,7 +686,7 @@ def _generate_pdf( dictionary of arguments for the regularizer seed: int seed for the NN - photons: :py:class:`validphys.photon.compute.Photon` + photons: :py:class:`validphys.photon.compute.Photon` function to compute the photon PDF see model_gen.pdfNN_layer_generator for more information @@ -744,14 +722,7 @@ def _prepare_reporting(self, partition): to select the bits necessary for reporting the chi2. Receives the chi2 partition data to see whether any dataset is to be left out """ - reported_keys = [ - "name", - "count_chi2", - "positivity", - "integrability", - "ndata", - "ndata_vl", - ] + reported_keys = ["name", "count_chi2", "positivity", "integrability", "ndata", "ndata_vl"] reporting_list = [] for exp_dict in self.all_info: reporting_dict = {k: exp_dict.get(k) for k in reported_keys} @@ -852,10 +823,7 @@ def evaluate(self, stopping_object): # training and the validation which are actually `chi2` and not part of the penalty train_chi2 = stopping_object.evaluate_training(self.training["model"]) val_chi2 = stopping_object.vl_chi2 - exp_chi2 = ( - self.experimental["model"].compute_losses()["loss"] - / self.experimental["ndata"] - ) + exp_chi2 = self.experimental["model"].compute_losses()["loss"] / self.experimental["ndata"] return train_chi2, val_chi2, exp_chi2 def hyperparametrizable(self, params): @@ -918,13 +886,13 @@ def hyperparametrizable(self, params): xinput = self._xgrid_generation() # Initialize all photon classes for the different replicas: if self.lux_params: - photons = Photon( + photons=Photon( theoryid=self.theoryid, lux_params=self.lux_params, replicas=self.replicas, ) else: - photons = None + photons=None ### Training loop for k, partition in enumerate(self.kpartitions): # Each partition of the kfolding needs to have its own separate model @@ -951,6 +919,7 @@ def hyperparametrizable(self, params): pl = m.get_layer("add_photon") pl.register_photon(xinput.input.tensor_content) + # Model generation joins all the different observable layers # together with pdf model generated above models = self._model_generation(xinput, pdf_models, partition, k) @@ -963,12 +932,8 @@ def hyperparametrizable(self, params): if k > 0: # Reset the positivity and integrability multipliers - pos_and_int = ( - self.training["posdatasets"] + self.training["integdatasets"] - ) - initial_values = ( - self.training["posinitials"] + self.training["posinitials"] - ) + pos_and_int = self.training["posdatasets"] + self.training["integdatasets"] + initial_values = self.training["posinitials"] + self.training["posinitials"] models["training"].reset_layer_weights_to(pos_and_int, initial_values) # Generate the list containing reporting info necessary for chi2 @@ -1009,14 +974,10 @@ def hyperparametrizable(self, params): validation_loss = np.mean(stopping_object.vl_chi2) # Compute experimental loss - exp_loss_raw = np.average( - models["experimental"].compute_losses()["loss"] - ) + exp_loss_raw = np.average(models["experimental"].compute_losses()["loss"]) # And divide by the number of active points in this fold # it would be nice to have a ndata_per_fold variable coming in the vp object... - ndata = np.sum( - [np.count_nonzero(i[k]) for i in self.experimental["folds"]] - ) + ndata = np.sum([np.count_nonzero(i[k]) for i in self.experimental["folds"]]) # If ndata == 0 then it's the opposite, all data is in! if ndata == 0: ndata = self.experimental["ndata"] @@ -1024,18 +985,12 @@ def hyperparametrizable(self, params): hyper_loss = experimental_loss if passed != self.pass_status: - log.info( - "Hyperparameter combination fail to find a good fit, breaking" - ) + log.info("Hyperparameter combination fail to find a good fit, breaking") # If the fit failed to fit, no need to add a penalty to the loss break for penalty in self.hyper_penalties: - hyper_loss += penalty( - pdf_models=pdf_models, stopping_object=stopping_object - ) - log.info( - "Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed - ) + hyper_loss += penalty(pdf_models=pdf_models, stopping_object=stopping_object) + log.info("Fold %d finished, loss=%.1f, pass=%s", k + 1, hyper_loss, passed) # Now save all information from this fold l_hyper.append(hyper_loss) @@ -1084,9 +1039,5 @@ def hyperparametrizable(self, params): # In a normal run, the only information we need to output is the stopping object # (which contains metadata about the stopping) # and the pdf models (which are used to generate the PDF grids and compute arclengths) - dict_out = { - "status": passed, - "stopping_object": stopping_object, - "pdf_models": pdf_models, - } + dict_out = {"status": passed, "stopping_object": stopping_object, "pdf_models": pdf_models} return dict_out diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index 55f74a3dab..b43ec40bfb 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -2,11 +2,11 @@ The constraint module include functions to impose the momentum sum rules on the PDFs """ import logging - import numpy as np +from n3fit.layers import xDivide, MSR_Normalization, xIntegrator from n3fit.backends import operations as op -from n3fit.layers import MSR_Normalization, xDivide, xIntegrator + log = logging.getLogger(__name__) @@ -36,28 +36,28 @@ def gen_integration_input(nx): return xgrid, weights_array -def msr_impose(nx=int(2e3), mode="All", scaler=None, photons=None): +def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): """ - This function receives: - Generates a function that applies a normalization layer to the fit. - - fit_layer: the 8-basis layer of PDF which we fit - The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) - - final_layer: the 14-basis which is fed to the fktable - and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). - It uses pdf_fit to compute the sum rule and returns a modified version of - the final_pdf layer with a normalisation by which the sum rule is imposed - - Parameters - ---------- - nx: int - number of points for the integration grid, default: 2000 - mode: str - what sum rules to compute (MSR, VSR or All), default: All - scaler: scaler - Function to apply to the input. If given the input to the model - will be a (1, None, 2) tensor where dim [:,:,0] is scaled - photon: :py:class:`validphys.photon.compute.Photon` - If given, gives the AddPhoton layer a function to compute the MSR component for the photon + This function receives: + Generates a function that applies a normalization layer to the fit. + - fit_layer: the 8-basis layer of PDF which we fit + The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) + - final_layer: the 14-basis which is fed to the fktable + and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). + It uses pdf_fit to compute the sum rule and returns a modified version of + the final_pdf layer with a normalisation by which the sum rule is imposed + + Parameters + ---------- + nx: int + number of points for the integration grid, default: 2000 + mode: str + what sum rules to compute (MSR, VSR or All), default: All + scaler: scaler + Function to apply to the input. If given the input to the model + will be a (1, None, 2) tensor where dim [:,:,0] is scaled + photon: :py:class:`validphys.photon.compute.Photon` + If given, gives the AddPhoton layer a function to compute the MSR component for the photon """ # 1. Generate the fake input which will be used to integrate @@ -86,16 +86,14 @@ def msr_impose(nx=int(2e3), mode="All", scaler=None, photons=None): # and will return it appropiately normalized. def apply_normalization(layer_pdf, ph_replica): """ - layer_pdf: output of the PDF, unnormalized, ready for the fktable + layer_pdf: output of the PDF, unnormalized, ready for the fktable """ x_original = op.op_gather_keep_dims(xgrid_input, -1, axis=-1) - pdf_integrand = op.op_multiply( - [division_by_x(x_original), layer_pdf(xgrid_input)] - ) + pdf_integrand = op.op_multiply([division_by_x(x_original), layer_pdf(xgrid_input)]) normalization = normalizer(integrator(pdf_integrand), ph_replica) def ultimate_pdf(x): - return layer_pdf(x) * normalization + return layer_pdf(x)*normalization return ultimate_pdf diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index c2334410d9..03c36aa1b1 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -5,9 +5,7 @@ # Backend-independent imports import copy import logging - import numpy as np - import n3fit.checks from n3fit.vpinterface import N3PDF @@ -20,8 +18,8 @@ @n3fit.checks.check_fiatlux_pdfs_id def performfit( *, - n3fit_checks_action, # wrapper for all checks - replicas, # checks specific to performfit + n3fit_checks_action, # wrapper for all checks + replicas, # checks specific to performfit replicas_nnseed_fitting_data_dict, posdatasets_fitting_pos_dict, integdatasets_fitting_integ_dict, @@ -44,86 +42,86 @@ def performfit( parallel_models=False, ): """ - This action will (upon having read a validcard) process a full PDF fit - for a set of replicas. - - The input to this function is provided by validphys - and/or defined in the runcards or commandline arguments. - - This controller is provided with: - 1. Seeds generated using the replica number and the seeds defined in the runcard. - 2. Loaded datasets with replicas generated. - 2.1 Loaded positivity/integrability sets. - - The workflow of this controller is as follows: - 1. Generate a ModelTrainer object holding information to create the NN and perform a fit - (at this point no NN object has been generated) - 1.1 (if hyperopt) generates the hyperopt scanning dictionary - taking as a base the fitting dictionary and the runcard's hyperscanner dictionary - 2. Pass the dictionary of parameters to ModelTrainer - for the NN to be generated and the fit performed - 2.1 (if hyperopt) Loop over point 4 for `hyperopt` number of times - 3. Once the fit is finished, output the PDF grid and accompanying files - - Parameters - ---------- - genrep: bool - Whether or not to generate MC replicas. (Only used for checks) - data: validphys.core.DataGroupSpec - containing the datasets to be included in the fit. (Only used - for checks) - replicas_nnseed_fitting_data_dict: list[tuple] - list with element for each replica (typically just one) to be - fitted. Each element - is a tuple containing the replica number, nnseed and - ``fitted_data_dict`` containing all of the data, metadata - for each group of datasets which is to be fitted. - posdatasets_fitting_pos_dict: list[dict] - list of dictionaries containing all data and metadata for each - positivity dataset - integdatasets_fitting_integ_dict: list[dict] - list of dictionaries containing all data and metadata for each - integrability dataset - theoryid: validphys.core.TheoryIDSpec - Theory which is used to generate theory predictions from model - during fit. Object also contains some metadata on the theory - settings. - fiatlux: dict - dictionary containing the params needed from LuxQED - basis: list[dict] - preprocessing information for each flavour to be fitted. - fitbasis: str - Valid basis which the fit is to be ran in. Available bases can - be found in :py:mod:`validphys.pdfbases`. - sum_rules: bool - Whether to impose sum rules in fit. By default set to True - parameters: dict - Mapping containing parameters which define the network - architecture/fitting methodology. - replica_path: pathlib.Path - path to the output of this run - output_path: str - name of the fit - save: None, str - model file where weights will be saved, used in conjunction with - ``load``. - load: None, str - model file from which to load weights from. - hyperscanner: dict - dictionary containing the details of the hyperscanner - hyperopt: int - if given, number of hyperopt iterations to run - kfold_parameters: None, dict - dictionary with kfold settings used in hyperopt. - tensorboard: None, dict - mapping containing tensorboard settings if it is to be used. By - default it is None and tensorboard is not enabled. - debug: bool - activate some debug options - maxcores: int - maximum number of (logical) cores that the backend should be aware of - parallel_models: bool - whether to run models in parallel + This action will (upon having read a validcard) process a full PDF fit + for a set of replicas. + + The input to this function is provided by validphys + and/or defined in the runcards or commandline arguments. + + This controller is provided with: + 1. Seeds generated using the replica number and the seeds defined in the runcard. + 2. Loaded datasets with replicas generated. + 2.1 Loaded positivity/integrability sets. + + The workflow of this controller is as follows: + 1. Generate a ModelTrainer object holding information to create the NN and perform a fit + (at this point no NN object has been generated) + 1.1 (if hyperopt) generates the hyperopt scanning dictionary + taking as a base the fitting dictionary and the runcard's hyperscanner dictionary + 2. Pass the dictionary of parameters to ModelTrainer + for the NN to be generated and the fit performed + 2.1 (if hyperopt) Loop over point 4 for `hyperopt` number of times + 3. Once the fit is finished, output the PDF grid and accompanying files + + Parameters + ---------- + genrep: bool + Whether or not to generate MC replicas. (Only used for checks) + data: validphys.core.DataGroupSpec + containing the datasets to be included in the fit. (Only used + for checks) + replicas_nnseed_fitting_data_dict: list[tuple] + list with element for each replica (typically just one) to be + fitted. Each element + is a tuple containing the replica number, nnseed and + ``fitted_data_dict`` containing all of the data, metadata + for each group of datasets which is to be fitted. + posdatasets_fitting_pos_dict: list[dict] + list of dictionaries containing all data and metadata for each + positivity dataset + integdatasets_fitting_integ_dict: list[dict] + list of dictionaries containing all data and metadata for each + integrability dataset + theoryid: validphys.core.TheoryIDSpec + Theory which is used to generate theory predictions from model + during fit. Object also contains some metadata on the theory + settings. + fiatlux: dict + dictionary containing the params needed from LuxQED + basis: list[dict] + preprocessing information for each flavour to be fitted. + fitbasis: str + Valid basis which the fit is to be ran in. Available bases can + be found in :py:mod:`validphys.pdfbases`. + sum_rules: bool + Whether to impose sum rules in fit. By default set to True + parameters: dict + Mapping containing parameters which define the network + architecture/fitting methodology. + replica_path: pathlib.Path + path to the output of this run + output_path: str + name of the fit + save: None, str + model file where weights will be saved, used in conjunction with + ``load``. + load: None, str + model file from which to load weights from. + hyperscanner: dict + dictionary containing the details of the hyperscanner + hyperopt: int + if given, number of hyperopt iterations to run + kfold_parameters: None, dict + dictionary with kfold settings used in hyperopt. + tensorboard: None, dict + mapping containing tensorboard settings if it is to be used. By + default it is None and tensorboard is not enabled. + debug: bool + activate some debug options + maxcores: int + maximum number of (logical) cores that the backend should be aware of + parallel_models: bool + whether to run models in parallel """ from n3fit.backends import set_initial_state @@ -136,8 +134,8 @@ def performfit( # All potentially backend dependent imports should come inside the fit function # so they can eventually be set from the runcard - from n3fit.io.writer import WriterWrapper from n3fit.model_trainer import ModelTrainer + from n3fit.io.writer import WriterWrapper # Note: there are three possible scenarios for the loop of replicas: # 1.- Only one replica is being run, in this case the loop is only evaluated once @@ -164,12 +162,10 @@ def performfit( training_data = [] validation_data = [] for i_rep in range(n_models): - training_data.append(replica_experiments[i_rep][i_exp]["expdata"]) - validation_data.append(replica_experiments[i_rep][i_exp]["expdata_vl"]) - all_experiments[i_exp]["expdata"] = np.concatenate(training_data, axis=0) - all_experiments[i_exp]["expdata_vl"] = np.concatenate( - validation_data, axis=0 - ) + training_data.append(replica_experiments[i_rep][i_exp]['expdata']) + validation_data.append(replica_experiments[i_rep][i_exp]['expdata_vl']) + all_experiments[i_exp]['expdata'] = np.concatenate(training_data, axis=0) + all_experiments[i_exp]['expdata_vl'] = np.concatenate(validation_data, axis=0) log.info( "Starting parallel fits from replica %d to %d", replicas[0], @@ -266,9 +262,7 @@ def performfit( log.info("Stopped at epoch=%d", stopping_object.stop_epoch) final_time = stopwatch.stop() - all_training_chi2, all_val_chi2, all_exp_chi2 = the_model_trainer.evaluate( - stopping_object - ) + all_training_chi2, all_val_chi2, all_exp_chi2 = the_model_trainer.evaluate(stopping_object) pdf_models = result["pdf_models"] for i, (replica_number, pdf_model) in enumerate(zip(replica_idxs, pdf_models)): @@ -298,12 +292,13 @@ def performfit( replica_path_set, output_path.name, training_chi2, val_chi2, exp_chi2 ) log.info( - "Best fit for replica #%d, chi2=%.3f (tr=%.3f, vl=%.3f)", - replica_number, - exp_chi2, - training_chi2, - val_chi2, - ) + "Best fit for replica #%d, chi2=%.3f (tr=%.3f, vl=%.3f)", + replica_number, + exp_chi2, + training_chi2, + val_chi2 + ) + # Save the weights to some file for the given replica if save: diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 71f4aada82..9b80cb5308 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -3,23 +3,28 @@ n3fit - performs fit using ml external frameworks """ -import argparse -import logging -import pathlib +import sys import re import shutil -import sys +import pathlib +import logging import warnings +import argparse +from validphys.app import App +from validphys.config import Environment, Config +from validphys.config import EnvironmentError_, ConfigError +from validphys.core import FitSpec from reportengine import colors from reportengine.compat import yaml from reportengine.namespaces import NSList -from validphys.app import App -from validphys.config import (Config, ConfigError, Environment, - EnvironmentError_) -from validphys.core import FitSpec -N3FIT_FIXED_CONFIG = dict(use_cuts="internal", use_t0=True, actions_=[]) + +N3FIT_FIXED_CONFIG = dict( + use_cuts = 'internal', + use_t0 = True, + actions_ = [] +) FIT_NAMESPACE = "datacuts::theory::fitting " CLOSURE_NAMESPACE = "datacuts::theory::closuretest::fitting " @@ -40,7 +45,6 @@ INPUT_FOLDER = "input" TAB_FOLDER = "tables" - class N3FitError(Exception): """Exception raised when n3fit cannot succeed and knows why""" @@ -60,12 +64,12 @@ def init_output(self): # check if results folder exists self.output_path = pathlib.Path(self.output_path).absolute() - if not (self.output_path / "nnfit").is_dir(): + if not (self.output_path/"nnfit").is_dir(): if not re.fullmatch(r"[\w.\-]+", self.output_path.name): raise N3FitError("Invalid output folder name. Must be alphanumeric.") try: self.output_path.mkdir(exist_ok=True) - (self.output_path / "nnfit").mkdir(exist_ok=True) + (self.output_path /"nnfit").mkdir(exist_ok=True) except OSError as e: raise EnvironmentError_(e) from e @@ -120,23 +124,18 @@ def from_yaml(cls, o, *args, **kwargs): except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError( - f"Expecting input runcard to be a mapping, " - f"not '{type(file_content)}'." - ) + raise ConfigError(f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'.") - if file_content.get("closuretest") is not None: + if file_content.get('closuretest') is not None: namespace = CLOSURE_NAMESPACE else: namespace = FIT_NAMESPACE - N3FIT_FIXED_CONFIG["actions_"].append(namespace + "performfit") + N3FIT_FIXED_CONFIG['actions_'].append(namespace + "performfit") if fps := file_content["fitting"].get("savepseudodata", True): if fps != True: - raise TypeError( - f"fitting::savepseudodata is neither True nor False ({fps})" - ) + raise TypeError(f"fitting::savepseudodata is neither True nor False ({fps})") if len(kwargs["environment"].replicas) != 1: raise ConfigError( "Cannot request that multiple replicas are fitted and that " @@ -147,40 +146,27 @@ def from_yaml(cls, o, *args, **kwargs): training_action = namespace + "training_pseudodata" validation_action = namespace + "validation_pseudodata" - N3FIT_FIXED_CONFIG["actions_"].extend((training_action, validation_action)) + N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) - if thconfig := file_content.get("fiatlux"): - N3FIT_FIXED_CONFIG["fiatlux"] = thconfig + if (thconfig:=file_content.get('fiatlux')): + N3FIT_FIXED_CONFIG['fiatlux']=thconfig else: - N3FIT_FIXED_CONFIG["fiatlux"] = None - # Theorycovmat flags and defaults - N3FIT_FIXED_CONFIG["theory_covmat_flag"] = False - N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = False - N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = False - if (thconfig := file_content.get("theorycovmatconfig")) is not None: - N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] = thconfig.get( - "use_thcovmat_in_fitting", True - ) - N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] = thconfig.get( - "use_thcovmat_in_sampling", True - ) - if ( - N3FIT_FIXED_CONFIG["use_thcovmat_in_sampling"] - or N3FIT_FIXED_CONFIG["use_thcovmat_in_fitting"] - ): - N3FIT_FIXED_CONFIG["theory_covmat_flag"] = True - N3FIT_FIXED_CONFIG["use_user_uncertainties"] = thconfig.get( - "use_user_uncertainties", False - ) - N3FIT_FIXED_CONFIG["use_scalevar_uncertainties"] = thconfig.get( - "use_scalevar_uncertainties", True - ) - # Sampling flags - if (sam_t0 := file_content.get("sampling")) is not None: - N3FIT_FIXED_CONFIG["separate_multiplicative"] = sam_t0.get( - "separate_multiplicative", True - ) - # Fitting flag + N3FIT_FIXED_CONFIG['fiatlux']=None + #Theorycovmat flags and defaults + N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False + N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = False + N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = False + if (thconfig:=file_content.get('theorycovmatconfig')) is not None: + N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = thconfig.get('use_thcovmat_in_fitting', True) + N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = thconfig.get('use_thcovmat_in_sampling', True) + if N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] or N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting']: + N3FIT_FIXED_CONFIG['theory_covmat_flag'] = True + N3FIT_FIXED_CONFIG['use_user_uncertainties'] = thconfig.get('use_user_uncertainties', False) + N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = thconfig.get('use_scalevar_uncertainties', True) + #Sampling flags + if (sam_t0:=file_content.get('sampling')) is not None: + N3FIT_FIXED_CONFIG['separate_multiplicative'] = sam_t0.get('separate_multiplicative', True) + #Fitting flag file_content.update(N3FIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -197,13 +183,12 @@ def parse_fakedata(self, fakedata: bool): """ if fakedata: log.warning("using filtered closure data") - if not (self.environment.output_path / "filter").is_dir(): + if not (self.environment.output_path/'filter').is_dir(): raise ConfigError( "Could not find filter result at " f"{self.environment.output_path/'filter'} " "to load commondata from. Did you run filter on the " - "runcard?" - ) + "runcard?") return fakedata def produce_use_fitcommondata(self, fakedata): @@ -213,7 +198,8 @@ def produce_use_fitcommondata(self, fakedata): return fakedata def produce_kfold_parameters(self, kfold=None, hyperopt=None): - """Return None even if there are kfolds in the runcard if the hyperopt flag is not active""" + """Return None even if there are kfolds in the runcard if the hyperopt flag is not active + """ if hyperopt is not None: return kfold return None @@ -251,27 +237,18 @@ def __init__(self): @property def argparser(self): parser = super().argparser - parser.add_argument( - "-o", "--output", help="Output folder and name of the fit", default=None - ) + parser.add_argument("-o", "--output", help="Output folder and name of the fit", default=None) def check_positive(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError( - "%s is an invalid positive int value." % value - ) + raise argparse.ArgumentTypeError("%s is an invalid positive int value." % value) return ivalue - parser.add_argument( - "--hyperopt", help="Enable hyperopt scan", default=None, type=int - ) + parser.add_argument("--hyperopt", help="Enable hyperopt scan", default=None, type=int) parser.add_argument("replica", help="MC replica number", type=check_positive) parser.add_argument( - "-r", - "--replica_range", - help="End of the range of replicas to compute", - type=check_positive, + "-r", "--replica_range", help="End of the range of replicas to compute", type=check_positive ) return parser @@ -283,9 +260,7 @@ def get_commandline_arguments(self, cmdline=None): def run(self): try: - self.environment.config_yml = pathlib.Path( - self.args["config_yml"] - ).absolute() + self.environment.config_yml = pathlib.Path(self.args["config_yml"]).absolute() replica = self.args["replica"] if self.args["replica_range"]: replicas = list(range(replica, self.args["replica_range"] + 1)) @@ -299,9 +274,7 @@ def run(self): sys.exit(1) except Exception as e: log.critical(f"Bug in n3fit ocurred. Please report it.") - print( - colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr - ) + print(colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr) sys.exit(1) diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index 24b39c8987..d6a4f4b509 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -25,40 +25,39 @@ # top. -import hashlib -import logging -import pathlib +import sys import re import shutil -import sys +import pathlib +import logging +import hashlib import warnings -from reportengine import colors -from reportengine.compat import yaml +from validphys.config import Environment, Config, EnvironmentError_, ConfigError from validphys.app import App -from validphys.config import (Config, ConfigError, Environment, - EnvironmentError_) +from reportengine.compat import yaml +from reportengine import colors + SETUPFIT_FIXED_CONFIG = dict( actions_=[ - "datacuts check_t0pdfset", - "theory check_positivity", - ] -) - -SETUPFIT_PROVIDERS = [ - "validphys.filters", - "validphys.theorycovariance.construction", - "validphys.results", - "validphys.covmats", - "n3fit.n3fit_checks_provider", + 'datacuts check_t0pdfset', + 'theory check_positivity', + ]) + +SETUPFIT_PROVIDERS = ['validphys.filters', + 'validphys.theorycovariance.construction', + 'validphys.results', + 'validphys.covmats', + 'n3fit.n3fit_checks_provider' ] SETUPFIT_DEFAULTS = dict( - use_cuts="internal", + use_cuts = 'internal', ) + log = logging.getLogger(__name__) RUNCARD_COPY_FILENAME = "filter.yml" @@ -70,13 +69,11 @@ class SetupFitError(Exception): """Exception raised when setup-fit cannot succeed and knows why""" - pass class SetupFitEnvironment(Environment): """Container for information to be filled at run time""" - def init_output(self): # check file exists, is a file, has extension. if not self.config_yml.exists(): @@ -89,11 +86,9 @@ def init_output(self): self.output_path = pathlib.Path(self.output_path).absolute() if self.output_path.is_dir(): - log.warning( - f"Output folder exists: {self.output_path} Overwriting contents" - ) + log.warning(f"Output folder exists: {self.output_path} Overwriting contents") else: - if not re.fullmatch(r"[\w\-]+", self.output_path.name): + if not re.fullmatch(r'[\w\-]+', self.output_path.name): raise SetupFitError("Invalid output folder name. Must be alphanumeric.") try: self.output_path.mkdir() @@ -119,18 +114,16 @@ def init_output(self): def save_md5(self): """Generate md5 key from file""" output_filename = self.output_path / MD5_FILENAME - with open(self.config_yml, "rb") as f: + with open(self.config_yml, 'rb') as f: hash_md5 = hashlib.md5(f.read()).hexdigest() - with open(output_filename, "w") as g: + with open(output_filename, 'w') as g: g.write(hash_md5) log.info(f"md5 {hash_md5} stored in {output_filename}") @classmethod def ns_dump_description(cls): - return { - "filter_path": "The filter output folder", - **super().ns_dump_description(), - } + return {'filter_path': "The filter output folder", + **super().ns_dump_description()} class SetupFitConfig(Config): @@ -140,41 +133,37 @@ class SetupFitConfig(Config): def from_yaml(cls, o, *args, **kwargs): try: with warnings.catch_warnings(): - warnings.simplefilter("ignore", yaml.error.MantissaNoDotYAML1_1Warning) - # We need to specify the older version 1.1 to support the - # older configuration files, which liked to use on/off for - # booleans. - # The floating point parsing yields warnings everywhere, which - # we suppress. - file_content = yaml.safe_load(o, version="1.1") + warnings.simplefilter('ignore', + yaml.error.MantissaNoDotYAML1_1Warning) + #We need to specify the older version 1.1 to support the + #older configuration files, which liked to use on/off for + #booleans. + #The floating point parsing yields warnings everywhere, which + #we suppress. + file_content = yaml.safe_load(o, version='1.1') except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError( - f"Expecting input runcard to be a mapping, " - f"not '{type(file_content)}'." - ) - - if file_content.get("closuretest") is not None: - filter_action = "datacuts::closuretest::theory::fitting filter" - check_n3fit_action = ( - "datacuts::theory::closuretest::fitting n3fit_checks_action" - ) + raise ConfigError(f"Expecting input runcard to be a mapping, " + f"not '{type(file_content)}'.") + + if file_content.get('closuretest') is not None: + filter_action = 'datacuts::closuretest::theory::fitting filter' + check_n3fit_action = 'datacuts::theory::closuretest::fitting n3fit_checks_action' else: - filter_action = "datacuts::theory::fitting filter" - check_n3fit_action = "datacuts::theory::fitting n3fit_checks_action" - SETUPFIT_FIXED_CONFIG["actions_"] += [check_n3fit_action, filter_action] - if file_content.get("theorycovmatconfig") is not None: - SETUPFIT_FIXED_CONFIG["actions_"].append( - "datacuts::theory::theorycovmatconfig nnfit_theory_covmat" - ) - if file_content.get("fiatlux") is not None: - SETUPFIT_FIXED_CONFIG["actions_"].append("fiatlux check_luxset") - if file_content.get("fiatlux")["additional_errors"]: - SETUPFIT_FIXED_CONFIG["actions_"].append( - "fiatlux check_additional_errors" - ) - for k, v in SETUPFIT_DEFAULTS.items(): + filter_action = 'datacuts::theory::fitting filter' + check_n3fit_action = 'datacuts::theory::fitting n3fit_checks_action' + SETUPFIT_FIXED_CONFIG['actions_'] += [check_n3fit_action, filter_action] + if file_content.get('theorycovmatconfig') is not None: + SETUPFIT_FIXED_CONFIG['actions_'].append( + 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat') + if file_content.get('fiatlux') is not None: + SETUPFIT_FIXED_CONFIG['actions_'].append( + 'fiatlux check_luxset') + if file_content.get('fiatlux')["additional_errors"]: + SETUPFIT_FIXED_CONFIG['actions_'].append( + 'fiatlux check_additional_errors') + for k,v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) file_content.update(SETUPFIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -182,35 +171,31 @@ def from_yaml(cls, o, *args, **kwargs): class SetupFitApp(App): """The class which parsers and perform the filtering""" - environment_class = SetupFitEnvironment config_class = SetupFitConfig def __init__(self): - super(SetupFitApp, self).__init__( - name="setup-fit", providers=SETUPFIT_PROVIDERS - ) + super(SetupFitApp, self).__init__(name='setup-fit', + providers=SETUPFIT_PROVIDERS) @property def argparser(self): parser = super().argparser - parser.add_argument( - "-o", "--output", help="Output folder and name of the fit", default=None - ) + parser.add_argument('-o','--output', + help="Output folder and name of the fit", + default=None) return parser def get_commandline_arguments(self, cmdline=None): args = super().get_commandline_arguments(cmdline) - if args["output"] is None: - args["output"] = pathlib.Path(args["config_yml"]).stem + if args['output'] is None: + args['output'] = pathlib.Path(args['config_yml']).stem return args def run(self): try: # set folder output name - self.environment.config_yml = pathlib.Path( - self.args["config_yml"] - ).absolute() + self.environment.config_yml = pathlib.Path(self.args['config_yml']).absolute() # proceed with default run super().run() @@ -223,8 +208,8 @@ def run(self): except Exception as e: log.critical(f"Bug in setup-fit ocurred. Please report it.") print( - colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr - ) + colors.color_exception(e.__class__, e, e.__traceback__), + file=sys.stderr) sys.exit(1) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 637aa9021c..6c76f73d87 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -20,13 +20,12 @@ """ import logging from collections.abc import Iterable - import numpy as np import numpy.linalg as la -from validphys.arclength import arc_lengths, integrability_number from validphys.core import PDF, MCStats -from validphys.lhapdfset import LHAPDFSet from validphys.pdfbases import ALL_FLAVOURS, check_basis +from validphys.lhapdfset import LHAPDFSet +from validphys.arclength import integrability_number, arc_lengths log = logging.getLogger(__name__) # Order of the evolution basis output from n3fit @@ -75,9 +74,7 @@ def xfxQ(self, x, Q, n, fl): """Return the value of the PDF member for the given value in x""" if Q != self._fitting_q: log.warning( - "Querying N3LHAPDFSet at a value of Q=%f different from %f", - Q, - self._fitting_q, + "Querying N3LHAPDFSet at a value of Q=%f different from %f", Q, self._fitting_q ) return self.grid_values([fl], [x]).squeeze()[n] @@ -86,7 +83,7 @@ def _register_photon(self, xgrid): for m in self._lhapdf_set: pl = m.get_layer_re("add_photon") # if pl is an empy list there's no photon - if not pl: + if not pl : continue pl[0].register_photon(xgrid) # Recompile the model if necessary @@ -123,9 +120,7 @@ def __call__(self, xarr, flavours=None, replica=None): if replica is None or replica == 0: # We need generate output values for all replicas - result = np.concatenate( - [m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0 - ) + result = np.concatenate([m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0) if replica == 0: # We want _only_ the central value result = np.mean(result, axis=0, keepdims=True) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index a1bdff9610..c71c2a8354 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -4,37 +4,60 @@ @author: Zahari Kassabov """ -import copy +import logging +import pathlib import functools -import glob import inspect -import logging import numbers -import pathlib +import copy +from importlib.resources import read_text, contents + from collections import ChainMap, defaultdict from collections.abc import Mapping, Sequence -from importlib.resources import contents, read_text import pandas as pd -import validphys.scalevariations -from reportengine import configparser, report -from reportengine.compat import yaml -from reportengine.configparser import (ConfigError, _parse_func, element_of, - record_from_defaults) +import glob + +from reportengine import configparser from reportengine.environment import Environment, EnvironmentError_ +from reportengine.configparser import ( + ConfigError, + element_of, + _parse_func, + record_from_defaults, +) from reportengine.helputils import get_parser_type from reportengine.namespaces import NSList -from validphys.core import (CutsPolicy, DataGroupSpec, DataSetInput, - ExperimentInput, MatchedCuts, SimilarCuts, - ThCovMatSpec) +from reportengine import report +from reportengine.compat import yaml + +from validphys.core import ( + DataGroupSpec, + DataSetInput, + ExperimentInput, + CutsPolicy, + MatchedCuts, + SimilarCuts, + ThCovMatSpec, +) from validphys.fitdata import fitted_replica_indexes, num_fitted_replicas +from validphys.loader import ( + Loader, + LoaderError, + LoadFailedError, + DataNotFoundError, + PDFNotFound, + FallbackLoader, + InconsistentMetaDataError, +) from validphys.gridvalues import LUMI_CHANNELS -from validphys.loader import (DataNotFoundError, FallbackLoader, - InconsistentMetaDataError, Loader, LoaderError, - LoadFailedError, PDFNotFound) + from validphys.paramfits.config import ParamfitsConfig + from validphys.plotoptions import get_info +import validphys.scalevariations + log = logging.getLogger(__name__) @@ -42,13 +65,7 @@ class Environment(Environment): """Container for information to be filled at run time""" def __init__( - self, - *, - this_folder=None, - net=True, - upload=False, - dry=False, - **kwargs, + self, *, this_folder=None, net=True, upload=False, dry=False, **kwargs, ): if this_folder: self.this_folder = pathlib.Path(this_folder) @@ -206,7 +223,7 @@ def produce_inclusive_use_scalevar_uncertainties( point_prescription: (str, None) = None, ): """Whether to use a scale variation uncertainty theory covmat. - Checks whether a point prescription is included in the runcard and if so + Checks whether a point prescription is included in the runcard and if so assumes scale uncertainties are to be used.""" if (not use_scalevar_uncertainties) and (point_prescription is not None): use_scalevar_uncertainties = True @@ -238,7 +255,8 @@ def produce_pdfreplicas(self, fitpdf): return NSList(replicas, nskey="replica") def produce_fitcontextwithcuts(self, fit, fitinputcontext): - """Like fitinputcontext but setting the cuts policy.""" + """Like fitinputcontext but setting the cuts policy. + """ theoryid = fitinputcontext["theoryid"] data_input = fitinputcontext["data_input"] @@ -318,7 +336,8 @@ def parse_hyperscan(self, hyperscan): ) from e def parse_hyperscan_config(self, hyperscan_config, hyperopt=None): - """Configuration of the hyperscan""" + """Configuration of the hyperscan + """ if "from_hyperscan" in hyperscan_config: hyperscan = self.parse_hyperscan(hyperscan_config["from_hyperscan"]) log.info( @@ -379,7 +398,7 @@ def produce_basisfromfit(self, fit): return {"basis": basis} def produce_fitpdfandbasis(self, fitpdf, basisfromfit): - """Set the PDF and basis from the fit config.""" + """ Set the PDF and basis from the fit config. """ return {**fitpdf, **basisfromfit} @element_of("dataset_inputs") @@ -501,7 +520,7 @@ def _produce_matched_cuts(self, commondata): return MatchedCuts(cut_list, ndata=ndata) def _produce_similarity_cuts(self, commondata): - """Compute the intersection between two namespaces (similar to + """ Compute the intersection between two namespaces (similar to `fromintersection`) but additionally require that the predictions computed for each dataset across the namespaces are *similar*, specifically that the ratio between the absolute difference in the @@ -540,13 +559,7 @@ def _produce_similarity_cuts(self, commondata): matched_cuts = self._produce_matched_cuts(commondata) inps = [] for i, ns in enumerate(nss): - with self.set_context( - ns=self._curr_ns.new_child( - { - **ns, - } - ) - ): + with self.set_context(ns=self._curr_ns.new_child({**ns,})): # TODO: find a way to not duplicate this and use a dict # instead of a linear search _, dins = self.parse_from_(None, "dataset_inputs", write=False) @@ -600,9 +613,9 @@ def produce_dataset( check_plotting: bool = False, ): """Dataset specification from the theory and CommonData. - Use the cuts from the fit, if provided. If check_plotting is set to - True, attempt to lod and check the PLOTTING files - (note this may cause a noticeable slowdown in general).""" + Use the cuts from the fit, if provided. If check_plotting is set to + True, attempt to lod and check the PLOTTING files + (note this may cause a noticeable slowdown in general).""" name = dataset_input.name sysnum = dataset_input.sys cfac = dataset_input.cfac @@ -639,8 +652,8 @@ def produce_dataset( @configparser.element_of("experiments") def parse_experiment(self, experiment: dict): """A set of datasets where correlated systematics are taken - into account. It is a mapping where the keys are the experiment - name 'experiment' and a list of datasets.""" + into account. It is a mapping where the keys are the experiment + name 'experiment' and a list of datasets.""" try: name, datasets = experiment["experiment"], experiment["datasets"] except KeyError as e: @@ -687,8 +700,8 @@ def produce_experiment_from_input( def produce_sep_mult(self, separate_multiplicative=None): """ - Specifies whether to separate the multiplicative errors in the - experimental covmat construction. The default is True. + Specifies whether to separate the multiplicative errors in the + experimental covmat construction. The default is True. """ if separate_multiplicative is False: return False @@ -748,7 +761,7 @@ def produce_loaded_theory_covmat( ): """ Loads the theory covmat from the correct file according to how it - was generated by vp-setupfit. + was generated by vp-setupfit. """ if theory_covmat_flag is False: return 0.0 @@ -905,12 +918,7 @@ def produce_matched_datasets_from_dataspecs(self, dataspecs): inner_spec_list = inres["dataspecs"] = [] for ispec, spec in enumerate(dataspecs): # Passing spec by referene - d = ChainMap( - { - "dataset_input": all_names[ispec][k], - }, - spec, - ) + d = ChainMap({"dataset_input": all_names[ispec][k],}, spec) inner_spec_list.append(d) res.append(inres) res.sort(key=lambda x: (x["process"], x["dataset_name"])) @@ -934,12 +942,7 @@ def produce_matched_positivity_from_dataspecs(self, dataspecs): l = inres["dataspecs"] = [] for ispec, spec in enumerate(dataspecs): # Passing spec by referene - d = ChainMap( - { - "posdataset": all_names[ispec][k], - }, - spec, - ) + d = ChainMap({"posdataset": all_names[ispec][k],}, spec) l.append(d) res.append(inres) res.sort(key=lambda x: (x["posdataset_name"])) @@ -1026,9 +1029,7 @@ def parse_use_t0(self, do_use_t0: bool): # TODO: Find a good name for this def produce_t0set( - self, - t0pdfset=None, - use_t0=False, + self, t0pdfset=None, use_t0=False, ): """Return the t0set if use_t0 is True and None otherwise. Raises an error if t0 is requested but no t0set is given. @@ -1038,11 +1039,11 @@ def produce_t0set( raise ConfigError("Setting use_t0 requires specifying a valid t0pdfset") return t0pdfset return None - + def parse_luxset(self, name): """PDF set used to generate the photon with fiatlux.""" return self.parse_pdf(name) - + def parse_additional_errors(self, bool): """PDF set used to generate the photon additional errors: they are constructed using the replicas 101-107 of the PDF set @@ -1060,17 +1061,15 @@ def parse_fakepdf(self, name): return self.parse_pdf(name) def _parse_lagrange_multiplier(self, kind, theoryid, setdict): - """Lagrange multiplier constraints are mappings + """ Lagrange multiplier constraints are mappings containing a `dataset` and a `maxlambda` argument which - defines the maximum value allowed for the multiplier""" + defines the maximum value allowed for the multiplier """ bad_msg = f"{kind} must be a mapping with a name ('dataset') and a float multiplier (maxlambda)" theoryno, _ = theoryid lambda_key = "maxlambda" - # BCH allow for old-style runcards with 'poslambda' instead of 'maxlambda' + #BCH allow for old-style runcards with 'poslambda' instead of 'maxlambda' if "poslambda" in setdict and "maxlambda" not in setdict: - log.warning( - "The `poslambda` argument has been deprecated in favour of `maxlambda`" - ) + log.warning("The `poslambda` argument has been deprecated in favour of `maxlambda`") lambda_key = "poslambda" try: name = setdict["dataset"] @@ -1186,20 +1185,21 @@ def produce_nnfit_theory_covmat( if inclusive_use_scalevar_uncertainties: if use_user_uncertainties: # Both scalevar and user uncertainties - from validphys.theorycovariance.construction import \ - total_theory_covmat_fitting + from validphys.theorycovariance.construction import ( + total_theory_covmat_fitting, + ) f = total_theory_covmat_fitting else: # Only scalevar uncertainties - from validphys.theorycovariance.construction import \ - theory_covmat_custom_fitting + from validphys.theorycovariance.construction import ( + theory_covmat_custom_fitting, + ) f = theory_covmat_custom_fitting elif use_user_uncertainties: # Only user uncertainties - from validphys.theorycovariance.construction import \ - user_covmat_fitting + from validphys.theorycovariance.construction import user_covmat_fitting f = user_covmat_fitting @@ -1280,7 +1280,7 @@ def parse_fitdeclaration(self, label: str): return label def produce_all_commondata(self): - """produces all commondata using the loader function""" + """produces all commondata using the loader function """ ds_names = self.loader.available_datasets ds_inputs = [self.parse_dataset_input({"dataset": ds}) for ds in ds_names] cd_out = [ @@ -1368,8 +1368,11 @@ def produce_rules( ): """Produce filter rules based on the user defined input and defaults.""" - from validphys.filters import (Rule, RuleProcessingError, - default_filter_rules_input) + from validphys.filters import ( + Rule, + RuleProcessingError, + default_filter_rules_input, + ) theory_parameters = theoryid.get_description() @@ -1481,10 +1484,7 @@ def produce_defaults( return filter_defaults def produce_data( - self, - data_input, - *, - group_name="data", + self, data_input, *, group_name="data", ): """A set of datasets where correlated systematics are taken into account @@ -1586,23 +1586,22 @@ def load_default_data_grouping(self, spec): """Load the default grouping of data""" # slightly superfluous, only one default at present but perhaps # somebody will want to add to this at some point e.g for th. uncertainties - allowed = {"standard_report": "experiment", "thcovmat_fit": "ALL"} + allowed = { + "standard_report": "experiment", + "thcovmat_fit": "ALL" + } return allowed[spec] def produce_processed_data_grouping( - self, - use_thcovmat_in_fitting=False, - use_thcovmat_in_sampling=False, - data_grouping=None, - data_grouping_recorded_spec_=None, + self, use_thcovmat_in_fitting=False, use_thcovmat_in_sampling=False, data_grouping=None, data_grouping_recorded_spec_=None ): """Process the data_grouping key from the runcard, or lockfile. If `data_grouping_recorded_spec_` is present then its value is taken, and the runcard is assumed to be a lockfile. If data_grouping is None, then, if either use_thcovmat_in_fitting or use_thcovmat_in_sampling - (or both) are true (which means that the fit is a thcovmat fit), group all the datasets - together, otherwise fall back to the default behaviour of grouping by + (or both) are true (which means that the fit is a thcovmat fit), group all the datasets + together, otherwise fall back to the default behaviour of grouping by experiment (called standard_report). Else, the user can specfiy their own grouping, for example metadata_process. @@ -1628,9 +1627,7 @@ def produce_processed_metadata_group( return metadata_group def produce_group_dataset_inputs_by_metadata( - self, - data_input, - processed_metadata_group, + self, data_input, processed_metadata_group, ): """Take the data and the processed_metadata_group key and attempt to group the data, returns a list where each element specifies the data_input @@ -1690,9 +1687,9 @@ def produce_group_dataset_inputs_by_process(self, data_input): def produce_scale_variation_theories(self, theoryid, point_prescription): """Produces a list of theoryids given a theoryid at central scales and a point - prescription. The options for the latter are '3 point', '5 point', '5bar point', '7 point' - and '9 point'. Note that these are defined in arXiv:1906.10698. This hard codes the - theories needed for each prescription to avoid user error.""" + prescription. The options for the latter are '3 point', '5 point', '5bar point', '7 point' + and '9 point'. Note that these are defined in arXiv:1906.10698. This hard codes the + theories needed for each prescription to avoid user error.""" pp = point_prescription th = theoryid.id diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index 9d837331e0..a1d6bf685d 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -8,12 +8,14 @@ from importlib.resources import read_text import numpy as np -import validphys.cuts + from reportengine.checks import check, make_check from reportengine.compat import yaml -from validphys.commondatawriter import (write_commondata_to_file, - write_systype_to_file) - +import validphys.cuts +from validphys.commondatawriter import ( + write_commondata_to_file, + write_systype_to_file, + ) log = logging.getLogger(__name__) KIN_LABEL = { @@ -51,7 +53,6 @@ class BadPerturbativeOrder(ValueError): """Exception raised when the perturbative order string is not recognized.""" - class MissingRuleAttribute(RuleProcessingError, AttributeError): """Exception raised when a rule is missing required attributes.""" @@ -74,14 +75,13 @@ def default_filter_rules_input(): return yaml.safe_load(read_text(validphys.cuts, "filters.yaml")) + def check_nonnegative(var: str): """Ensure that `var` is positive""" - @make_check def run_check(ns, **kwargs): val = ns[var] check(val >= 0, f"'{var}' must be positive or equal zero, but it is {val!r}.") - return run_check @@ -95,7 +95,7 @@ def make_dataset_dir(path): def export_mask(path, mask): """Dump mask to file""" - np.savetxt(path, mask, fmt="%d") + np.savetxt(path, mask, fmt='%d') def filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed): @@ -105,7 +105,7 @@ def filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed): being shifted away from the underlying law. """ - log.info("Filtering closure-test data.") + log.info('Filtering closure-test data.') return _filter_closure_data(filter_path, data, fakepdf, fakenoise, filterseed) @@ -137,17 +137,16 @@ def filter_closure_data_by_experiment( def filter_real_data(filter_path, data): - """Filter real data, cutting any points which do not pass the filter rules.""" - log.info("Filtering real data.") + """Filter real data, cutting any points which do not pass the filter rules. + """ + log.info('Filtering real data.') return _filter_real_data(filter_path, data) def filter(filter_data): """Summarise filters applied to all datasets""" total_data, total_cut_data = np.atleast_2d(filter_data).sum(axis=0) - log.info( - f"Summary: {total_cut_data}/{total_data} datapoints passed kinematic cuts." - ) + log.info(f'Summary: {total_cut_data}/{total_data} datapoints passed kinematic cuts.') def _write_ds_cut_data(path, dataset): @@ -159,13 +158,11 @@ def _write_ds_cut_data(path, dataset): log.info("All {all_ndata} points in in {dataset.name} passed kinematic cuts.") else: filtered_dsndata = len(datamask) - log.info( - f"{len(datamask)}/{all_dsndata} datapoints " - f"in {dataset.name} passed kinematic cuts." - ) + log.info(f"{len(datamask)}/{all_dsndata} datapoints " + f"in {dataset.name} passed kinematic cuts.") # save to disk if datamask is not None: - export_mask(path / f"FKMASK_{dataset.name}.dat", datamask) + export_mask(path / f'FKMASK_{dataset.name}.dat', datamask) return all_dsndata, filtered_dsndata @@ -236,24 +233,24 @@ def _filter_closure_data( closure_data = level0_commondata_wc(data, fakepdf) for dataset in data.datasets: - # == print number of points passing cuts, make dataset directory and write FKMASK ==# + #== print number of points passing cuts, make dataset directory and write FKMASK ==# path = filter_path / dataset.name nfull, ncut = _write_ds_cut_data(path, dataset) make_dataset_dir(path / "systypes") total_data_points += nfull total_cut_data_points += ncut - + if fakenoise: - # ======= Level 1 closure test =======# + #======= Level 1 closure test =======# closure_data = make_level1_data( - data, - closure_data, - filterseed, - experiments_index, - ) + data, + closure_data, + filterseed, + experiments_index, + ) - # ====== write commondata and systype files ======# + #====== write commondata and systype files ======# if fakenoise: log.info("Writing Level1 data") else: @@ -262,7 +259,10 @@ def _filter_closure_data( for cd in closure_data: path_cd = filter_path / cd.setname / f"DATA_{cd.setname}.dat" path_sys = ( - filter_path / cd.setname / "systypes" / f"SYSTYPE_{cd.setname}_DEFAULT.dat" + filter_path + / cd.setname + / "systypes" + / f"SYSTYPE_{cd.setname}_DEFAULT.dat" ) write_commondata_to_file(commondata=cd, path=path_cd) write_systype_to_file(commondata=cd, path=path_sys) @@ -273,36 +273,31 @@ def _filter_closure_data( def check_t0pdfset(t0pdfset): """T0 pdf check""" t0pdfset.load() - log.info(f"{t0pdfset} T0 checked.") - + log.info(f'{t0pdfset} T0 checked.') def check_luxset(luxset): """Lux pdf check""" luxset.load() - log.info(f"{luxset} Lux pdf checked.") - + log.info(f'{luxset} Lux pdf checked.') def check_additional_errors(additional_errors): """Lux additional errors pdf check""" additional_errors.load() - log.info(f"{additional_errors} Lux additional errors pdf checked.") - + log.info(f'{additional_errors} Lux additional errors pdf checked.') def check_positivity(posdatasets): """Verify positive datasets are ready for the fit.""" - log.info("Verifying positivity tables:") + log.info('Verifying positivity tables:') for pos in posdatasets: pos.load_commondata() - log.info(f"{pos.name} checked.") - + log.info(f'{pos.name} checked.') def check_integrability(integdatasets): """Verify positive datasets are ready for the fit.""" - log.info("Verifying integrability tables:") + log.info('Verifying integrability tables:') for integ in integdatasets: integ.load_commondata() - log.info(f"{integ.name} checked.") - + log.info(f'{integ.name} checked.') class PerturbativeOrder: """Class that conveniently handles @@ -374,7 +369,6 @@ def __contains__(self, i): else: return i == self.numeric_pto - class Rule: """Rule object to be used to generate cuts mask. @@ -509,12 +503,7 @@ def _properties(self): """Attributes of the Rule class that are defining. Two Rules with identical ``_properties`` are considered equal. """ - return ( - self.rule_string, - self.dataset, - self.process_type, - self.theory_params["ID"], - ) + return (self.rule_string, self.dataset, self.process_type, self.theory_params['ID']) def __eq__(self, other): return self._properties == other._properties @@ -545,7 +534,9 @@ def __call__(self, dataset, idat): if k == "PTO" and hasattr(self, "PTO"): if v not in self.PTO: return None - elif hasattr(self, k) and (getattr(self, k) != v): + elif hasattr(self, k) and ( + getattr(self, k) != v + ): return None # Will return True if datapoint passes through the filter @@ -559,12 +550,12 @@ def __call__(self, dataset, idat): **ns, }, ) - except Exception as e: # pragma: no cover + except Exception as e: # pragma: no cover raise FatalRuleError( f"Error when applying rule {self.rule_string!r}: {e}" ) from e - def __repr__(self): # pragma: no cover + def __repr__(self): # pragma: no cover return self.rule_string def _make_kinematics_dict(self, dataset, idat) -> dict: @@ -581,7 +572,6 @@ def _make_point_namespace(self, dataset, idat) -> dict: ns[key] = eval(value, {**self.numpy_functions, **ns}) return ns - def get_cuts_for_dataset(commondata, rules) -> list: """Function to generate a list containing the index of all experimental points that passed kinematic diff --git a/validphys2/src/validphys/n3fit_data.py b/validphys2/src/validphys/n3fit_data.py index d0c2b6ec6f..2019aadc88 100644 --- a/validphys2/src/validphys/n3fit_data.py +++ b/validphys2/src/validphys/n3fit_data.py @@ -5,16 +5,20 @@ :py:func:`n3fit.performfit.performfit`. """ import functools +from collections import defaultdict import hashlib import logging -from collections import defaultdict import numpy as np import pandas as pd + from reportengine import collect from reportengine.table import table + +from validphys.n3fit_data_utils import ( + validphys_group_extractor, +) from validphys.core import IntegrabilitySetSpec, TupleComp -from validphys.n3fit_data_utils import validphys_group_extractor log = logging.getLogger(__name__) @@ -48,7 +52,6 @@ def replica_mcseed(replica, mcseed, genrep): res = np.random.randint(0, pow(2, 31)) return res - def replica_luxseed(replica, luxseed): """Generate the ``luxseed`` for a ``replica``. Identical to replica_nnseed but used for a different purpose. @@ -94,7 +97,7 @@ def tr_masks(data, replica_trvlseed): ndata = len(cuts.load()) if cuts else dataset.commondata.ndata frac = dataset.frac # We do this so that a given dataset will always have the same number of points masked - trmax = int(ndata * frac) + trmax = int(ndata*frac) if trmax == 0: # If that number is 0, then get 1 point with probability frac trmax = int(rng.random() < frac) @@ -235,13 +238,11 @@ def fitting_data_dict( """ # TODO: Plug in the python data loading when available. Including but not # limited to: central values, ndata, replica generation, covmat construction - expdata_true = np.concatenate( - [d.central_values for d in dataset_inputs_loaded_cd_with_cuts] - ) + expdata_true = np.concatenate([d.central_values for d in dataset_inputs_loaded_cd_with_cuts]) expdata = make_replica tr_masks = tr_masks.masks - covmat = dataset_inputs_fitting_covmat # t0 covmat, or theory covmat or whatever was decided by the runcard + covmat = dataset_inputs_fitting_covmat # t0 covmat, or theory covmat or whatever was decided by the runcard inv_true = np.linalg.inv(covmat) fittable_datasets = fittable_datasets_masked @@ -318,9 +319,7 @@ def fitting_data_dict( return dict_out -exps_fitting_data_dict = collect( - "fitting_data_dict", ("group_dataset_inputs_by_metadata",) -) +exps_fitting_data_dict = collect("fitting_data_dict", ("group_dataset_inputs_by_metadata",)) def replica_nnseed_fitting_data_dict(replica, exps_fitting_data_dict, replica_nnseed): @@ -337,9 +336,7 @@ def replica_nnseed_fitting_data_dict(replica, exps_fitting_data_dict, replica_nn return (replica, exps_fitting_data_dict, replica_nnseed) -replicas_nnseed_fitting_data_dict = collect( - "replica_nnseed_fitting_data_dict", ("replicas",) -) +replicas_nnseed_fitting_data_dict = collect("replica_nnseed_fitting_data_dict", ("replicas",)) groups_replicas_indexed_make_replica = collect( "indexed_make_replica", ("group_dataset_inputs_by_experiment", "replicas") ) @@ -354,7 +351,7 @@ def pseudodata_table(groups_replicas_indexed_make_replica, replicas): Notes ----- Whilst running ``n3fit``, this action will only be called if - `fitting::savepseudodata` is `true` (as per the default setting) and + `fitting::savepseudodata` is `true` (as per the default setting) and replicas are fitted one at a time. The table can be found in the replica folder i.e. /nnfit/replica_*/ @@ -447,12 +444,8 @@ def replica_training_mask(exps_tr_masks, replica, experiments_index): [345 rows x 1 columns] """ - all_masks = np.concatenate( - [ds_mask for exp_masks in exps_tr_masks for ds_mask in exp_masks] - ) - return pd.DataFrame( - all_masks, columns=[f"replica {replica}"], index=experiments_index - ) + all_masks = np.concatenate([ds_mask for exp_masks in exps_tr_masks for ds_mask in exp_masks]) + return pd.DataFrame(all_masks, columns=[f"replica {replica}"], index=experiments_index) replicas_training_mask = collect("replica_training_mask", ("replicas",)) diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index bca895386b..5fd5814d98 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -87,10 +87,8 @@ def __init__(self, theoryid, lux_params, replicas): for replica in replicas: f2 = sf.InterpStructureFunction(path_to_F2, self.luxpdfset.members[replica]) fl = sf.InterpStructureFunction(path_to_FL, self.luxpdfset.members[replica]) - if not np.isclose(f2.q2_max, fl.q2_max): - log.error( - "FKtables for fiatlux_dis_F2 and fiatlux_dis_FL have two different q2_max" - ) + if not np.isclose(f2.q2_max, fl.q2_max) : + log.error("FKtables for fiatlux_dis_F2 and fiatlux_dis_FL have two different q2_max") fiatlux_runcard["q2_max"] = float(f2.q2_max) f2lo = sf.F2LO(self.luxpdfset.members[replica], theory) From 9e9be873273935245fe61a250438e5e73320f13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 16:15:52 +0200 Subject: [PATCH 197/204] Add references to qed doc --- doc/sphinx/source/n3fit/runcard_detailed.rst | 7 +++-- doc/sphinx/source/references.bib | 29 ++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/doc/sphinx/source/n3fit/runcard_detailed.rst b/doc/sphinx/source/n3fit/runcard_detailed.rst index 0ee697f508..1e92e232ac 100644 --- a/doc/sphinx/source/n3fit/runcard_detailed.rst +++ b/doc/sphinx/source/n3fit/runcard_detailed.rst @@ -427,7 +427,8 @@ QED fit It is possible to perform a QED fit adding the key `fiatlux` to the runcard. In this way a photon PDF will be generated using the `FiatLux` public library that implements the `LuxQED` -approach. The parameters to be added are the following: +(see :cite:p:`Manohar:2016nzj` and :cite:p:`Manohar:2017eqh`) approach. +The parameters to be added are the following: .. code-block:: yaml @@ -436,8 +437,8 @@ approach. The parameters to be added are the following: additional_errors: true luxseed: 1234567890 -`luxset` is the PDF set used to generate the photon PDF with `FiatLux`. The code wil generate as -much photon replicas as the number of replicas contained in the `luxset`. Therefore, if the user +`luxset` is the PDF set used to generate the photon PDF with `FiatLux `. +The code wil generate as much photon replicas as the number of replicas contained in the `luxset`. Therefore, if the user tries to generate a replica with ID higher than the maximum ID of the `luxset`, the code will raise an error. Moreover, being the `LuxQED` approach an iterated prcedure, and given that some replicas do not pass the `postfit` selection criteria, the user should make sure that the number of replicas in diff --git a/doc/sphinx/source/references.bib b/doc/sphinx/source/references.bib index 3dc9bb8782..bb646dcae4 100644 --- a/doc/sphinx/source/references.bib +++ b/doc/sphinx/source/references.bib @@ -604,3 +604,32 @@ @article{Carrazza:2016htc pages = "205", year = "2016" } + +@article{Manohar:2017eqh, + author = "Manohar, Aneesh V. and Nason, Paolo and Salam, Gavin P. and Zanderighi, Giulia", + title = "{The Photon Content of the Proton}", + eprint = "1708.01256", + archivePrefix = "arXiv", + primaryClass = "hep-ph", + reportNumber = "CERN-TH-2017-141", + doi = "10.1007/JHEP12(2017)046", + journal = "JHEP", + volume = "12", + pages = "046", + year = "2017" +} + +@article{Manohar:2016nzj, + author = "Manohar, Aneesh and Nason, Paolo and Salam, Gavin P. and Zanderighi, Giulia", + title = "{How bright is the proton? A precise determination of the photon parton distribution function}", + eprint = "1607.04266", + archivePrefix = "arXiv", + primaryClass = "hep-ph", + reportNumber = "CERN-TH-2016-155", + doi = "10.1103/PhysRevLett.117.242002", + journal = "Phys. Rev. Lett.", + volume = "117", + number = "24", + pages = "242002", + year = "2016" +} From ea97997d803b57f0d9efe316b95faebfc91823d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 16:24:07 +0200 Subject: [PATCH 198/204] Move documentation in tutorials --- doc/sphinx/source/n3fit/runcard_detailed.rst | 27 -------------------- doc/sphinx/source/tutorials/run-fit.md | 5 ++++ 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/doc/sphinx/source/n3fit/runcard_detailed.rst b/doc/sphinx/source/n3fit/runcard_detailed.rst index 1e92e232ac..58aa9b94b6 100644 --- a/doc/sphinx/source/n3fit/runcard_detailed.rst +++ b/doc/sphinx/source/n3fit/runcard_detailed.rst @@ -421,30 +421,3 @@ It is however possible to disable them by setting to false the ``sum_rules`` fla It is also possible to impose just the valence or the momentum sum rules by using the ``VSR`` or ``MSR`` flags, respectively (``True`` is equal to ``All``). - -QED fit -^^^^^^^ - -It is possible to perform a QED fit adding the key `fiatlux` to the runcard. In this way -a photon PDF will be generated using the `FiatLux` public library that implements the `LuxQED` -(see :cite:p:`Manohar:2016nzj` and :cite:p:`Manohar:2017eqh`) approach. -The parameters to be added are the following: - -.. code-block:: yaml - - fiatlux: - luxset: NNPDF40_nnlo_as_01180 - additional_errors: true - luxseed: 1234567890 - -`luxset` is the PDF set used to generate the photon PDF with `FiatLux `. -The code wil generate as much photon replicas as the number of replicas contained in the `luxset`. Therefore, if the user -tries to generate a replica with ID higher than the maximum ID of the `luxset`, the code will -raise an error. Moreover, being the `LuxQED` approach an iterated prcedure, and given that some replicas -do not pass the `postfit` selection criteria, the user should make sure that the number of replicas in -the `luxset` is high enough such that in the final iteration there will be a number of replicas -higher than the final replicas desired. -`additional_errors` is the parameter that switches on and off the additional errors of the `LuxQED` approach, -while `luxseed` is the seed used to generate such errors. - -Whenever the photon PDF is generated, it will remain constant during the fit and will enter in the `MSR`. diff --git a/doc/sphinx/source/tutorials/run-fit.md b/doc/sphinx/source/tutorials/run-fit.md index dc26ec8d94..f0a1d10201 100644 --- a/doc/sphinx/source/tutorials/run-fit.md +++ b/doc/sphinx/source/tutorials/run-fit.md @@ -256,3 +256,8 @@ Iterate the fit It may be desirable to iterate a fit to achieve a higher degree of convergence/stability in the fit. To read more about this, see [How to run an iterated fit](run-iterated-fit). + +QED fit +------- + +In order to run a QED fit see [How to run a QED fit](run-qed-fit) From 3e5428765702ae696688d70fca21e15caa4c88ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 16:32:59 +0200 Subject: [PATCH 199/204] Run black and isort with pyproject.toml --- n3fit/src/n3fit/checks.py | 13 +- n3fit/src/n3fit/layers/__init__.py | 11 +- n3fit/src/n3fit/layers/msr_normalization.py | 6 +- n3fit/src/n3fit/layers/rotations.py | 12 +- n3fit/src/n3fit/model_gen.py | 28 +- n3fit/src/n3fit/model_trainer.py | 26 +- n3fit/src/n3fit/msr.py | 48 +-- n3fit/src/n3fit/performfit.py | 181 +++++------ n3fit/src/n3fit/scripts/n3fit_exec.py | 89 +++--- n3fit/src/n3fit/scripts/vp_setupfit.py | 80 +++-- n3fit/src/n3fit/vpinterface.py | 14 +- validphys2/src/validphys/config.py | 327 ++++++++------------ validphys2/src/validphys/filters.py | 89 +++--- validphys2/src/validphys/n3fit_data.py | 18 +- validphys2/src/validphys/photon/compute.py | 33 +- 15 files changed, 452 insertions(+), 523 deletions(-) diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 4573926822..dc9c6a9d22 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -1,15 +1,17 @@ """ This module contains checks to be perform by n3fit on the input """ -import os import logging import numbers +import os + import numpy as np -from reportengine.checks import make_argcheck, CheckError -from validphys.core import PDF -from validphys.pdfbases import check_basis + from n3fit.hyper_optimization import penalties as penalties_module from n3fit.hyper_optimization import rewards as rewards_module +from reportengine.checks import CheckError, make_argcheck +from validphys.core import PDF +from validphys.pdfbases import check_basis log = logging.getLogger(__name__) @@ -395,11 +397,12 @@ def check_deprecated_options(fitting): if option in fitting: log.warning("'fitting::%s' is an nnfit-only key, it will be ignored", option) + @make_argcheck def check_fiatlux_pdfs_id(replicas, fiatlux, replica_path): if fiatlux is not None: luxset = fiatlux["luxset"] - pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 + pdfs_ids = luxset.get_members() - 1 # get_members counts also replica0 max_id = max(replicas) if max_id > pdfs_ids: raise CheckError( diff --git a/n3fit/src/n3fit/layers/__init__.py b/n3fit/src/n3fit/layers/__init__.py index d7d288bf9b..a9168bda13 100644 --- a/n3fit/src/n3fit/layers/__init__.py +++ b/n3fit/src/n3fit/layers/__init__.py @@ -1,8 +1,9 @@ -from .preprocessing import Preprocessing -from .rotations import FkRotation, FlavourToEvolution, ObsRotation, AddPhoton -from .x_operations import xIntegrator, xDivide -from .msr_normalization import MSR_Normalization +from n3fit.backends import MetaLayer + from .DIS import DIS from .DY import DY from .mask import Mask -from n3fit.backends import MetaLayer +from .msr_normalization import MSR_Normalization +from .preprocessing import Preprocessing +from .rotations import AddPhoton, FkRotation, FlavourToEvolution, ObsRotation +from .x_operations import xDivide, xIntegrator diff --git a/n3fit/src/n3fit/layers/msr_normalization.py b/n3fit/src/n3fit/layers/msr_normalization.py index 991d92718f..51219ad73f 100644 --- a/n3fit/src/n3fit/layers/msr_normalization.py +++ b/n3fit/src/n3fit/layers/msr_normalization.py @@ -56,10 +56,12 @@ def call(self, pdf_integrated, ph_replica): if self._photons: photon_integral = self._photons[ph_replica] else: - photon_integral = 0. + photon_integral = 0.0 if self._msr_enabled: - n_ag = [(1.0 - y[GLUON_IDX[0][0]-1] - photon_integral) / y[GLUON_IDX[0][0]]] * len(GLUON_IDX) + n_ag = [(1.0 - y[GLUON_IDX[0][0] - 1] - photon_integral) / y[GLUON_IDX[0][0]]] * len( + GLUON_IDX + ) norm_constants += n_ag if self._vsr_enabled: diff --git a/n3fit/src/n3fit/layers/rotations.py b/n3fit/src/n3fit/layers/rotations.py index 6db811ec11..686d3c5bbb 100644 --- a/n3fit/src/n3fit/layers/rotations.py +++ b/n3fit/src/n3fit/layers/rotations.py @@ -2,6 +2,7 @@ This module includes rotation layers """ import numpy as np + from n3fit.backends import MetaLayer from n3fit.backends import operations as op from validphys import pdfbases @@ -28,7 +29,7 @@ def __init__(self, rotation_matrix, axes=1, **kwargs): super().__init__(**kwargs) def is_identity(self): - """ Returns true if the rotation is an identity """ + """Returns true if the rotation is an identity""" # check whether it is a mxm matrix if self.rotation_matrix.shape[0] == self.rotation_matrix.shape[1]: # check whether it is the identity @@ -41,12 +42,15 @@ def call(self, x_raw): class FlavourToEvolution(Rotation): """ - Rotates from the flavour basis to - the evolution basis. + Rotates from the flavour basis to + the evolution basis. """ def __init__( - self, flav_info, fitbasis, **kwargs, + self, + flav_info, + fitbasis, + **kwargs, ): rotation_matrix = pdfbases.fitbasis_to_NN31IC(flav_info, fitbasis) super().__init__(rotation_matrix, axes=1, **kwargs) diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index d360b43ea3..ca7844d12a 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -10,16 +10,24 @@ """ from dataclasses import dataclass + import numpy as np -from n3fit.msr import msr_impose -from n3fit.layers import DIS, DY, ObsRotation, losses -from n3fit.layers import Preprocessing, FkRotation, FlavourToEvolution, AddPhoton -from n3fit.layers.observable import is_unique -from n3fit.backends import MetaModel, Input +from n3fit.backends import Input, Lambda, MetaLayer, MetaModel, base_layer_selector from n3fit.backends import operations as op -from n3fit.backends import MetaLayer, Lambda -from n3fit.backends import base_layer_selector, regularizer_selector +from n3fit.backends import regularizer_selector +from n3fit.layers import ( + DIS, + DY, + AddPhoton, + FkRotation, + FlavourToEvolution, + ObsRotation, + Preprocessing, + losses, +) +from n3fit.layers.observable import is_unique +from n3fit.msr import msr_impose @dataclass @@ -496,7 +504,7 @@ def pdfNN_layer_generator( will be a (1, None, 2) tensor where dim [:,:,0] is scaled parallel_models: int How many models should be trained in parallel - photon: :py:class:`validphys.photon.compute.Photon` + photon: :py:class:`validphys.photon.compute.Photon` If given, gives the AddPhoton layer a function to compute a photon which will be added at the index 0 of the 14-size FK basis This same function will also be used to compute the MSR component for the photon @@ -573,7 +581,9 @@ def pdfNN_layer_generator( # Normalization and sum rules if impose_sumrule: - sumrule_layer, integrator_input = msr_impose(mode=impose_sumrule, scaler=scaler, photons=photons) + sumrule_layer, integrator_input = msr_impose( + mode=impose_sumrule, scaler=scaler, photons=photons + ) model_input["integrator_input"] = integrator_input else: sumrule_layer = lambda x: x diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index fb165aa111..1f1a5a4011 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -8,18 +8,20 @@ This allows to use hyperscanning libraries, that need to change the parameters of the network between iterations while at the same time keeping the amount of redundant calls to a minimum """ -import logging from collections import namedtuple from itertools import zip_longest +import logging + import numpy as np from scipy.interpolate import PchipInterpolator + from n3fit import model_gen -from n3fit.backends import MetaModel, clear_backend_state, callbacks +from n3fit.backends import MetaModel, callbacks, clear_backend_state from n3fit.backends import operations as op -from n3fit.stopping import Stopping -from n3fit.vpinterface import N3PDF import n3fit.hyper_optimization.penalties import n3fit.hyper_optimization.rewards +from n3fit.stopping import Stopping +from n3fit.vpinterface import N3PDF from validphys.photon.compute import Photon log = logging.getLogger(__name__) @@ -37,6 +39,7 @@ # See ModelTrainer::_xgrid_generation for the definition of each field and how they are generated InputInfo = namedtuple("InputInfo", ["input", "split", "idx"]) + def _pdf_injection(pdf_layers, observables, masks): """ Takes as input a list of PDF layers each corresponding to one observable (also given as a list) @@ -365,9 +368,7 @@ def _xgrid_generation(self): # now the output needs to be splitted so that each experiment takes its corresponding input sp_ar = [[i.shape[1] for i in inputs_unique]] sp_kw = {"axis": 1} - sp_layer = op.as_layer( - op.split, op_args=sp_ar, op_kwargs=sp_kw, name="pdf_split" - ) + sp_layer = op.as_layer(op.split, op_args=sp_ar, op_kwargs=sp_kw, name="pdf_split") return InputInfo(input_layer, sp_layer, inputs_idx) @@ -423,9 +424,7 @@ def _model_generation(self, xinput, pdf_models, partition, partition_idx): for pdf_model in pdf_models: # The input to the full model also works as the input to the PDF model # We apply the Model as Layers and save for later the model (full_pdf) - full_model_input_dict, full_pdf = pdf_model.apply_as_layer( - {"pdf_input": xinput.input} - ) + full_model_input_dict, full_pdf = pdf_model.apply_as_layer({"pdf_input": xinput.input}) all_replicas_pdf.append(full_pdf) # Note that all models share the same symbolic input so we take as input the last @@ -686,7 +685,7 @@ def _generate_pdf( dictionary of arguments for the regularizer seed: int seed for the NN - photons: :py:class:`validphys.photon.compute.Photon` + photons: :py:class:`validphys.photon.compute.Photon` function to compute the photon PDF see model_gen.pdfNN_layer_generator for more information @@ -886,13 +885,13 @@ def hyperparametrizable(self, params): xinput = self._xgrid_generation() # Initialize all photon classes for the different replicas: if self.lux_params: - photons=Photon( + photons = Photon( theoryid=self.theoryid, lux_params=self.lux_params, replicas=self.replicas, ) else: - photons=None + photons = None ### Training loop for k, partition in enumerate(self.kpartitions): # Each partition of the kfolding needs to have its own separate model @@ -919,7 +918,6 @@ def hyperparametrizable(self, params): pl = m.get_layer("add_photon") pl.register_photon(xinput.input.tensor_content) - # Model generation joins all the different observable layers # together with pdf model generated above models = self._model_generation(xinput, pdf_models, partition, k) diff --git a/n3fit/src/n3fit/msr.py b/n3fit/src/n3fit/msr.py index b43ec40bfb..9db718f285 100644 --- a/n3fit/src/n3fit/msr.py +++ b/n3fit/src/n3fit/msr.py @@ -2,11 +2,11 @@ The constraint module include functions to impose the momentum sum rules on the PDFs """ import logging + import numpy as np -from n3fit.layers import xDivide, MSR_Normalization, xIntegrator from n3fit.backends import operations as op - +from n3fit.layers import MSR_Normalization, xDivide, xIntegrator log = logging.getLogger(__name__) @@ -38,26 +38,26 @@ def gen_integration_input(nx): def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): """ - This function receives: - Generates a function that applies a normalization layer to the fit. - - fit_layer: the 8-basis layer of PDF which we fit - The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) - - final_layer: the 14-basis which is fed to the fktable - and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). - It uses pdf_fit to compute the sum rule and returns a modified version of - the final_pdf layer with a normalisation by which the sum rule is imposed - - Parameters - ---------- - nx: int - number of points for the integration grid, default: 2000 - mode: str - what sum rules to compute (MSR, VSR or All), default: All - scaler: scaler - Function to apply to the input. If given the input to the model - will be a (1, None, 2) tensor where dim [:,:,0] is scaled - photon: :py:class:`validphys.photon.compute.Photon` - If given, gives the AddPhoton layer a function to compute the MSR component for the photon + This function receives: + Generates a function that applies a normalization layer to the fit. + - fit_layer: the 8-basis layer of PDF which we fit + The normalization is computed from the direct output of the NN (so the 7,8-flavours basis) + - final_layer: the 14-basis which is fed to the fktable + and it is applied to the input of the fktable (i.e., to the 14-flavours fk-basis). + It uses pdf_fit to compute the sum rule and returns a modified version of + the final_pdf layer with a normalisation by which the sum rule is imposed + + Parameters + ---------- + nx: int + number of points for the integration grid, default: 2000 + mode: str + what sum rules to compute (MSR, VSR or All), default: All + scaler: scaler + Function to apply to the input. If given the input to the model + will be a (1, None, 2) tensor where dim [:,:,0] is scaled + photon: :py:class:`validphys.photon.compute.Photon` + If given, gives the AddPhoton layer a function to compute the MSR component for the photon """ # 1. Generate the fake input which will be used to integrate @@ -86,14 +86,14 @@ def msr_impose(nx=int(2e3), mode='All', scaler=None, photons=None): # and will return it appropiately normalized. def apply_normalization(layer_pdf, ph_replica): """ - layer_pdf: output of the PDF, unnormalized, ready for the fktable + layer_pdf: output of the PDF, unnormalized, ready for the fktable """ x_original = op.op_gather_keep_dims(xgrid_input, -1, axis=-1) pdf_integrand = op.op_multiply([division_by_x(x_original), layer_pdf(xgrid_input)]) normalization = normalizer(integrator(pdf_integrand), ph_replica) def ultimate_pdf(x): - return layer_pdf(x)*normalization + return layer_pdf(x) * normalization return ultimate_pdf diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 03c36aa1b1..431bb1f5a6 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -5,7 +5,9 @@ # Backend-independent imports import copy import logging + import numpy as np + import n3fit.checks from n3fit.vpinterface import N3PDF @@ -18,8 +20,8 @@ @n3fit.checks.check_fiatlux_pdfs_id def performfit( *, - n3fit_checks_action, # wrapper for all checks - replicas, # checks specific to performfit + n3fit_checks_action, # wrapper for all checks + replicas, # checks specific to performfit replicas_nnseed_fitting_data_dict, posdatasets_fitting_pos_dict, integdatasets_fitting_integ_dict, @@ -42,86 +44,86 @@ def performfit( parallel_models=False, ): """ - This action will (upon having read a validcard) process a full PDF fit - for a set of replicas. - - The input to this function is provided by validphys - and/or defined in the runcards or commandline arguments. - - This controller is provided with: - 1. Seeds generated using the replica number and the seeds defined in the runcard. - 2. Loaded datasets with replicas generated. - 2.1 Loaded positivity/integrability sets. - - The workflow of this controller is as follows: - 1. Generate a ModelTrainer object holding information to create the NN and perform a fit - (at this point no NN object has been generated) - 1.1 (if hyperopt) generates the hyperopt scanning dictionary - taking as a base the fitting dictionary and the runcard's hyperscanner dictionary - 2. Pass the dictionary of parameters to ModelTrainer - for the NN to be generated and the fit performed - 2.1 (if hyperopt) Loop over point 4 for `hyperopt` number of times - 3. Once the fit is finished, output the PDF grid and accompanying files - - Parameters - ---------- - genrep: bool - Whether or not to generate MC replicas. (Only used for checks) - data: validphys.core.DataGroupSpec - containing the datasets to be included in the fit. (Only used - for checks) - replicas_nnseed_fitting_data_dict: list[tuple] - list with element for each replica (typically just one) to be - fitted. Each element - is a tuple containing the replica number, nnseed and - ``fitted_data_dict`` containing all of the data, metadata - for each group of datasets which is to be fitted. - posdatasets_fitting_pos_dict: list[dict] - list of dictionaries containing all data and metadata for each - positivity dataset - integdatasets_fitting_integ_dict: list[dict] - list of dictionaries containing all data and metadata for each - integrability dataset - theoryid: validphys.core.TheoryIDSpec - Theory which is used to generate theory predictions from model - during fit. Object also contains some metadata on the theory - settings. - fiatlux: dict - dictionary containing the params needed from LuxQED - basis: list[dict] - preprocessing information for each flavour to be fitted. - fitbasis: str - Valid basis which the fit is to be ran in. Available bases can - be found in :py:mod:`validphys.pdfbases`. - sum_rules: bool - Whether to impose sum rules in fit. By default set to True - parameters: dict - Mapping containing parameters which define the network - architecture/fitting methodology. - replica_path: pathlib.Path - path to the output of this run - output_path: str - name of the fit - save: None, str - model file where weights will be saved, used in conjunction with - ``load``. - load: None, str - model file from which to load weights from. - hyperscanner: dict - dictionary containing the details of the hyperscanner - hyperopt: int - if given, number of hyperopt iterations to run - kfold_parameters: None, dict - dictionary with kfold settings used in hyperopt. - tensorboard: None, dict - mapping containing tensorboard settings if it is to be used. By - default it is None and tensorboard is not enabled. - debug: bool - activate some debug options - maxcores: int - maximum number of (logical) cores that the backend should be aware of - parallel_models: bool - whether to run models in parallel + This action will (upon having read a validcard) process a full PDF fit + for a set of replicas. + + The input to this function is provided by validphys + and/or defined in the runcards or commandline arguments. + + This controller is provided with: + 1. Seeds generated using the replica number and the seeds defined in the runcard. + 2. Loaded datasets with replicas generated. + 2.1 Loaded positivity/integrability sets. + + The workflow of this controller is as follows: + 1. Generate a ModelTrainer object holding information to create the NN and perform a fit + (at this point no NN object has been generated) + 1.1 (if hyperopt) generates the hyperopt scanning dictionary + taking as a base the fitting dictionary and the runcard's hyperscanner dictionary + 2. Pass the dictionary of parameters to ModelTrainer + for the NN to be generated and the fit performed + 2.1 (if hyperopt) Loop over point 4 for `hyperopt` number of times + 3. Once the fit is finished, output the PDF grid and accompanying files + + Parameters + ---------- + genrep: bool + Whether or not to generate MC replicas. (Only used for checks) + data: validphys.core.DataGroupSpec + containing the datasets to be included in the fit. (Only used + for checks) + replicas_nnseed_fitting_data_dict: list[tuple] + list with element for each replica (typically just one) to be + fitted. Each element + is a tuple containing the replica number, nnseed and + ``fitted_data_dict`` containing all of the data, metadata + for each group of datasets which is to be fitted. + posdatasets_fitting_pos_dict: list[dict] + list of dictionaries containing all data and metadata for each + positivity dataset + integdatasets_fitting_integ_dict: list[dict] + list of dictionaries containing all data and metadata for each + integrability dataset + theoryid: validphys.core.TheoryIDSpec + Theory which is used to generate theory predictions from model + during fit. Object also contains some metadata on the theory + settings. + fiatlux: dict + dictionary containing the params needed from LuxQED + basis: list[dict] + preprocessing information for each flavour to be fitted. + fitbasis: str + Valid basis which the fit is to be ran in. Available bases can + be found in :py:mod:`validphys.pdfbases`. + sum_rules: bool + Whether to impose sum rules in fit. By default set to True + parameters: dict + Mapping containing parameters which define the network + architecture/fitting methodology. + replica_path: pathlib.Path + path to the output of this run + output_path: str + name of the fit + save: None, str + model file where weights will be saved, used in conjunction with + ``load``. + load: None, str + model file from which to load weights from. + hyperscanner: dict + dictionary containing the details of the hyperscanner + hyperopt: int + if given, number of hyperopt iterations to run + kfold_parameters: None, dict + dictionary with kfold settings used in hyperopt. + tensorboard: None, dict + mapping containing tensorboard settings if it is to be used. By + default it is None and tensorboard is not enabled. + debug: bool + activate some debug options + maxcores: int + maximum number of (logical) cores that the backend should be aware of + parallel_models: bool + whether to run models in parallel """ from n3fit.backends import set_initial_state @@ -134,8 +136,8 @@ def performfit( # All potentially backend dependent imports should come inside the fit function # so they can eventually be set from the runcard - from n3fit.model_trainer import ModelTrainer from n3fit.io.writer import WriterWrapper + from n3fit.model_trainer import ModelTrainer # Note: there are three possible scenarios for the loop of replicas: # 1.- Only one replica is being run, in this case the loop is only evaluated once @@ -292,13 +294,12 @@ def performfit( replica_path_set, output_path.name, training_chi2, val_chi2, exp_chi2 ) log.info( - "Best fit for replica #%d, chi2=%.3f (tr=%.3f, vl=%.3f)", - replica_number, - exp_chi2, - training_chi2, - val_chi2 - ) - + "Best fit for replica #%d, chi2=%.3f (tr=%.3f, vl=%.3f)", + replica_number, + exp_chi2, + training_chi2, + val_chi2, + ) # Save the weights to some file for the given replica if save: diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 9b80cb5308..01b21b682d 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -3,28 +3,22 @@ n3fit - performs fit using ml external frameworks """ -import sys +import argparse +import logging +import pathlib import re import shutil -import pathlib -import logging +import sys import warnings -import argparse -from validphys.app import App -from validphys.config import Environment, Config -from validphys.config import EnvironmentError_, ConfigError -from validphys.core import FitSpec from reportengine import colors from reportengine.compat import yaml from reportengine.namespaces import NSList +from validphys.app import App +from validphys.config import Config, ConfigError, Environment, EnvironmentError_ +from validphys.core import FitSpec - -N3FIT_FIXED_CONFIG = dict( - use_cuts = 'internal', - use_t0 = True, - actions_ = [] -) +N3FIT_FIXED_CONFIG = dict(use_cuts='internal', use_t0=True, actions_=[]) FIT_NAMESPACE = "datacuts::theory::fitting " CLOSURE_NAMESPACE = "datacuts::theory::closuretest::fitting " @@ -45,6 +39,7 @@ INPUT_FOLDER = "input" TAB_FOLDER = "tables" + class N3FitError(Exception): """Exception raised when n3fit cannot succeed and knows why""" @@ -64,12 +59,12 @@ def init_output(self): # check if results folder exists self.output_path = pathlib.Path(self.output_path).absolute() - if not (self.output_path/"nnfit").is_dir(): + if not (self.output_path / "nnfit").is_dir(): if not re.fullmatch(r"[\w.\-]+", self.output_path.name): raise N3FitError("Invalid output folder name. Must be alphanumeric.") try: self.output_path.mkdir(exist_ok=True) - (self.output_path /"nnfit").mkdir(exist_ok=True) + (self.output_path / "nnfit").mkdir(exist_ok=True) except OSError as e: raise EnvironmentError_(e) from e @@ -124,7 +119,9 @@ def from_yaml(cls, o, *args, **kwargs): except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError(f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'.") + raise ConfigError( + f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'." + ) if file_content.get('closuretest') is not None: namespace = CLOSURE_NAMESPACE @@ -148,25 +145,38 @@ def from_yaml(cls, o, *args, **kwargs): N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action)) - if (thconfig:=file_content.get('fiatlux')): - N3FIT_FIXED_CONFIG['fiatlux']=thconfig + if thconfig := file_content.get('fiatlux'): + N3FIT_FIXED_CONFIG['fiatlux'] = thconfig else: - N3FIT_FIXED_CONFIG['fiatlux']=None - #Theorycovmat flags and defaults + N3FIT_FIXED_CONFIG['fiatlux'] = None + # Theorycovmat flags and defaults N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = False N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = False - if (thconfig:=file_content.get('theorycovmatconfig')) is not None: - N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = thconfig.get('use_thcovmat_in_fitting', True) - N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = thconfig.get('use_thcovmat_in_sampling', True) - if N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] or N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting']: + if (thconfig := file_content.get('theorycovmatconfig')) is not None: + N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] = thconfig.get( + 'use_thcovmat_in_fitting', True + ) + N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] = thconfig.get( + 'use_thcovmat_in_sampling', True + ) + if ( + N3FIT_FIXED_CONFIG['use_thcovmat_in_sampling'] + or N3FIT_FIXED_CONFIG['use_thcovmat_in_fitting'] + ): N3FIT_FIXED_CONFIG['theory_covmat_flag'] = True - N3FIT_FIXED_CONFIG['use_user_uncertainties'] = thconfig.get('use_user_uncertainties', False) - N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = thconfig.get('use_scalevar_uncertainties', True) - #Sampling flags - if (sam_t0:=file_content.get('sampling')) is not None: - N3FIT_FIXED_CONFIG['separate_multiplicative'] = sam_t0.get('separate_multiplicative', True) - #Fitting flag + N3FIT_FIXED_CONFIG['use_user_uncertainties'] = thconfig.get( + 'use_user_uncertainties', False + ) + N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = thconfig.get( + 'use_scalevar_uncertainties', True + ) + # Sampling flags + if (sam_t0 := file_content.get('sampling')) is not None: + N3FIT_FIXED_CONFIG['separate_multiplicative'] = sam_t0.get( + 'separate_multiplicative', True + ) + # Fitting flag file_content.update(N3FIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -183,12 +193,13 @@ def parse_fakedata(self, fakedata: bool): """ if fakedata: log.warning("using filtered closure data") - if not (self.environment.output_path/'filter').is_dir(): + if not (self.environment.output_path / 'filter').is_dir(): raise ConfigError( "Could not find filter result at " f"{self.environment.output_path/'filter'} " "to load commondata from. Did you run filter on the " - "runcard?") + "runcard?" + ) return fakedata def produce_use_fitcommondata(self, fakedata): @@ -198,8 +209,7 @@ def produce_use_fitcommondata(self, fakedata): return fakedata def produce_kfold_parameters(self, kfold=None, hyperopt=None): - """Return None even if there are kfolds in the runcard if the hyperopt flag is not active - """ + """Return None even if there are kfolds in the runcard if the hyperopt flag is not active""" if hyperopt is not None: return kfold return None @@ -237,7 +247,9 @@ def __init__(self): @property def argparser(self): parser = super().argparser - parser.add_argument("-o", "--output", help="Output folder and name of the fit", default=None) + parser.add_argument( + "-o", "--output", help="Output folder and name of the fit", default=None + ) def check_positive(value): ivalue = int(value) @@ -248,7 +260,10 @@ def check_positive(value): parser.add_argument("--hyperopt", help="Enable hyperopt scan", default=None, type=int) parser.add_argument("replica", help="MC replica number", type=check_positive) parser.add_argument( - "-r", "--replica_range", help="End of the range of replicas to compute", type=check_positive + "-r", + "--replica_range", + help="End of the range of replicas to compute", + type=check_positive, ) return parser diff --git a/n3fit/src/n3fit/scripts/vp_setupfit.py b/n3fit/src/n3fit/scripts/vp_setupfit.py index d6a4f4b509..c2ce728acf 100644 --- a/n3fit/src/n3fit/scripts/vp_setupfit.py +++ b/n3fit/src/n3fit/scripts/vp_setupfit.py @@ -25,39 +25,39 @@ # top. -import sys +import hashlib +import logging +import pathlib import re import shutil -import pathlib -import logging -import hashlib +import sys import warnings -from validphys.config import Environment, Config, EnvironmentError_, ConfigError -from validphys.app import App -from reportengine.compat import yaml from reportengine import colors - +from reportengine.compat import yaml +from validphys.app import App +from validphys.config import Config, ConfigError, Environment, EnvironmentError_ SETUPFIT_FIXED_CONFIG = dict( actions_=[ 'datacuts check_t0pdfset', 'theory check_positivity', - ]) + ] +) -SETUPFIT_PROVIDERS = ['validphys.filters', - 'validphys.theorycovariance.construction', - 'validphys.results', - 'validphys.covmats', - 'n3fit.n3fit_checks_provider' +SETUPFIT_PROVIDERS = [ + 'validphys.filters', + 'validphys.theorycovariance.construction', + 'validphys.results', + 'validphys.covmats', + 'n3fit.n3fit_checks_provider', ] SETUPFIT_DEFAULTS = dict( - use_cuts = 'internal', + use_cuts='internal', ) - log = logging.getLogger(__name__) RUNCARD_COPY_FILENAME = "filter.yml" @@ -69,11 +69,13 @@ class SetupFitError(Exception): """Exception raised when setup-fit cannot succeed and knows why""" + pass class SetupFitEnvironment(Environment): """Container for information to be filled at run time""" + def init_output(self): # check file exists, is a file, has extension. if not self.config_yml.exists(): @@ -122,8 +124,7 @@ def save_md5(self): @classmethod def ns_dump_description(cls): - return {'filter_path': "The filter output folder", - **super().ns_dump_description()} + return {'filter_path': "The filter output folder", **super().ns_dump_description()} class SetupFitConfig(Config): @@ -133,19 +134,19 @@ class SetupFitConfig(Config): def from_yaml(cls, o, *args, **kwargs): try: with warnings.catch_warnings(): - warnings.simplefilter('ignore', - yaml.error.MantissaNoDotYAML1_1Warning) - #We need to specify the older version 1.1 to support the - #older configuration files, which liked to use on/off for - #booleans. - #The floating point parsing yields warnings everywhere, which - #we suppress. + warnings.simplefilter('ignore', yaml.error.MantissaNoDotYAML1_1Warning) + # We need to specify the older version 1.1 to support the + # older configuration files, which liked to use on/off for + # booleans. + # The floating point parsing yields warnings everywhere, which + # we suppress. file_content = yaml.safe_load(o, version='1.1') except yaml.error.YAMLError as e: raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): - raise ConfigError(f"Expecting input runcard to be a mapping, " - f"not '{type(file_content)}'.") + raise ConfigError( + f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'." + ) if file_content.get('closuretest') is not None: filter_action = 'datacuts::closuretest::theory::fitting filter' @@ -156,14 +157,13 @@ def from_yaml(cls, o, *args, **kwargs): SETUPFIT_FIXED_CONFIG['actions_'] += [check_n3fit_action, filter_action] if file_content.get('theorycovmatconfig') is not None: SETUPFIT_FIXED_CONFIG['actions_'].append( - 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat') + 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat' + ) if file_content.get('fiatlux') is not None: - SETUPFIT_FIXED_CONFIG['actions_'].append( - 'fiatlux check_luxset') + SETUPFIT_FIXED_CONFIG['actions_'].append('fiatlux check_luxset') if file_content.get('fiatlux')["additional_errors"]: - SETUPFIT_FIXED_CONFIG['actions_'].append( - 'fiatlux check_additional_errors') - for k,v in SETUPFIT_DEFAULTS.items(): + SETUPFIT_FIXED_CONFIG['actions_'].append('fiatlux check_additional_errors') + for k, v in SETUPFIT_DEFAULTS.items(): file_content.setdefault(k, v) file_content.update(SETUPFIT_FIXED_CONFIG) return cls(file_content, *args, **kwargs) @@ -171,19 +171,19 @@ def from_yaml(cls, o, *args, **kwargs): class SetupFitApp(App): """The class which parsers and perform the filtering""" + environment_class = SetupFitEnvironment config_class = SetupFitConfig def __init__(self): - super(SetupFitApp, self).__init__(name='setup-fit', - providers=SETUPFIT_PROVIDERS) + super(SetupFitApp, self).__init__(name='setup-fit', providers=SETUPFIT_PROVIDERS) @property def argparser(self): parser = super().argparser - parser.add_argument('-o','--output', - help="Output folder and name of the fit", - default=None) + parser.add_argument( + '-o', '--output', help="Output folder and name of the fit", default=None + ) return parser def get_commandline_arguments(self, cmdline=None): @@ -207,9 +207,7 @@ def run(self): sys.exit(1) except Exception as e: log.critical(f"Bug in setup-fit ocurred. Please report it.") - print( - colors.color_exception(e.__class__, e, e.__traceback__), - file=sys.stderr) + print(colors.color_exception(e.__class__, e, e.__traceback__), file=sys.stderr) sys.exit(1) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 6c76f73d87..52cf001980 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -18,14 +18,16 @@ """ -import logging from collections.abc import Iterable +import logging + import numpy as np import numpy.linalg as la + +from validphys.arclength import arc_lengths, integrability_number from validphys.core import PDF, MCStats -from validphys.pdfbases import ALL_FLAVOURS, check_basis from validphys.lhapdfset import LHAPDFSet -from validphys.arclength import integrability_number, arc_lengths +from validphys.pdfbases import ALL_FLAVOURS, check_basis log = logging.getLogger(__name__) # Order of the evolution basis output from n3fit @@ -83,7 +85,7 @@ def _register_photon(self, xgrid): for m in self._lhapdf_set: pl = m.get_layer_re("add_photon") # if pl is an empy list there's no photon - if not pl : + if not pl: continue pl[0].register_photon(xgrid) # Recompile the model if necessary @@ -120,7 +122,9 @@ def __call__(self, xarr, flavours=None, replica=None): if replica is None or replica == 0: # We need generate output values for all replicas - result = np.concatenate([m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0) + result = np.concatenate( + [m.predict({"pdf_input": mod_xgrid}) for m in self._lhapdf_set], axis=0 + ) if replica == 0: # We want _only_ the central value result = np.mean(result, axis=0, keepdims=True) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index c71c2a8354..a9ee17a651 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -4,58 +4,47 @@ @author: Zahari Kassabov """ -import logging -import pathlib +from collections import ChainMap, defaultdict +from collections.abc import Mapping, Sequence +import copy import functools +import glob +from importlib.resources import contents, read_text import inspect +import logging import numbers -import copy -from importlib.resources import read_text, contents - -from collections import ChainMap, defaultdict -from collections.abc import Mapping, Sequence +import pathlib import pandas as pd -import glob -from reportengine import configparser +from reportengine import configparser, report +from reportengine.compat import yaml +from reportengine.configparser import ConfigError, _parse_func, element_of, record_from_defaults from reportengine.environment import Environment, EnvironmentError_ -from reportengine.configparser import ( - ConfigError, - element_of, - _parse_func, - record_from_defaults, -) from reportengine.helputils import get_parser_type from reportengine.namespaces import NSList -from reportengine import report -from reportengine.compat import yaml - from validphys.core import ( + CutsPolicy, DataGroupSpec, DataSetInput, ExperimentInput, - CutsPolicy, MatchedCuts, SimilarCuts, ThCovMatSpec, ) from validphys.fitdata import fitted_replica_indexes, num_fitted_replicas +from validphys.gridvalues import LUMI_CHANNELS from validphys.loader import ( + DataNotFoundError, + FallbackLoader, + InconsistentMetaDataError, Loader, LoaderError, LoadFailedError, - DataNotFoundError, PDFNotFound, - FallbackLoader, - InconsistentMetaDataError, ) -from validphys.gridvalues import LUMI_CHANNELS - from validphys.paramfits.config import ParamfitsConfig - from validphys.plotoptions import get_info - import validphys.scalevariations log = logging.getLogger(__name__) @@ -65,7 +54,13 @@ class Environment(Environment): """Container for information to be filled at run time""" def __init__( - self, *, this_folder=None, net=True, upload=False, dry=False, **kwargs, + self, + *, + this_folder=None, + net=True, + upload=False, + dry=False, + **kwargs, ): if this_folder: self.this_folder = pathlib.Path(this_folder) @@ -84,10 +79,7 @@ def __init__( try: self.loader = loader_class() except LoaderError as e: - log.error( - "Failed to find the paths. These are configured " - "in the nnprofile settings" - ) + log.error("Failed to find the paths. These are configured " "in the nnprofile settings") raise EnvironmentError_(e) from e self.deta_path = self.loader.datapath self.results_path = self.loader.resultspath @@ -207,9 +199,7 @@ def parse_use_cuts(self, use_cuts: (bool, str)): elif isinstance(use_cuts, str) and use_cuts in valid_cuts: res = CutsPolicy(use_cuts) else: - raise ConfigError( - f"Invalid use_cuts setting: '{use_cuts}'.", use_cuts, valid_cuts - ) + raise ConfigError(f"Invalid use_cuts setting: '{use_cuts}'.", use_cuts, valid_cuts) return res @@ -223,7 +213,7 @@ def produce_inclusive_use_scalevar_uncertainties( point_prescription: (str, None) = None, ): """Whether to use a scale variation uncertainty theory covmat. - Checks whether a point prescription is included in the runcard and if so + Checks whether a point prescription is included in the runcard and if so assumes scale uncertainties are to be used.""" if (not use_scalevar_uncertainties) and (point_prescription is not None): use_scalevar_uncertainties = True @@ -255,8 +245,7 @@ def produce_pdfreplicas(self, fitpdf): return NSList(replicas, nskey="replica") def produce_fitcontextwithcuts(self, fit, fitinputcontext): - """Like fitinputcontext but setting the cuts policy. - """ + """Like fitinputcontext but setting the cuts policy.""" theoryid = fitinputcontext["theoryid"] data_input = fitinputcontext["data_input"] @@ -331,18 +320,13 @@ def parse_hyperscan(self, hyperscan): try: return self.loader.check_hyperscan(hyperscan) except LoadFailedError as e: - raise ConfigError( - str(e), hyperscan, self.loader.available_hyperscans - ) from e + raise ConfigError(str(e), hyperscan, self.loader.available_hyperscans) from e def parse_hyperscan_config(self, hyperscan_config, hyperopt=None): - """Configuration of the hyperscan - """ + """Configuration of the hyperscan""" if "from_hyperscan" in hyperscan_config: hyperscan = self.parse_hyperscan(hyperscan_config["from_hyperscan"]) - log.info( - "Using previous hyperscan: '%s' to generate the search space", hyperscan - ) + log.info("Using previous hyperscan: '%s' to generate the search space", hyperscan) return hyperscan.as_input().get("hyperscan_config") if "use_tries_from" in hyperscan_config: @@ -378,8 +362,7 @@ def produce_multiclosure_underlyinglaw(self, fits): if len(laws) != 1: raise ConfigError( - "Did not find unique underlying law from fits, " - f"instead found: {laws}" + "Did not find unique underlying law from fits, " f"instead found: {laws}" ) return self.parse_pdf(laws.pop()) @@ -398,7 +381,7 @@ def produce_basisfromfit(self, fit): return {"basis": basis} def produce_fitpdfandbasis(self, fitpdf, basisfromfit): - """ Set the PDF and basis from the fit config. """ + """Set the PDF and basis from the fit config.""" return {**fitpdf, **basisfromfit} @element_of("dataset_inputs") @@ -417,9 +400,7 @@ def parse_dataset_input(self, dataset: Mapping): if name.startswith("POS"): raise ConfigError("Please, use `posdataset` for positivity") except KeyError: - raise ConfigError( - "'dataset' must be a mapping with " "'dataset' and 'sysnum'" - ) + raise ConfigError("'dataset' must be a mapping with " "'dataset' and 'sysnum'") sysnum = dataset.get("sys") cfac = dataset.get("cfac", tuple()) @@ -437,9 +418,7 @@ def parse_dataset_input(self, dataset: Mapping): kdiff = dataset.keys() - known_keys for k in kdiff: # Abuse ConfigError to get the suggestions. - log.warning( - ConfigError(f"Key '{k}' in dataset_input not known.", k, known_keys) - ) + log.warning(ConfigError(f"Key '{k}' in dataset_input not known.", k, known_keys)) return DataSetInput( name=name, sys=sysnum, @@ -502,15 +481,11 @@ def _produce_matched_cuts(self, commondata): self._check_dataspecs_type(nss) if not nss: - raise ConfigError( - "'cuts_intersection_spec' must contain at least one namespace." - ) + raise ConfigError("'cuts_intersection_spec' must contain at least one namespace.") for ns in nss: with self.set_context( - ns=self._curr_ns.new_child(ns).new_child( - {"use_cuts": CutsPolicy.INTERNAL} - ) + ns=self._curr_ns.new_child(ns).new_child({"use_cuts": CutsPolicy.INTERNAL}) ): # Note: Do not call _produce_internal_cuts directly here: # That doesn't correctly set the namespace in a way that `rules` @@ -520,7 +495,7 @@ def _produce_matched_cuts(self, commondata): return MatchedCuts(cut_list, ndata=ndata) def _produce_similarity_cuts(self, commondata): - """ Compute the intersection between two namespaces (similar to + """Compute the intersection between two namespaces (similar to `fromintersection`) but additionally require that the predictions computed for each dataset across the namespaces are *similar*, specifically that the ratio between the absolute difference in the @@ -544,22 +519,24 @@ def _produce_similarity_cuts(self, commondata): ) try: - _, exclusion_list = self.parse_from_( - None, "do_not_require_similarity_for", write=False - ) + _, exclusion_list = self.parse_from_(None, "do_not_require_similarity_for", write=False) except configparser.InputNotFoundError: exclusion_list = [] name = commondata.name # slightly circular here, since matched cuts will re-produce nss if name in exclusion_list: - with self.set_context( - ns=self._curr_ns.new_child({"use_cuts": CutsPolicy.INTERNAL}) - ): + with self.set_context(ns=self._curr_ns.new_child({"use_cuts": CutsPolicy.INTERNAL})): return self.parse_from_(None, "cuts", write=False)[1] matched_cuts = self._produce_matched_cuts(commondata) inps = [] for i, ns in enumerate(nss): - with self.set_context(ns=self._curr_ns.new_child({**ns,})): + with self.set_context( + ns=self._curr_ns.new_child( + { + **ns, + } + ) + ): # TODO: find a way to not duplicate this and use a dict # instead of a linear search _, dins = self.parse_from_(None, "dataset_inputs", write=False) @@ -613,9 +590,9 @@ def produce_dataset( check_plotting: bool = False, ): """Dataset specification from the theory and CommonData. - Use the cuts from the fit, if provided. If check_plotting is set to - True, attempt to lod and check the PLOTTING files - (note this may cause a noticeable slowdown in general).""" + Use the cuts from the fit, if provided. If check_plotting is set to + True, attempt to lod and check the PLOTTING files + (note this may cause a noticeable slowdown in general).""" name = dataset_input.name sysnum = dataset_input.sys cfac = dataset_input.cfac @@ -652,8 +629,8 @@ def produce_dataset( @configparser.element_of("experiments") def parse_experiment(self, experiment: dict): """A set of datasets where correlated systematics are taken - into account. It is a mapping where the keys are the experiment - name 'experiment' and a list of datasets.""" + into account. It is a mapping where the keys are the experiment + name 'experiment' and a list of datasets.""" try: name, datasets = experiment["experiment"], experiment["datasets"] except KeyError as e: @@ -684,9 +661,7 @@ def parse_experiment_input(self, ei: dict): return ExperimentInput(name=name, datasets=datasets) # TODO: Do away with the mapping and make the conversion implicitly - def produce_experiment_from_input( - self, experiment_input, theoryid, use_cuts, fit=None - ): + def produce_experiment_from_input(self, experiment_input, theoryid, use_cuts, fit=None): """Return a mapping containing a single experiment from an experiment input. NOTE: This might be deprecated in the future.""" return { @@ -700,8 +675,8 @@ def produce_experiment_from_input( def produce_sep_mult(self, separate_multiplicative=None): """ - Specifies whether to separate the multiplicative errors in the - experimental covmat construction. The default is True. + Specifies whether to separate the multiplicative errors in the + experimental covmat construction. The default is True. """ if separate_multiplicative is False: return False @@ -761,7 +736,7 @@ def produce_loaded_theory_covmat( ): """ Loads the theory covmat from the correct file according to how it - was generated by vp-setupfit. + was generated by vp-setupfit. """ if theory_covmat_flag is False: return 0.0 @@ -769,34 +744,21 @@ def produce_loaded_theory_covmat( generic_path = "datacuts_theory_theorycovmatconfig_theory_covmat_custom.csv" if use_user_uncertainties is True: if use_scalevar_uncertainties is True: - generic_path = ( - "datacuts_theory_theorycovmatconfig_total_theory_covmat.csv" - ) + generic_path = "datacuts_theory_theorycovmatconfig_total_theory_covmat.csv" else: generic_path = "datacuts_theory_theorycovmatconfig_user_covmat.csv" # check if there are multiple files files = glob.glob(str(output_path / "tables/*theorycovmat*")) paths = [ - str( - output_path - / "tables/datacuts_theory_theorycovmatconfig_theory_covmat_custom.csv" - ), - str( - output_path - / "tables/datacuts_theory_theorycovmatconfig_total_theory_covmat.csv" - ), - str( - output_path - / "tables/datacuts_theory_theorycovmatconfig_user_covmat.csv" - ), + str(output_path / "tables/datacuts_theory_theorycovmatconfig_theory_covmat_custom.csv"), + str(output_path / "tables/datacuts_theory_theorycovmatconfig_total_theory_covmat.csv"), + str(output_path / "tables/datacuts_theory_theorycovmatconfig_user_covmat.csv"), ] paths.remove(str(output_path / "tables" / generic_path)) for f in files: for path in paths: if f == path: - raise ValueError( - "More than one theory_covmat file in folder tables" - ) + raise ValueError("More than one theory_covmat file in folder tables") theorypath = output_path / "tables" / generic_path theory_covmat = pd.read_csv( theorypath, @@ -863,8 +825,7 @@ def produce_dataset_inputs_covariance_matrix(self, use_pdferr: bool = False): def _check_dataspecs_type(dataspecs): if not isinstance(dataspecs, Sequence): raise ConfigError( - "dataspecs should be a sequence of mappings, not " - f"{type(dataspecs).__name__}" + "dataspecs should be a sequence of mappings, not " f"{type(dataspecs).__name__}" ) for spec in dataspecs: @@ -918,7 +879,12 @@ def produce_matched_datasets_from_dataspecs(self, dataspecs): inner_spec_list = inres["dataspecs"] = [] for ispec, spec in enumerate(dataspecs): # Passing spec by referene - d = ChainMap({"dataset_input": all_names[ispec][k],}, spec) + d = ChainMap( + { + "dataset_input": all_names[ispec][k], + }, + spec, + ) inner_spec_list.append(d) res.append(inres) res.sort(key=lambda x: (x["process"], x["dataset_name"])) @@ -942,7 +908,12 @@ def produce_matched_positivity_from_dataspecs(self, dataspecs): l = inres["dataspecs"] = [] for ispec, spec in enumerate(dataspecs): # Passing spec by referene - d = ChainMap({"posdataset": all_names[ispec][k],}, spec) + d = ChainMap( + { + "posdataset": all_names[ispec][k], + }, + spec, + ) l.append(d) res.append(inres) res.sort(key=lambda x: (x["posdataset_name"])) @@ -1011,13 +982,9 @@ def produce_combined_shift_and_theory_dataspecs(self, theoryconfig, shiftconfig) # TODO: Worth it to do some black magic to not pass params explicitly? # Note that `parse_experiments` doesn't exist yet. - def parse_reweighting_experiments( - self, experiments, *, theoryid, use_cuts, fit=None - ): + def parse_reweighting_experiments(self, experiments, *, theoryid, use_cuts, fit=None): """A list of experiments to be used for reweighting.""" - return self.parse_experiments( - experiments, theoryid=theoryid, use_cuts=use_cuts, fit=fit - ) + return self.parse_experiments(experiments, theoryid=theoryid, use_cuts=use_cuts, fit=fit) def parse_t0pdfset(self, name): """PDF set used to generate the t0 covmat.""" @@ -1029,7 +996,9 @@ def parse_use_t0(self, do_use_t0: bool): # TODO: Find a good name for this def produce_t0set( - self, t0pdfset=None, use_t0=False, + self, + t0pdfset=None, + use_t0=False, ): """Return the t0set if use_t0 is True and None otherwise. Raises an error if t0 is requested but no t0set is given. @@ -1039,11 +1008,11 @@ def produce_t0set( raise ConfigError("Setting use_t0 requires specifying a valid t0pdfset") return t0pdfset return None - + def parse_luxset(self, name): """PDF set used to generate the photon with fiatlux.""" return self.parse_pdf(name) - + def parse_additional_errors(self, bool): """PDF set used to generate the photon additional errors: they are constructed using the replicas 101-107 of the PDF set @@ -1061,13 +1030,15 @@ def parse_fakepdf(self, name): return self.parse_pdf(name) def _parse_lagrange_multiplier(self, kind, theoryid, setdict): - """ Lagrange multiplier constraints are mappings + """Lagrange multiplier constraints are mappings containing a `dataset` and a `maxlambda` argument which - defines the maximum value allowed for the multiplier """ - bad_msg = f"{kind} must be a mapping with a name ('dataset') and a float multiplier (maxlambda)" + defines the maximum value allowed for the multiplier""" + bad_msg = ( + f"{kind} must be a mapping with a name ('dataset') and a float multiplier (maxlambda)" + ) theoryno, _ = theoryid lambda_key = "maxlambda" - #BCH allow for old-style runcards with 'poslambda' instead of 'maxlambda' + # BCH allow for old-style runcards with 'poslambda' instead of 'maxlambda' if "poslambda" in setdict and "maxlambda" not in setdict: log.warning("The `poslambda` argument has been deprecated in favour of `maxlambda`") lambda_key = "poslambda" @@ -1094,8 +1065,7 @@ def parse_posdataset(self, posset: dict, *, theoryid): def produce_posdatasets(self, positivity): if not isinstance(positivity, dict) or "posdatasets" not in positivity: raise ConfigError( - "Failed to get 'posdatasets' from positivity. " - "Expected that key to be present." + "Failed to get 'posdatasets' from positivity. " "Expected that key to be present." ) return positivity["posdatasets"] @@ -1118,12 +1088,8 @@ def produce_reweight_all_datasets(self, experiments): ret = [] for experiment in experiments: for dsinput, dataset in zip(experiment, experiment.datasets): - single_exp = DataGroupSpec( - experiment.name, datasets=[dataset], dsinputs=[dsinput] - ) - ret.append( - {"reweighting_experiments": [single_exp], "dataset_input": dsinput} - ) + single_exp = DataGroupSpec(experiment.name, datasets=[dataset], dsinputs=[dsinput]) + ret.append({"reweighting_experiments": [single_exp], "dataset_input": dsinput}) return ret """ @@ -1185,16 +1151,12 @@ def produce_nnfit_theory_covmat( if inclusive_use_scalevar_uncertainties: if use_user_uncertainties: # Both scalevar and user uncertainties - from validphys.theorycovariance.construction import ( - total_theory_covmat_fitting, - ) + from validphys.theorycovariance.construction import total_theory_covmat_fitting f = total_theory_covmat_fitting else: # Only scalevar uncertainties - from validphys.theorycovariance.construction import ( - theory_covmat_custom_fitting, - ) + from validphys.theorycovariance.construction import theory_covmat_custom_fitting f = theory_covmat_custom_fitting elif use_user_uncertainties: @@ -1219,20 +1181,14 @@ def produce_fitthcovmat( theory covariance matrix then returns `False`. """ if not isinstance(use_thcovmat_if_present, bool): - raise ConfigError( - "use_thcovmat_if_present should be a boolean, by default it is False" - ) + raise ConfigError("use_thcovmat_if_present should be a boolean, by default it is False") if use_thcovmat_if_present and not fit: - raise ConfigError( - "`use_thcovmat_if_present` was true but no `fit` was specified." - ) + raise ConfigError("`use_thcovmat_if_present` was true but no `fit` was specified.") if use_thcovmat_if_present and fit: try: - thcovmat_present = fit.as_input()["theorycovmatconfig"][ - "use_thcovmat_in_fitting" - ] + thcovmat_present = fit.as_input()["theorycovmatconfig"]["use_thcovmat_in_fitting"] except KeyError: # assume covmat wasn't used and fill in key accordingly but warn user log.warning( @@ -1280,21 +1236,17 @@ def parse_fitdeclaration(self, label: str): return label def produce_all_commondata(self): - """produces all commondata using the loader function """ + """produces all commondata using the loader function""" ds_names = self.loader.available_datasets ds_inputs = [self.parse_dataset_input({"dataset": ds}) for ds in ds_names] - cd_out = [ - self.produce_commondata(dataset_input=ds_input) for ds_input in ds_inputs - ] + cd_out = [self.produce_commondata(dataset_input=ds_input) for ds_input in ds_inputs] return cd_out def parse_groupby(self, grouping: str): """parses the groupby key and checks it is an allowed grouping""" # TODO: think if better way to do this properly if grouping not in ["experiment", "nnpdf31_process"]: - raise ConfigError( - f"Grouping not available: {grouping}, did you spell it " "correctly?" - ) + raise ConfigError(f"Grouping not available: {grouping}, did you spell it " "correctly?") return grouping def parse_norm_threshold(self, val: (numbers.Number, type(None))): @@ -1328,9 +1280,7 @@ def load_default_default_filter_rules(self, spec): lock_token = "_filters.lock.yaml" try: - return yaml.safe_load( - read_text(validphys.cuts.lockfiles, f"{spec}{lock_token}") - ) + return yaml.safe_load(read_text(validphys.cuts.lockfiles, f"{spec}{lock_token}")) except FileNotFoundError as e: alternatives = [ el.strip(lock_token) @@ -1368,11 +1318,7 @@ def produce_rules( ): """Produce filter rules based on the user defined input and defaults.""" - from validphys.filters import ( - Rule, - RuleProcessingError, - default_filter_rules_input, - ) + from validphys.filters import Rule, RuleProcessingError, default_filter_rules_input theory_parameters = theoryid.get_description() @@ -1409,9 +1355,7 @@ def load_default_default_filter_settings(self, spec): lock_token = "_defaults.lock.yaml" try: - return yaml.safe_load( - read_text(validphys.cuts.lockfiles, f"{spec}{lock_token}") - ) + return yaml.safe_load(read_text(validphys.cuts.lockfiles, f"{spec}{lock_token}")) except FileNotFoundError as e: alternatives = alternatives = [ el.strip(lock_token) @@ -1447,23 +1391,13 @@ def produce_defaults( """ from validphys.filters import default_filter_settings_input - if ( - q2min is not None - and "q2min" in filter_defaults - and q2min != filter_defaults["q2min"] - ): + if q2min is not None and "q2min" in filter_defaults and q2min != filter_defaults["q2min"]: raise ConfigError("q2min defined multiple times with different values") - if ( - w2min is not None - and "w2min" in filter_defaults - and w2min != filter_defaults["w2min"] - ): + if w2min is not None and "w2min" in filter_defaults and w2min != filter_defaults["w2min"]: raise ConfigError("w2min defined multiple times with different values") if default_filter_settings_recorded_spec_ is not None: - filter_defaults = default_filter_settings_recorded_spec_[ - default_filter_settings - ] + filter_defaults = default_filter_settings_recorded_spec_[default_filter_settings] # If we find recorded specs return immediately and don't read q2min and w2min # from runcard return filter_defaults @@ -1484,7 +1418,10 @@ def produce_defaults( return filter_defaults def produce_data( - self, data_input, *, group_name="data", + self, + data_input, + *, + group_name="data", ): """A set of datasets where correlated systematics are taken into account @@ -1541,15 +1478,9 @@ def _parse_data_input_from_( ) # We need to make theoryid available if using experiments try: - _, experiments = self.parse_from_( - parse_from_value, data_key, write=False - ) + _, experiments = self.parse_from_(parse_from_value, data_key, write=False) data_val = NSList( - [ - dsinput - for experiment in experiments - for dsinput in experiment.dsinputs - ], + [dsinput for experiment in experiments for dsinput in experiment.dsinputs], nskey="dataset_input", ) except ConfigError as inner_error: @@ -1586,22 +1517,23 @@ def load_default_data_grouping(self, spec): """Load the default grouping of data""" # slightly superfluous, only one default at present but perhaps # somebody will want to add to this at some point e.g for th. uncertainties - allowed = { - "standard_report": "experiment", - "thcovmat_fit": "ALL" - } + allowed = {"standard_report": "experiment", "thcovmat_fit": "ALL"} return allowed[spec] def produce_processed_data_grouping( - self, use_thcovmat_in_fitting=False, use_thcovmat_in_sampling=False, data_grouping=None, data_grouping_recorded_spec_=None + self, + use_thcovmat_in_fitting=False, + use_thcovmat_in_sampling=False, + data_grouping=None, + data_grouping_recorded_spec_=None, ): """Process the data_grouping key from the runcard, or lockfile. If `data_grouping_recorded_spec_` is present then its value is taken, and the runcard is assumed to be a lockfile. If data_grouping is None, then, if either use_thcovmat_in_fitting or use_thcovmat_in_sampling - (or both) are true (which means that the fit is a thcovmat fit), group all the datasets - together, otherwise fall back to the default behaviour of grouping by + (or both) are true (which means that the fit is a thcovmat fit), group all the datasets + together, otherwise fall back to the default behaviour of grouping by experiment (called standard_report). Else, the user can specfiy their own grouping, for example metadata_process. @@ -1615,9 +1547,7 @@ def produce_processed_data_grouping( return data_grouping_recorded_spec_[data_grouping] return self.load_default_data_grouping(data_grouping) - def produce_processed_metadata_group( - self, processed_data_grouping, metadata_group=None - ): + def produce_processed_metadata_group(self, processed_data_grouping, metadata_group=None): """Expose the final data grouping result. Either metadata_group is specified by user, in which case uses `processed_data_grouping` which is experiment by default. @@ -1627,7 +1557,9 @@ def produce_processed_metadata_group( return metadata_group def produce_group_dataset_inputs_by_metadata( - self, data_input, processed_metadata_group, + self, + data_input, + processed_metadata_group, ): """Take the data and the processed_metadata_group key and attempt to group the data, returns a list where each element specifies the data_input @@ -1681,28 +1613,21 @@ def produce_group_dataset_inputs_by_experiment(self, data_input): return self.produce_group_dataset_inputs_by_metadata(data_input, "experiment") def produce_group_dataset_inputs_by_process(self, data_input): - return self.produce_group_dataset_inputs_by_metadata( - data_input, "nnpdf31_process" - ) + return self.produce_group_dataset_inputs_by_metadata(data_input, "nnpdf31_process") def produce_scale_variation_theories(self, theoryid, point_prescription): """Produces a list of theoryids given a theoryid at central scales and a point - prescription. The options for the latter are '3 point', '5 point', '5bar point', '7 point' - and '9 point'. Note that these are defined in arXiv:1906.10698. This hard codes the - theories needed for each prescription to avoid user error.""" + prescription. The options for the latter are '3 point', '5 point', '5bar point', '7 point' + and '9 point'. Note that these are defined in arXiv:1906.10698. This hard codes the + theories needed for each prescription to avoid user error.""" pp = point_prescription th = theoryid.id - lsv = yaml.safe_load( - read_text(validphys.scalevariations, "scalevariationtheoryids.yaml") - ) + lsv = yaml.safe_load(read_text(validphys.scalevariations, "scalevariationtheoryids.yaml")) scalevarsfor_list = lsv["scale_variations_for"] # Allowed central theoryids - cent_thids = [ - str(scalevarsfor_dict["theoryid"]) - for scalevarsfor_dict in scalevarsfor_list - ] + cent_thids = [str(scalevarsfor_dict["theoryid"]) for scalevarsfor_dict in scalevarsfor_list] if th not in cent_thids: valid_thids = ", ".join(cent_thids) raise ConfigError( diff --git a/validphys2/src/validphys/filters.py b/validphys2/src/validphys/filters.py index a121b0def3..91c80ce2fb 100644 --- a/validphys2/src/validphys/filters.py +++ b/validphys2/src/validphys/filters.py @@ -2,20 +2,18 @@ Filters for NNPDF fits """ -import logging -import re from collections.abc import Mapping from importlib.resources import read_text +import logging +import re import numpy as np from reportengine.checks import check, make_check from reportengine.compat import yaml +from validphys.commondatawriter import write_commondata_to_file, write_systype_to_file import validphys.cuts -from validphys.commondatawriter import ( - write_commondata_to_file, - write_systype_to_file, - ) + log = logging.getLogger(__name__) KIN_LABEL = { @@ -53,6 +51,7 @@ class BadPerturbativeOrder(ValueError): """Exception raised when the perturbative order string is not recognized.""" + class MissingRuleAttribute(RuleProcessingError, AttributeError): """Exception raised when a rule is missing required attributes.""" @@ -75,13 +74,14 @@ def default_filter_rules_input(): return yaml.safe_load(read_text(validphys.cuts, "filters.yaml")) - def check_nonnegative(var: str): """Ensure that `var` is positive""" + @make_check def run_check(ns, **kwargs): val = ns[var] check(val >= 0, f"'{var}' must be positive or equal zero, but it is {val!r}.") + return run_check @@ -124,9 +124,7 @@ def filter_closure_data_by_experiment( res = [] for exp in experiments_data: - experiment_index = experiments_index[ - experiments_index.isin([exp.name], level=0) - ] + experiment_index = experiments_index[experiments_index.isin([exp.name], level=0)] res.append( _filter_closure_data( filter_path, exp, fakepdf, fakenoise, filterseed, experiment_index, sep_mult @@ -137,8 +135,7 @@ def filter_closure_data_by_experiment( def filter_real_data(filter_path, data): - """Filter real data, cutting any points which do not pass the filter rules. - """ + """Filter real data, cutting any points which do not pass the filter rules.""" log.info('Filtering real data.') return _filter_real_data(filter_path, data) @@ -158,8 +155,9 @@ def _write_ds_cut_data(path, dataset): log.info("All {all_ndata} points in in {dataset.name} passed kinematic cuts.") else: filtered_dsndata = len(datamask) - log.info(f"{len(datamask)}/{all_dsndata} datapoints " - f"in {dataset.name} passed kinematic cuts.") + log.info( + f"{len(datamask)}/{all_dsndata} datapoints " f"in {dataset.name} passed kinematic cuts." + ) # save to disk if datamask is not None: export_mask(path / f'FKMASK_{dataset.name}.dat', datamask) @@ -181,8 +179,7 @@ def _filter_real_data(filter_path, data): def _filter_closure_data( - filter_path, data, fakepdf, fakenoise, filterseed, experiments_index - , sep_mult + filter_path, data, fakepdf, fakenoise, filterseed, experiments_index, sep_mult ): """ This function is accessed within a closure test only, that is, the fakedata @@ -234,25 +231,19 @@ def _filter_closure_data( closure_data = level0_commondata_wc(data, fakepdf) for dataset in data.datasets: - #== print number of points passing cuts, make dataset directory and write FKMASK ==# + # == print number of points passing cuts, make dataset directory and write FKMASK ==# path = filter_path / dataset.name nfull, ncut = _write_ds_cut_data(path, dataset) make_dataset_dir(path / "systypes") total_data_points += nfull total_cut_data_points += ncut - + if fakenoise: - #======= Level 1 closure test =======# - - closure_data = make_level1_data( - data, - closure_data, - filterseed, - experiments_index, - sep_mult - ) + # ======= Level 1 closure test =======# + + closure_data = make_level1_data(data, closure_data, filterseed, experiments_index, sep_mult) - #====== write commondata and systype files ======# + # ====== write commondata and systype files ======# if fakenoise: log.info("Writing Level1 data") else: @@ -260,12 +251,7 @@ def _filter_closure_data( for cd in closure_data: path_cd = filter_path / cd.setname / f"DATA_{cd.setname}.dat" - path_sys = ( - filter_path - / cd.setname - / "systypes" - / f"SYSTYPE_{cd.setname}_DEFAULT.dat" - ) + path_sys = filter_path / cd.setname / "systypes" / f"SYSTYPE_{cd.setname}_DEFAULT.dat" write_commondata_to_file(commondata=cd, path=path_cd) write_systype_to_file(commondata=cd, path=path_sys) @@ -277,16 +263,19 @@ def check_t0pdfset(t0pdfset): t0pdfset.load() log.info(f'{t0pdfset} T0 checked.') + def check_luxset(luxset): """Lux pdf check""" luxset.load() log.info(f'{luxset} Lux pdf checked.') + def check_additional_errors(additional_errors): """Lux additional errors pdf check""" additional_errors.load() log.info(f'{additional_errors} Lux additional errors pdf checked.') + def check_positivity(posdatasets): """Verify positive datasets are ready for the fit.""" log.info('Verifying positivity tables:') @@ -294,6 +283,7 @@ def check_positivity(posdatasets): pos.load_commondata() log.info(f'{pos.name} checked.') + def check_integrability(integdatasets): """Verify positive datasets are ready for the fit.""" log.info('Verifying integrability tables:') @@ -301,6 +291,7 @@ def check_integrability(integdatasets): integ.load_commondata() log.info(f'{integ.name} checked.') + class PerturbativeOrder: """Class that conveniently handles perturbative order declarations for use @@ -371,6 +362,7 @@ def __contains__(self, i): else: return i == self.numeric_pto + class Rule: """Rule object to be used to generate cuts mask. @@ -419,9 +411,7 @@ def __init__( raise MissingRuleAttribute("No rule defined.") if self.dataset is None and self.process_type is None: - raise MissingRuleAttribute( - "Please define either a process type or dataset." - ) + raise MissingRuleAttribute("Please define either a process type or dataset.") if self.process_type is None: from validphys.loader import Loader, LoaderError @@ -431,9 +421,7 @@ def __init__( try: cd = loader.check_commondata(self.dataset) except LoaderError as e: - raise RuleProcessingError( - f"Could not find dataset {self.dataset}" - ) from e + raise RuleProcessingError(f"Could not find dataset {self.dataset}") from e if cd.process_type[:3] == "DIS": self.variables = KIN_LABEL["DIS"] else: @@ -454,9 +442,7 @@ def __init__( if hasattr(self, "PTO"): if not isinstance(self.PTO, str): - raise RuleProcessingError( - f"Expecting PTO to be a string, not {type(self.PTO)}." - ) + raise RuleProcessingError(f"Expecting PTO to be a string, not {type(self.PTO)}.") try: self.PTO = PerturbativeOrder(self.PTO) except BadPerturbativeOrder as e: @@ -491,9 +477,7 @@ def __init__( try: self.rule = compile(self.rule, "rule", "eval") except Exception as e: - raise RuleProcessingError( - f"Could not process rule {self.rule_string!r}: {e}" - ) from e + raise RuleProcessingError(f"Could not process rule {self.rule_string!r}: {e}") from e for name in self.rule.co_names: if name not in ns: raise RuleProcessingError( @@ -536,9 +520,7 @@ def __call__(self, dataset, idat): if k == "PTO" and hasattr(self, "PTO"): if v not in self.PTO: return None - elif hasattr(self, k) and ( - getattr(self, k) != v - ): + elif hasattr(self, k) and (getattr(self, k) != v): return None # Will return True if datapoint passes through the filter @@ -552,12 +534,10 @@ def __call__(self, dataset, idat): **ns, }, ) - except Exception as e: # pragma: no cover - raise FatalRuleError( - f"Error when applying rule {self.rule_string!r}: {e}" - ) from e + except Exception as e: # pragma: no cover + raise FatalRuleError(f"Error when applying rule {self.rule_string!r}: {e}") from e - def __repr__(self): # pragma: no cover + def __repr__(self): # pragma: no cover return self.rule_string def _make_kinematics_dict(self, dataset, idat) -> dict: @@ -574,6 +554,7 @@ def _make_point_namespace(self, dataset, idat) -> dict: ns[key] = eval(value, {**self.numpy_functions, **ns}) return ns + def get_cuts_for_dataset(commondata, rules) -> list: """Function to generate a list containing the index of all experimental points that passed kinematic diff --git a/validphys2/src/validphys/n3fit_data.py b/validphys2/src/validphys/n3fit_data.py index 2019aadc88..38f946248d 100644 --- a/validphys2/src/validphys/n3fit_data.py +++ b/validphys2/src/validphys/n3fit_data.py @@ -4,8 +4,8 @@ Providers which prepare the data ready for :py:func:`n3fit.performfit.performfit`. """ -import functools from collections import defaultdict +import functools import hashlib import logging @@ -14,11 +14,8 @@ from reportengine import collect from reportengine.table import table - -from validphys.n3fit_data_utils import ( - validphys_group_extractor, -) from validphys.core import IntegrabilitySetSpec, TupleComp +from validphys.n3fit_data_utils import validphys_group_extractor log = logging.getLogger(__name__) @@ -52,6 +49,7 @@ def replica_mcseed(replica, mcseed, genrep): res = np.random.randint(0, pow(2, 31)) return res + def replica_luxseed(replica, luxseed): """Generate the ``luxseed`` for a ``replica``. Identical to replica_nnseed but used for a different purpose. @@ -97,13 +95,11 @@ def tr_masks(data, replica_trvlseed): ndata = len(cuts.load()) if cuts else dataset.commondata.ndata frac = dataset.frac # We do this so that a given dataset will always have the same number of points masked - trmax = int(ndata*frac) + trmax = int(ndata * frac) if trmax == 0: # If that number is 0, then get 1 point with probability frac trmax = int(rng.random() < frac) - mask = np.concatenate( - [np.ones(trmax, dtype=bool), np.zeros(ndata - trmax, dtype=bool)] - ) + mask = np.concatenate([np.ones(trmax, dtype=bool), np.zeros(ndata - trmax, dtype=bool)]) rng.shuffle(mask) trmask_partial.append(mask) return _TrMasks(str(data), replica_trvlseed, trmask_partial) @@ -242,7 +238,7 @@ def fitting_data_dict( expdata = make_replica tr_masks = tr_masks.masks - covmat = dataset_inputs_fitting_covmat # t0 covmat, or theory covmat or whatever was decided by the runcard + covmat = dataset_inputs_fitting_covmat # t0 covmat, or theory covmat or whatever was decided by the runcard inv_true = np.linalg.inv(covmat) fittable_datasets = fittable_datasets_masked @@ -351,7 +347,7 @@ def pseudodata_table(groups_replicas_indexed_make_replica, replicas): Notes ----- Whilst running ``n3fit``, this action will only be called if - `fitting::savepseudodata` is `true` (as per the default setting) and + `fitting::savepseudodata` is `true` (as per the default setting) and replicas are fitted one at a time. The table can be found in the replica folder i.e. /nnfit/replica_*/ diff --git a/validphys2/src/validphys/photon/compute.py b/validphys2/src/validphys/photon/compute.py index 5fd5814d98..bbbdfc650f 100644 --- a/validphys2/src/validphys/photon/compute.py +++ b/validphys2/src/validphys/photon/compute.py @@ -4,13 +4,13 @@ import fiatlux import numpy as np -import yaml -from eko.io import EKO from scipy.integrate import trapezoid from scipy.interpolate import interp1d -from validphys.n3fit_data import replica_luxseed +import yaml +from eko.io import EKO from n3fit.io.writer import XGRID +from validphys.n3fit_data import replica_luxseed from . import structure_functions as sf from .constants import ED2, EU2, NC, NL @@ -54,9 +54,7 @@ class Photon: def __init__(self, theoryid, lux_params, replicas): theory = theoryid.get_description() fiatlux_runcard = FIATLUX_DEFAULT - fiatlux_runcard["qed_running"] = bool( - np.isclose(theory["Qedref"], theory["Qref"]) - ) + fiatlux_runcard["qed_running"] = bool(np.isclose(theory["Qedref"], theory["Qref"])) # cast explicitly from np.bool_ to bool otherwise problems in dumping it # TODO: for the time being, we trigger alphaem running if Qedref=Qref. # This is going to be changed in favor of a bool em_running @@ -87,8 +85,10 @@ def __init__(self, theoryid, lux_params, replicas): for replica in replicas: f2 = sf.InterpStructureFunction(path_to_F2, self.luxpdfset.members[replica]) fl = sf.InterpStructureFunction(path_to_FL, self.luxpdfset.members[replica]) - if not np.isclose(f2.q2_max, fl.q2_max) : - log.error("FKtables for fiatlux_dis_F2 and fiatlux_dis_FL have two different q2_max") + if not np.isclose(f2.q2_max, fl.q2_max): + log.error( + "FKtables for fiatlux_dis_F2 and fiatlux_dis_FL have two different q2_max" + ) fiatlux_runcard["q2_max"] = float(f2.q2_max) f2lo = sf.F2LO(self.luxpdfset.members[replica], theory) @@ -110,9 +110,7 @@ def __init__(self, theoryid, lux_params, replicas): self.lux[replica].PlugStructureFunctions(f2.fxq, fl.fxq, f2lo.fxq) photon_array = self.compute_photon_array(replica) - self.interpolator.append( - interp1d(XGRID, photon_array, fill_value=0.0, kind="cubic") - ) + self.interpolator.append(interp1d(XGRID, photon_array, fill_value=0.0, kind="cubic")) self.integral.append(trapezoid(photon_array, XGRID)) def compute_photon_array(self, replica): @@ -131,9 +129,7 @@ def compute_photon_array(self, replica): """ # Compute photon PDF log.info(f"Computing photon") - photon_qin = np.array( - [self.lux[replica].EvaluatePhoton(x, Q_IN**2).total for x in XGRID] - ) + photon_qin = np.array([self.lux[replica].EvaluatePhoton(x, Q_IN**2).total for x in XGRID]) photon_qin += self.generate_errors(replica) # fiatlux computes x * gamma(x) photon_qin /= XGRID @@ -307,9 +303,7 @@ def set_thresholds_alpha_em(self): nfref = 6 thresh_list.insert(nfref - 3, self.qref) - thresh = { - nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1) - } + thresh = {nf: thresh_list[nf - 3] for nf in range(3, self.theory["MaxNfAs"] + 1)} alpha_thresh = {nfref: self.alpha_em_ref} @@ -335,9 +329,6 @@ def set_betas(self): nd = nf - nu beta0[nf] = (-4.0 / 3 * (NL + NC * (nu * EU2 + nd * ED2))) / (4 * np.pi) b1[nf] = ( - -4.0 - * (NL + NC * (nu * EU2**2 + nd * ED2**2)) - / beta0[nf] - / (4 * np.pi) ** 2 + -4.0 * (NL + NC * (nu * EU2**2 + nd * ED2**2)) / beta0[nf] / (4 * np.pi) ** 2 ) return beta0, b1 From f6f211e878c0d340b3092497f7d7e3d5774bdb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 17:00:20 +0200 Subject: [PATCH 200/204] Add run-qed-fit.rst --- doc/sphinx/source/tutorials/run-qed-fit.rst | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 doc/sphinx/source/tutorials/run-qed-fit.rst diff --git a/doc/sphinx/source/tutorials/run-qed-fit.rst b/doc/sphinx/source/tutorials/run-qed-fit.rst new file mode 100644 index 0000000000..9eb8796f2b --- /dev/null +++ b/doc/sphinx/source/tutorials/run-qed-fit.rst @@ -0,0 +1,30 @@ +.. _run-qed-fit: + +========================== +How to run a QED fit +========================== + +It is possible to perform a QED fit adding the key `fiatlux` to the runcard. In this way +a photon PDF will be generated using the `FiatLux` public library that implements the `LuxQED` +(see :cite:p:`Manohar:2016nzj` and :cite:p:`Manohar:2017eqh`) approach. +The parameters to be added are the following: + +.. code-block:: yaml + + fiatlux: + luxset: NNPDF40_nnlo_as_01180 + additional_errors: true + luxseed: 1234567890 + +`luxset` is the PDF set used to generate the photon PDF with `FiatLux `. +The code wil generate as much photon replicas as the number of replicas contained in the `luxset`. Therefore, if the user +tries to generate a replica with ID higher than the maximum ID of the `luxset`, the code will +raise an error. Moreover, being the `LuxQED` approach an iterated prcedure, and given that some replicas +do not pass the `postfit` selection criteria, the user should make sure that the number of replicas in +the `luxset` is high enough such that in the final iteration there will be a number of replicas +higher than the final replicas desired. +`additional_errors` is the parameter that switches on and off the additional errors of the `LuxQED` approach, +while `luxseed` is the seed used to generate such errors. +This errors should be switched on only in the very last iteration of the procedure. + +Whenever the photon PDF is generated, it will remain constant during the fit and will enter in the `MSR`. From ffce850f23767a71c9262d96c5dc1da564dd0cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Wed, 17 May 2023 17:34:07 +0200 Subject: [PATCH 201/204] Update n3fit/src/n3fit/scripts/n3fit_exec.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/scripts/n3fit_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 01b21b682d..cf6b276037 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -120,7 +120,7 @@ def from_yaml(cls, o, *args, **kwargs): raise ConfigError(f"Failed to parse yaml file: {e}") if not isinstance(file_content, dict): raise ConfigError( - f"Expecting input runcard to be a mapping, " f"not '{type(file_content)}'." + f"Expecting input runcard to be a mapping, not '{type(file_content)}'." ) if file_content.get('closuretest') is not None: From e778bd6b4b94117909f5df2690eabb55575e74f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Wed, 17 May 2023 21:51:14 +0200 Subject: [PATCH 202/204] Fix tests --- validphys2/src/validphys/tests/photon/test_compute.py | 1 + 1 file changed, 1 insertion(+) diff --git a/validphys2/src/validphys/tests/photon/test_compute.py b/validphys2/src/validphys/tests/photon/test_compute.py index 8928a4c28a..30805be889 100644 --- a/validphys2/src/validphys/tests/photon/test_compute.py +++ b/validphys2/src/validphys/tests/photon/test_compute.py @@ -73,6 +73,7 @@ class FakeStructureFunction: def __init__(self, path, pdfs): self.path = path self.pdfs = pdfs + self.q2_max = 1e8 def fxq(self): return 0 From 0ba508c8feb07b394c28e0e1bdd4f9c3b3a2bdde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= Date: Thu, 18 May 2023 15:38:12 +0200 Subject: [PATCH 203/204] Fix evolven3fit_new bug --- n3fit/src/evolven3fit_new/evolve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/evolven3fit_new/evolve.py b/n3fit/src/evolven3fit_new/evolve.py index 033989a776..08f139e8e7 100644 --- a/n3fit/src/evolven3fit_new/evolve.py +++ b/n3fit/src/evolven3fit_new/evolve.py @@ -68,7 +68,7 @@ def evolve_fit( stdout_log = logging.StreamHandler(sys.stdout) for log in [log_file, stdout_log]: log.setLevel(LOGGING_SETTINGS["level"]) - log.setFormatter(logging.Formatter(LOGGING_SETTINGS["formatter"])) + log.setFormatter(LOGGING_SETTINGS["formatter"]) for logger in (_logger, *[logging.getLogger("eko")]): logger.handlers = [] logger.setLevel(LOGGING_SETTINGS["level"]) From c092d2509d399eb36f4fd7668ffd2f9300c4d972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niccol=C3=B2=20Laurenti?= <57321974+niclaurenti@users.noreply.github.com> Date: Fri, 19 May 2023 11:20:34 +0200 Subject: [PATCH 204/204] Update doc/sphinx/source/tutorials/run-qed-fit.rst Co-authored-by: Alessandro Candido --- doc/sphinx/source/tutorials/run-qed-fit.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/tutorials/run-qed-fit.rst b/doc/sphinx/source/tutorials/run-qed-fit.rst index 9eb8796f2b..051694cc8d 100644 --- a/doc/sphinx/source/tutorials/run-qed-fit.rst +++ b/doc/sphinx/source/tutorials/run-qed-fit.rst @@ -17,7 +17,7 @@ The parameters to be added are the following: luxseed: 1234567890 `luxset` is the PDF set used to generate the photon PDF with `FiatLux `. -The code wil generate as much photon replicas as the number of replicas contained in the `luxset`. Therefore, if the user +The code will generate as many photon replicas as the number of replicas contained in the `luxset`. Therefore, if the user tries to generate a replica with ID higher than the maximum ID of the `luxset`, the code will raise an error. Moreover, being the `LuxQED` approach an iterated prcedure, and given that some replicas do not pass the `postfit` selection criteria, the user should make sure that the number of replicas in