From 5c1a820b8ba6aecb0d8414a5bda6707956065e62 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Wed, 10 Apr 2019 17:46:35 +0100 Subject: [PATCH 01/13] extremely primitive version of grouping datasets by experiment label and being able to calculate chi2 with thcovmat --- validphys2/src/validphys/config.py | 30 ++++++++++++++++++++++ validphys2/src/validphys/results.py | 39 +++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index 9a21bf9de4..b1b438a32b 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -29,6 +29,8 @@ from validphys.paramfits.config import ParamfitsConfig +from validphys.plotoptions import get_info + log = logging.getLogger(__name__) @@ -744,6 +746,34 @@ def parse_fitdeclaration(self, label:str): """ return label + def produce_experiments_from_plotting(self, fit): + """Used to produce experiments from the fit, where each experiment is a group of datasets + according to the plotting info + """ + #TODO: consider this an implimentation detail + from reportengine.namespaces import NSList + + with self.set_context(ns=self._curr_ns.new_child({'fit':fit})): + _, experiments = self.parse_from_('fit', 'experiments', write=False) + + flat = (ds for exp in experiments for ds in exp.datasets) + metaexps = [get_info(ds).experiment for ds in flat] + res = {} + for exp in experiments: + for dsinput, ds in zip(exp.dsinputs, exp.datasets): + metaexp = get_info(ds).experiment + if metaexp in res: + res[metaexp].append(ds) + else: + res[metaexp] = [ds] + exps = [] + for exp in res: + exps.append(ExperimentSpec(exp, res[exp])) + + experiments = NSList(exps, nskey='experiment') + return {'experiments': experiments} + + class Config(report.Config, CoreConfig, ParamfitsConfig): """The effective configuration parser class.""" pass diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index 49429a6602..eab04bfb35 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -65,7 +65,13 @@ def __init__(self, dataobj, thcovmat=False): if isinstance(thcovmat, ThCovMatSpec): self._sqrtcovmat = dataobj.GetSqrtFitCovMat(str(thcovmat), []) elif isinstance(thcovmat, pd.DataFrame): - thcovslice = get_df_block(thcovmat, dataobj.GetSetName(), level=1) + if isinstance(dataobj, Experiment): + level = 0 + name = dataobj.GetExpName() + else: + level = 1 + name = dataobj.GetSetName() + thcovslice = get_df_block(thcovmat, name, level=level) self._sqrtcovmat = np.linalg.cholesky(self._covmat + thcovslice) else: self._sqrtcovmat = dataobj.get_sqrtcovmat() @@ -364,7 +370,7 @@ def closure_pseudodata_replicas(experiments, pdf, nclosure:int, return df -def results(dataset:(DataSetSpec), pdf:PDF, t0set:(PDF, type(None))=None, fitthcovmat=False): +def results(dataset:(DataSetSpec), pdf:PDF, t0set:(PDF, type(None))=None, fixup_fitthcovmat=False): """Tuple of data and theory results for a single pdf. The theory is specified as part of the dataset. An experiment is also allowed. @@ -372,9 +378,6 @@ def results(dataset:(DataSetSpec), pdf:PDF, t0set:(PDF, type(None))=None, fitthc data = dataset.load() - # only need to load thcov if specified and if dataset - if fitthcovmat and isinstance(dataset, DataSetSpec): - fitthcovmat = fitthcovmat.load() if t0set: #Copy data to avoid chaos @@ -382,12 +385,12 @@ def results(dataset:(DataSetSpec), pdf:PDF, t0set:(PDF, type(None))=None, fitthc log.debug("Setting T0 predictions for %s" % dataset) data.SetT0(t0set.load_t0()) - return (DataResult(data, thcovmat=fitthcovmat), + return (DataResult(data, thcovmat=fixup_fitthcovmat), ThPredictionsResult.from_convolution(pdf, dataset, loaded_data=data)) -def experiment_results(experiment, pdf:PDF, t0set:(PDF, type(None))=None, fitthcovmat=False): +def experiment_results(experiment, pdf:PDF, t0set:(PDF, type(None))=None, fixup_fitthcovmat=False): """Like `results` but for a whole experiment""" - return results(experiment, pdf, t0set, fitthcovmat=fitthcovmat) + return results(experiment, pdf, t0set, fixup_fitthcovmat=fixup_fitthcovmat) #It's better to duplicate a few lines than to complicate the logic of #``results`` to support this. @@ -916,3 +919,23 @@ def experiments_central_values(experiment_result_table): dataspecs_commondata = collect('commondata', ('dataspecs',)) dataspecs_pdf = collect('pdf', ('dataspecs',)) dataspecs_fit = collect('fit', ('dataspecs',)) + +def fixup_fitthcovmat(fitthcovmat, experiments=None): + """If both `fitthcovmat` and `experiments` exist then change the level 1 keys of `fitthcovmat` + to match `experiments` + """ + if fitthcovmat: + loaded_covmat = fitthcovmat.load() + if experiments: + exp_names = np.array(loaded_covmat.index.get_level_values(0), copy=True) + ds_names = loaded_covmat.index.get_level_values(1) + ds_index = loaded_covmat.index.get_level_values(2) + for exp in experiments: + for ds in exp.datasets: + choose = np.where(ds_names == ds.name) + exp_names[choose] = exp.name + new_index = pd.MultiIndex.from_arrays([exp_names, ds_names, ds_index]) + loaded_covmat = pd.DataFrame(loaded_covmat.values, index=new_index, columns=new_index) + return loaded_covmat + else: + return fitthcovmat From 865d33bed9501f10a3024b6615a8db03be8d0c70 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 12 Apr 2019 10:47:37 +0100 Subject: [PATCH 02/13] Added necessary providers for experiments by plotting plots to exist in vp-comparefits. Changed templates accordingly and updated documentation. --- .../comparefittemplates/comparecard.yaml | 10 +- .../validphys/comparefittemplates/report.md | 7 +- validphys2/src/validphys/config.py | 14 +- validphys2/src/validphys/dataplots.py | 24 ++- validphys2/src/validphys/fitdata.py | 2 +- validphys2/src/validphys/results.py | 166 ++++++++++++------ 6 files changed, 151 insertions(+), 72 deletions(-) diff --git a/validphys2/src/validphys/comparefittemplates/comparecard.yaml b/validphys2/src/validphys/comparefittemplates/comparecard.yaml index 3678b0acab..adf2b71be1 100644 --- a/validphys2/src/validphys/comparefittemplates/comparecard.yaml +++ b/validphys2/src/validphys/comparefittemplates/comparecard.yaml @@ -74,14 +74,14 @@ pdf_report: template: report.md -experiments: - from_: fit positivity: from_: fit dataspecs: - - theoryid: + - experiments: + from_: fit + theoryid: from_: current pdf: from_: current @@ -91,7 +91,9 @@ dataspecs: from_: current - - theoryid: + - experiments: + from_: fit + theoryid: from_: reference pdf: from_: reference diff --git a/validphys2/src/validphys/comparefittemplates/report.md b/validphys2/src/validphys/comparefittemplates/report.md index f1c3cc72ed..08d7bb89ba 100644 --- a/validphys2/src/validphys/comparefittemplates/report.md +++ b/validphys2/src/validphys/comparefittemplates/report.md @@ -33,7 +33,7 @@ Training validation $\chi^2$ by experiment ---------------------- -{@plot_fits_experiments_chi2@} +{@plot_fits_exp_by_plotting_chi2@} $\chi^2$ by dataset comparisons ------------------------------- @@ -44,10 +44,7 @@ $\chi^2$ by dataset comparisons $\phi$ by experiment -------------------- -{@with dataspecs@} -### {@fit@} -{@plot_phi@} -{@endwith@} +{@plot_phi_fits_exp_by_plotting@} Experiment plots --------------- diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index b1b438a32b..f6d7dbeaee 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -748,7 +748,7 @@ def parse_fitdeclaration(self, label:str): def produce_experiments_from_plotting(self, fit): """Used to produce experiments from the fit, where each experiment is a group of datasets - according to the plotting info + according to the experiment key in the plotting info file """ #TODO: consider this an implimentation detail from reportengine.namespaces import NSList @@ -773,6 +773,18 @@ def produce_experiments_from_plotting(self, fit): experiments = NSList(exps, nskey='experiment') return {'experiments': experiments} + def produce_experiments_from_plotting_withcontext(self, fit): + """produces experiments similarly to `experiments_from_plotting` but also sets fitcontext + (pdf and theoryid) + """ + _, pdf = self.parse_from_('fit', 'pdf', write=False) + _, theory = self.parse_from_('fit', 'theory', write=False) + thid = theory['theoryid'] + with self.set_context(ns=self._curr_ns.new_child({'theoryid':thid})): + experiments = self.produce_experiments_from_plotting(fit)['experiments'] + return {'pdf': pdf, 'theoryid':thid, 'experiments': experiments} + + class Config(report.Config, CoreConfig, ParamfitsConfig): """The effective configuration parser class.""" diff --git a/validphys2/src/validphys/dataplots.py b/validphys2/src/validphys/dataplots.py index c930772fb6..632ee51ee7 100644 --- a/validphys2/src/validphys/dataplots.py +++ b/validphys2/src/validphys/dataplots.py @@ -54,21 +54,19 @@ def plot_phi(experiments, experiments_phi): See `phi_data` for information on how phi is calculated """ - phi = experiments_phi + phi = [exp_phi for (exp_phi, npoints) in experiments_phi] xticks = [experiment.name for experiment in experiments] fig, ax = plotutils.barplot(phi, collabels=xticks, datalabels=[r'$\phi$']) ax.set_title(r"$\phi$ for each experiment") return fig @figure -def plot_phi_pdfs(experiments, pdfs, experiments_pdfs_phi): - """Like `plot_phi` but plots a set of bars for each PDF input""" - phi = experiments_pdfs_phi - phi_labels = [pdf.name for pdf in pdfs] - xticks = [experiment.name for experiment in experiments] - fig, ax = plotutils.barplot(phi, collabels=xticks, datalabels=phi_labels) +def plot_phi_fits_exp_by_plotting(fits_phi_table): + """Plots a set of bars for each fit, each bar represents the value of phi for the corresponding + experiment, where the experiment is a group of datasets according to the `experiment` key in + the plotting info file""" + fig, ax = _plot_chis_df(fits_phi_table) ax.set_title(r"$\phi$ for each experiment") - ax.legend() return fig @figure @@ -533,6 +531,16 @@ def plot_fits_experiments_chi2(fits_experiments_chi2_table): ax.set_title(r"$\chi^2$ for experiments") return fig +@figure +def plot_fits_exp_by_plotting_chi2(fits_exp_by_plotting_chi2_table): + """Generate a plot equivalent to ``plot_experiments_chi2`` using all fits + and grouping datasets into experiments according to the `experiment` key defined in the + plotting file. + """ + fig, ax = _plot_chis_df(fits_exp_by_plotting_chi2_table) + ax.set_title(r"$\chi^2$ for experiments") + return fig + @figure def plot_dataspecs_experiments_chi2(dataspecs_experiments_chi2_table): """Same as plot_fits_experiments_chi2 but for arbitrary dataspecs""" diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index 98fd20e8c5..9f63b10420 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -146,7 +146,7 @@ def fit_summary(fit, replica_data, total_experiments_chi2data): etrain = [x.training for x in replica_data] evalid = [x.validation for x in replica_data] - phi = phi_data(total_experiments_chi2data) + phi, _ = phi_data(total_experiments_chi2data) phi_err = np.std(member_chi2)/(2.0*phi*np.sqrt(nrep)) VET = ValueErrorTuple diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index eab04bfb35..da29f28f76 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -59,22 +59,10 @@ def std_error(self): class DataResult(NNPDFDataResult): - def __init__(self, dataobj, thcovmat=False): + def __init__(self, dataobj, covmat, sqrtcovmat): super().__init__(dataobj) - self._covmat = dataobj.get_covmat() - if isinstance(thcovmat, ThCovMatSpec): - self._sqrtcovmat = dataobj.GetSqrtFitCovMat(str(thcovmat), []) - elif isinstance(thcovmat, pd.DataFrame): - if isinstance(dataobj, Experiment): - level = 0 - name = dataobj.GetExpName() - else: - level = 1 - name = dataobj.GetSetName() - thcovslice = get_df_block(thcovmat, name, level=level) - self._sqrtcovmat = np.linalg.cholesky(self._covmat + thcovslice) - else: - self._sqrtcovmat = dataobj.get_sqrtcovmat() + self._covmat = covmat + self._sqrtcovmat = sqrtcovmat @property @@ -369,13 +357,40 @@ def closure_pseudodata_replicas(experiments, pdf, nclosure:int, return df +#TODO: Add check here that dataset appears in fitthcovmat (if true) and that cuts match +def covariance_matrix(dataset:DataSetSpec, fitthcovmat): + """Returns a tuple of Covariance matrix and sqrt covariance matrix for the given dataset + which includes theory contribution from scale variations if `use_theorycovmat` is True and + an appropriate fit from which the covariance matrix will be loaded is given + """ + loaded_data = dataset.load() + if fitthcovmat: + loaded_thcov = fitthcovmat.load() + covmat = get_df_block(loaded_thcov, dataset.name, level=1) + loaded_data.get_covmat() + sqrtcovmat = np.linalg.cholesky(covmat) + else: + covmat = loaded_data.get_covmat() + sqrtcovmat = loaded_data.get_sqrtcovmat() + return covmat, sqrtcovmat + +def experiment_covariance_matrix(experiment: ExperimentSpec, fitthcovmat): + loaded_data = experiment.load() + if fitthcovmat: + loaded_thcov = fitthcovmat.load() + ds_names = loaded_thcov.index.get_level_values(1) + indices = np.in1d(ds_names, [ds.name for ds in experiment.datasets]).nonzero()[0] + covmat = loaded_thcov.iloc[indices, indices].values + loaded_data.get_covmat() + sqrtcovmat = np.linalg.cholesky(covmat) + else: + covmat = loaded_data.get_covmat() + sqrtcovmat = loaded_data.get_sqrtcovmat() + return covmat, sqrtcovmat -def results(dataset:(DataSetSpec), pdf:PDF, t0set:(PDF, type(None))=None, fixup_fitthcovmat=False): +def results(dataset:(DataSetSpec), pdf:PDF, covariance_matrix, t0set:(PDF, type(None))=None): """Tuple of data and theory results for a single pdf. The theory is specified as part of the dataset. An experiment is also allowed. (as a result of the C++ code layout).""" - data = dataset.load() @@ -385,12 +400,13 @@ def results(dataset:(DataSetSpec), pdf:PDF, t0set:(PDF, type(None))=None, fixup_ log.debug("Setting T0 predictions for %s" % dataset) data.SetT0(t0set.load_t0()) - return (DataResult(data, thcovmat=fixup_fitthcovmat), + return (DataResult(data, *covariance_matrix), ThPredictionsResult.from_convolution(pdf, dataset, loaded_data=data)) -def experiment_results(experiment, pdf:PDF, t0set:(PDF, type(None))=None, fixup_fitthcovmat=False): +def experiment_results(experiment, pdf:PDF, experiment_covariance_matrix, + t0set:(PDF, type(None))=None): """Like `results` but for a whole experiment""" - return results(experiment, pdf, t0set, fixup_fitthcovmat=fixup_fitthcovmat) + return results(experiment, pdf, experiment_covariance_matrix, t0set) #It's better to duplicate a few lines than to complicate the logic of #``results`` to support this. @@ -455,11 +471,13 @@ def abs_chi2_data_experiment(experiment_results): def phi_data(abs_chi2_data): """Calculate phi using values returned by `abs_chi2_data`. + Returns tuple of (phi, numpoints) + For more information on how phi is calculated see Eq.(24) in 1410.8849 """ alldata, central, npoints = abs_chi2_data - return np.sqrt((alldata.data.mean() - central)/npoints) + return (np.sqrt((alldata.data.mean() - central)/npoints), npoints) def phi_data_experiment(abs_chi2_data_experiment): """Like `phi_data` but for whole experiment""" @@ -656,7 +674,40 @@ def dataset_chi2_table(chi2_stats, dataset): """Show the chi² estimators for a given dataset""" return pd.DataFrame(chi2_stats, index=[dataset.name]) +fits_exp_from_plotting_chi2 = collect( + 'experiments_chi2', ('fits', 'experiments_from_plotting_withcontext',)) +fits_exp_from_plotting = collect( + 'experiments', ('fits', 'experiments_from_plotting_withcontext',)) +@table +def fits_exp_by_plotting_chi2_table(fits, fits_exp_from_plotting, fits_exp_from_plotting_chi2, + per_point_data:bool=True): + """For every fit, returns the chi2 and number of data points per experiment, where experiment is + a collection of datasets grouped according to the experiment key in the plotting info file + """ + dfs = [] + cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') + for fit, experiments, exps_chi2 in zip(fits, fits_exp_from_plotting, fits_exp_from_plotting_chi2): + records = [] + for experiment, exp_chi2 in zip(experiments, exps_chi2): + mean_chi2 = exp_chi2.central_result.mean() + npoints = exp_chi2.ndata + records.append(dict( + experiment=str(experiment), + npoints=npoints, + mean_chi2 = mean_chi2 + + )) + df = pd.DataFrame.from_records(records, + columns=('experiment', 'npoints', 'mean_chi2'), + index = ('experiment', ) + ) + if per_point_data: + df['mean_chi2'] /= df['npoints'] + df.columns = pd.MultiIndex.from_product(([str(fit)], cols)) + dfs.append(df) + res = pd.concat(dfs, axis=1) + return res #TODO: Possibly get rid of the per_point_data parameter and have separate #actions for absolute and relative tables. @@ -691,6 +742,35 @@ def fits_experiments_chi2_table(fits, fits_experiments, fits_experiment_chi2_dat res = pd.concat(dfs, axis=1) return res +fits_exp_by_plotting_phi = collect( + 'experiments_phi', ('fits', 'experiments_from_plotting_withcontext')) + +@table +def fits_phi_table(fits, fits_exp_from_plotting, fits_exp_by_plotting_phi): + """For every fit, returns phi and number of data points per experiment, where experiment is + a collection of datasets grouped according to the experiment key in the plotting info file + """ + dfs = [] + cols = ('ndata', r'$\phi$') + for fit, experiments, exps_phi in zip(fits, fits_exp_from_plotting, fits_exp_by_plotting_phi): + records = [] + for experiment, (exp_phi, npoints) in zip(experiments, exps_phi): + npoints = npoints + records.append(dict( + experiment=str(experiment), + npoints=npoints, + phi = exp_phi + + )) + df = pd.DataFrame.from_records(records, + columns=('experiment', 'npoints', 'phi'), + index = ('experiment', ) + ) + df.columns = pd.MultiIndex.from_product(([str(fit)], cols)) + dfs.append(df) + res = pd.concat(dfs, axis=1) + return res + @table @check_speclabels_different def dataspecs_experiments_chi2_table(dataspecs_speclabel, dataspecs_experiments, @@ -704,20 +784,20 @@ def dataspecs_experiments_chi2_table(dataspecs_speclabel, dataspecs_experiments, @table -def fits_datasets_chi2_table(fits, fits_experiments, fits_chi2_data, +def fits_datasets_chi2_table(fits, fits_exp_from_plotting, fits_chi2_data, per_point_data:bool=True): """A table with the chi2 for each included dataset in the fits, computed with the theory corresponding to the fit. The result are indexed in two - levels by experiment and dataset. If points_per_data is True, - the chi² will be shown divided by ndata. - Otherwise they will be absolute.""" + levels by experiment and dataset, where experiment is the grouping of datasets according to the + `experiment` key in the plotting info file. If points_per_data is True, the chi² will be shown + divided by ndata. Otherwise they will be absolute.""" chi2_it = iter(fits_chi2_data) cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') dfs = [] - for fit, experiments in zip(fits, fits_experiments): + for fit, experiments in zip(fits, fits_exp_from_plotting): records = [] for experiment in experiments: for dataset, chi2 in zip(experiment.datasets, chi2_it): @@ -755,15 +835,16 @@ def dataspecs_datasets_chi2_table(dataspecs_speclabel, dataspecs_experiments, def fits_chi2_table( fits_experiments_chi2_table, fits_datasets_chi2_table, + fits_exp_by_plotting_chi2_table, show_total:bool=False): """Show the chi² of each and number of points of each dataset and experiment - of each fit, - computed with the theory corresponding to the fit. Dataset that are not - included in some fit appear as "Not Fitted". This is itended for display - purposes.""" - lvs = fits_experiments_chi2_table.index + of each fit, where experiment is a group of datasets according to the `experiment` key in + the plotting info file, computed with the theory corresponding to the fit. Dataset that are not + included in some fit appear as `NaN` + """ + lvs = fits_exp_by_plotting_chi2_table.index expanded_index = pd.MultiIndex.from_product((lvs, ["Total"])) - edf = fits_experiments_chi2_table.set_index(expanded_index) + edf = fits_exp_by_plotting_chi2_table.set_index(expanded_index) ddf = fits_datasets_chi2_table dfs = [] #TODO: Better way to do the merge preserving the order? @@ -883,7 +964,6 @@ def experiments_central_values(experiment_result_table): experiments_results = collect(experiment_results, ('experiments',)) experiments_phi = collect(phi_data_experiment, ('experiments',)) -experiments_pdfs_phi = collect('experiments_phi', ('pdfs',)) pdfs_total_chi2 = collect(total_experiments_chi2, ('pdfs',)) @@ -919,23 +999,3 @@ def experiments_central_values(experiment_result_table): dataspecs_commondata = collect('commondata', ('dataspecs',)) dataspecs_pdf = collect('pdf', ('dataspecs',)) dataspecs_fit = collect('fit', ('dataspecs',)) - -def fixup_fitthcovmat(fitthcovmat, experiments=None): - """If both `fitthcovmat` and `experiments` exist then change the level 1 keys of `fitthcovmat` - to match `experiments` - """ - if fitthcovmat: - loaded_covmat = fitthcovmat.load() - if experiments: - exp_names = np.array(loaded_covmat.index.get_level_values(0), copy=True) - ds_names = loaded_covmat.index.get_level_values(1) - ds_index = loaded_covmat.index.get_level_values(2) - for exp in experiments: - for ds in exp.datasets: - choose = np.where(ds_names == ds.name) - exp_names[choose] = exp.name - new_index = pd.MultiIndex.from_arrays([exp_names, ds_names, ds_index]) - loaded_covmat = pd.DataFrame(loaded_covmat.values, index=new_index, columns=new_index) - return loaded_covmat - else: - return fitthcovmat From 8f01a540f43583e159de203cf5555b55aa8911c0 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 12 Apr 2019 13:28:26 +0100 Subject: [PATCH 03/13] fixed fits_chi2_table for totals to work, added warning about future deprecation to fits_chi2_table and summarise_fits --- validphys2/src/validphys/fitdata.py | 3 +++ validphys2/src/validphys/results.py | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index 9f63b10420..64c768c852 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -165,6 +165,9 @@ def fit_summary(fit, replica_data, total_experiments_chi2data): def summarise_fits(collected_fit_summaries): """ Produces a table of basic comparisons between fits, includes all the fields used in fit_summary """ + log.warning("Using `collected_fit_summaries` which collects over `experiments` defined in fit " + "can affect construction of total covariance matrix, this will be deprecated " + "and should be changed to collect over `data` when that keyword is implemented") return pd.concat(collected_fit_summaries, axis=1) diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index da29f28f76..e065c67944 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -830,10 +830,13 @@ def dataspecs_datasets_chi2_table(dataspecs_speclabel, dataspecs_experiments, dataspecs_chi2_data, per_point_data=per_point_data) +#NOTE: This will need to be changed to work with `data` when that gets added +fits_total_chi2_data = collect('total_experiments_chi2data', ('fits', 'fitcontext')) + #TODO: Decide what to do with the horrible totals code. @table def fits_chi2_table( - fits_experiments_chi2_table, + fits_total_chi2_data, fits_datasets_chi2_table, fits_exp_by_plotting_chi2_table, show_total:bool=False): @@ -851,15 +854,17 @@ def fits_chi2_table( for lv in lvs: dfs.append(pd.concat((edf.loc[lv],ddf.loc[lv]), copy=False, axis=0)) if show_total: - total_points = fits_experiments_chi2_table.iloc[:, 0::2].sum().values - total_chi = (fits_experiments_chi2_table.iloc[:, 0::2].values * - fits_experiments_chi2_table.iloc[:,1::2].values).sum(axis=0) + log.warning("Using `fits_total_chi2_data` which collects over `experiments` defined in fit " + "can affect construction of total covariance matrix, this will be deprecated " + "and should be changed to collect over `data` when that keyword is implemented") + total_points = np.array([total_chi2_data.ndata for total_chi2_data in fits_total_chi2_data]) + total_chi = np.array([total_chi2_data.central_result for total_chi2_data in fits_total_chi2_data]) total_chi /= total_points row = np.zeros(len(total_points)*2) row[::2] = total_points row[1::2] = total_chi df = pd.DataFrame(np.atleast_2d(row), - columns=fits_experiments_chi2_table.columns, + columns=fits_exp_by_plotting_chi2_table.columns, index=['Total']) dfs.append(df) keys = [*lvs, 'Total'] @@ -974,15 +979,13 @@ def experiments_central_values(experiment_result_table): #These are convenient ways to iterate and extract varios data from fits fits_chi2_data = collect(abs_chi2_data, ('fits', 'fitcontext', 'experiments', 'experiment')) -fits_experiment_chi2_data = collect('experiments_chi2', ('fits', 'fitcontext')) + fits_total_chi2 = collect('total_experiments_chi2', ('fits', 'fitcontext')) fits_total_chi2_for_experiments = collect('total_experiment_chi2', ('fits', 'fittheoryandpdf', 'expspec', 'experiment')) - -fits_experiments = collect('experiments', ('fits', 'fitcontext')) fits_pdf = collect('pdf', ('fits', 'fitpdf')) #Dataspec is so From 08524d63ba0266856eadf0098cb147845dc5b3ed Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 12 Apr 2019 13:44:41 +0100 Subject: [PATCH 04/13] deprecated/changed old experiments functions to use new behaviour --- .../validphys/comparefittemplates/report.md | 4 +- validphys2/src/validphys/dataplots.py | 16 +---- validphys2/src/validphys/results.py | 58 +++++-------------- 3 files changed, 19 insertions(+), 59 deletions(-) diff --git a/validphys2/src/validphys/comparefittemplates/report.md b/validphys2/src/validphys/comparefittemplates/report.md index 08d7bb89ba..1f7030d63b 100644 --- a/validphys2/src/validphys/comparefittemplates/report.md +++ b/validphys2/src/validphys/comparefittemplates/report.md @@ -33,7 +33,7 @@ Training validation $\chi^2$ by experiment ---------------------- -{@plot_fits_exp_by_plotting_chi2@} +{@plot_fits_experiments_chi2@} $\chi^2$ by dataset comparisons ------------------------------- @@ -44,7 +44,7 @@ $\chi^2$ by dataset comparisons $\phi$ by experiment -------------------- -{@plot_phi_fits_exp_by_plotting@} +{@plot_fits_experiments_phi@} Experiment plots --------------- diff --git a/validphys2/src/validphys/dataplots.py b/validphys2/src/validphys/dataplots.py index 632ee51ee7..6ecdb2adb8 100644 --- a/validphys2/src/validphys/dataplots.py +++ b/validphys2/src/validphys/dataplots.py @@ -61,11 +61,11 @@ def plot_phi(experiments, experiments_phi): return fig @figure -def plot_phi_fits_exp_by_plotting(fits_phi_table): +def plot_fits_experiments_phi(fits_experiments_phi_table): """Plots a set of bars for each fit, each bar represents the value of phi for the corresponding experiment, where the experiment is a group of datasets according to the `experiment` key in - the plotting info file""" - fig, ax = _plot_chis_df(fits_phi_table) + the PLOTTING info file""" + fig, ax = _plot_chis_df(fits_experiments_phi_table) ax.set_title(r"$\phi$ for each experiment") return fig @@ -531,16 +531,6 @@ def plot_fits_experiments_chi2(fits_experiments_chi2_table): ax.set_title(r"$\chi^2$ for experiments") return fig -@figure -def plot_fits_exp_by_plotting_chi2(fits_exp_by_plotting_chi2_table): - """Generate a plot equivalent to ``plot_experiments_chi2`` using all fits - and grouping datasets into experiments according to the `experiment` key defined in the - plotting file. - """ - fig, ax = _plot_chis_df(fits_exp_by_plotting_chi2_table) - ax.set_title(r"$\chi^2$ for experiments") - return fig - @figure def plot_dataspecs_experiments_chi2(dataspecs_experiments_chi2_table): """Same as plot_fits_experiments_chi2 but for arbitrary dataspecs""" diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index e065c67944..e089cbb238 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -674,41 +674,11 @@ def dataset_chi2_table(chi2_stats, dataset): """Show the chi² estimators for a given dataset""" return pd.DataFrame(chi2_stats, index=[dataset.name]) -fits_exp_from_plotting_chi2 = collect( +fits_experiment_chi2_data = collect( 'experiments_chi2', ('fits', 'experiments_from_plotting_withcontext',)) -fits_exp_from_plotting = collect( +fits_experiments = collect( 'experiments', ('fits', 'experiments_from_plotting_withcontext',)) -@table -def fits_exp_by_plotting_chi2_table(fits, fits_exp_from_plotting, fits_exp_from_plotting_chi2, - per_point_data:bool=True): - """For every fit, returns the chi2 and number of data points per experiment, where experiment is - a collection of datasets grouped according to the experiment key in the plotting info file - """ - dfs = [] - cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') - for fit, experiments, exps_chi2 in zip(fits, fits_exp_from_plotting, fits_exp_from_plotting_chi2): - records = [] - for experiment, exp_chi2 in zip(experiments, exps_chi2): - mean_chi2 = exp_chi2.central_result.mean() - npoints = exp_chi2.ndata - records.append(dict( - experiment=str(experiment), - npoints=npoints, - mean_chi2 = mean_chi2 - - )) - df = pd.DataFrame.from_records(records, - columns=('experiment', 'npoints', 'mean_chi2'), - index = ('experiment', ) - ) - if per_point_data: - df['mean_chi2'] /= df['npoints'] - df.columns = pd.MultiIndex.from_product(([str(fit)], cols)) - dfs.append(df) - res = pd.concat(dfs, axis=1) - return res - #TODO: Possibly get rid of the per_point_data parameter and have separate #actions for absolute and relative tables. @table @@ -742,17 +712,17 @@ def fits_experiments_chi2_table(fits, fits_experiments, fits_experiment_chi2_dat res = pd.concat(dfs, axis=1) return res -fits_exp_by_plotting_phi = collect( +fits_experiments_phi = collect( 'experiments_phi', ('fits', 'experiments_from_plotting_withcontext')) @table -def fits_phi_table(fits, fits_exp_from_plotting, fits_exp_by_plotting_phi): +def fits_experiments_phi_table(fits, fits_experiments, fits_experiments_phi): """For every fit, returns phi and number of data points per experiment, where experiment is - a collection of datasets grouped according to the experiment key in the plotting info file + a collection of datasets grouped according to the experiment key in the PLOTTING info file """ dfs = [] cols = ('ndata', r'$\phi$') - for fit, experiments, exps_phi in zip(fits, fits_exp_from_plotting, fits_exp_by_plotting_phi): + for fit, experiments, exps_phi in zip(fits, fits_experiments, fits_experiments_phi): records = [] for experiment, (exp_phi, npoints) in zip(experiments, exps_phi): npoints = npoints @@ -784,12 +754,12 @@ def dataspecs_experiments_chi2_table(dataspecs_speclabel, dataspecs_experiments, @table -def fits_datasets_chi2_table(fits, fits_exp_from_plotting, fits_chi2_data, +def fits_datasets_chi2_table(fits, fits_experiments, fits_chi2_data, per_point_data:bool=True): """A table with the chi2 for each included dataset in the fits, computed with the theory corresponding to the fit. The result are indexed in two levels by experiment and dataset, where experiment is the grouping of datasets according to the - `experiment` key in the plotting info file. If points_per_data is True, the chi² will be shown + `experiment` key in the PLOTTING info file. If points_per_data is True, the chi² will be shown divided by ndata. Otherwise they will be absolute.""" chi2_it = iter(fits_chi2_data) @@ -797,7 +767,7 @@ def fits_datasets_chi2_table(fits, fits_exp_from_plotting, fits_chi2_data, cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') dfs = [] - for fit, experiments in zip(fits, fits_exp_from_plotting): + for fit, experiments in zip(fits, fits_experiments): records = [] for experiment in experiments: for dataset, chi2 in zip(experiment.datasets, chi2_it): @@ -838,16 +808,16 @@ def dataspecs_datasets_chi2_table(dataspecs_speclabel, dataspecs_experiments, def fits_chi2_table( fits_total_chi2_data, fits_datasets_chi2_table, - fits_exp_by_plotting_chi2_table, + fits_experiments_chi2_table, show_total:bool=False): """Show the chi² of each and number of points of each dataset and experiment of each fit, where experiment is a group of datasets according to the `experiment` key in - the plotting info file, computed with the theory corresponding to the fit. Dataset that are not + the PLOTTING info file, computed with the theory corresponding to the fit. Dataset that are not included in some fit appear as `NaN` """ - lvs = fits_exp_by_plotting_chi2_table.index + lvs = fits_experiments_chi2_table.index expanded_index = pd.MultiIndex.from_product((lvs, ["Total"])) - edf = fits_exp_by_plotting_chi2_table.set_index(expanded_index) + edf = fits_experiments_chi2_table.set_index(expanded_index) ddf = fits_datasets_chi2_table dfs = [] #TODO: Better way to do the merge preserving the order? @@ -864,7 +834,7 @@ def fits_chi2_table( row[::2] = total_points row[1::2] = total_chi df = pd.DataFrame(np.atleast_2d(row), - columns=fits_exp_by_plotting_chi2_table.columns, + columns=fits_experiments_chi2_table.columns, index=['Total']) dfs.append(df) keys = [*lvs, 'Total'] From 13ceab356cdcae058491dcff8a7424a509915ecf Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 12 Apr 2019 14:31:25 +0100 Subject: [PATCH 05/13] attempt to get tests working --- validphys2/src/validphys/results.py | 10 ++++++---- validphys2/src/validphys/tests/conftest.py | 10 +++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index e089cbb238..ae319956f3 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -411,7 +411,8 @@ def experiment_results(experiment, pdf:PDF, experiment_covariance_matrix, #It's better to duplicate a few lines than to complicate the logic of #``results`` to support this. #TODO: The above comment doesn't make sense after adding T0. Deprecate this -def pdf_results(dataset:(DataSetSpec, ExperimentSpec), pdfs:Sequence, t0set:(PDF, type(None))): +def pdf_results(dataset:(DataSetSpec, ExperimentSpec), pdfs:Sequence, covariance_matrix:tuple, + t0set:(PDF, type(None))): """Return a list of results, the first for the data and the rest for each of the PDFs.""" @@ -430,11 +431,12 @@ def pdf_results(dataset:(DataSetSpec, ExperimentSpec), pdfs:Sequence, t0set:(PD th_results.append(th_result) - return (DataResult(data), *th_results) + return (DataResult(data, *covariance_matrix), *th_results) @require_one('pdfs', 'pdf') @remove_outer('pdfs', 'pdf') def one_or_more_results(dataset:(DataSetSpec, ExperimentSpec), + covariance_matrix: tuple, pdfs:(type(None), Sequence)=None, pdf:(type(None), PDF)=None, t0set:(PDF, type(None))=None): @@ -443,9 +445,9 @@ def one_or_more_results(dataset:(DataSetSpec, ExperimentSpec), Which of the two is selected intelligently depending on the namespace, when executing as an action.""" if pdf: - return results(dataset, pdf, t0set) + return results(dataset, pdf, covariance_matrix, t0set) else: - return pdf_results(dataset, pdfs, t0set) + return pdf_results(dataset, pdfs, covariance_matrix, t0set) raise ValueError("Either 'pdf' or 'pdfs' is required") diff --git a/validphys2/src/validphys/tests/conftest.py b/validphys2/src/validphys/tests/conftest.py index 90d9720494..f57bbd25e1 100644 --- a/validphys2/src/validphys/tests/conftest.py +++ b/validphys2/src/validphys/tests/conftest.py @@ -36,7 +36,9 @@ def data(): def convolution_results_implement(data): pdf, exps = data - return [results.experiment_results(exp, pdf, pdf) for exp in exps] + #no theory covmat here + covs = [results.experiment_covariance_matrix(exp, False) for exp in exps] + return [results.experiment_results(exp, pdf, cov, pdf) for exp, cov in zip(exps, covs)] @pytest.fixture(scope='module') def convolution_results(data): @@ -46,7 +48,8 @@ def convolution_results(data): def dataset_t0_convolution_results(data): pdf, exps = data ds = [x.datasets[0] for x in exps] - return [results.results(x, pdf, t0set=pdf) for x in ds] + covs = [results.covariance_matrix(x, False) for x in ds] + return [results.results(x, pdf, cov, t0set=pdf) for x, cov in zip(ds, covs)] @pytest.fixture(scope='module') def single_exp_data(): @@ -62,7 +65,8 @@ def single_exp_data(): @pytest.fixture(scope='module') def dataset_convolution_results(single_exp_data): pdf, exp = single_exp_data - return [results.results(ds, pdf, pdf) for ds in exp.datasets] + covs = [results.covariance_matrix(ds, False) for ds in exp.datasets] + return [results.results(ds, pdf, cov, pdf) for ds, cov in zip(exp.datasets, covs)] @pytest.fixture(scope='module') def dataset_chi2data(dataset_convolution_results): From 0cb17f8165d5965a6ec213735420b46d3cf07715 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 12 Apr 2019 16:18:57 +0100 Subject: [PATCH 06/13] included t0set option in covariance_matrix --- validphys2/src/validphys/results.py | 53 +++++++++++----------- validphys2/src/validphys/tests/conftest.py | 12 ++--- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index ae319956f3..f43fc3561e 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -358,12 +358,21 @@ def closure_pseudodata_replicas(experiments, pdf, nclosure:int, return df #TODO: Add check here that dataset appears in fitthcovmat (if true) and that cuts match -def covariance_matrix(dataset:DataSetSpec, fitthcovmat): +def covariance_matrix(dataset:DataSetSpec, fitthcovmat, t0set:(PDF, type(None))=None): """Returns a tuple of Covariance matrix and sqrt covariance matrix for the given dataset which includes theory contribution from scale variations if `use_theorycovmat` is True and an appropriate fit from which the covariance matrix will be loaded is given + + if t0set is specified then t0 predictions will be used to construct the covariance matrix. """ loaded_data = dataset.load() + + if t0set: + #Copy data to avoid chaos + loaded_data = type(loaded_data)(loaded_data) + log.debug("Setting T0 predictions for %s" % dataset) + loaded_data.SetT0(t0set.load_t0()) + if fitthcovmat: loaded_thcov = fitthcovmat.load() covmat = get_df_block(loaded_thcov, dataset.name, level=1) + loaded_data.get_covmat() @@ -373,8 +382,16 @@ def covariance_matrix(dataset:DataSetSpec, fitthcovmat): sqrtcovmat = loaded_data.get_sqrtcovmat() return covmat, sqrtcovmat -def experiment_covariance_matrix(experiment: ExperimentSpec, fitthcovmat): +def experiment_covariance_matrix(experiment: ExperimentSpec, fitthcovmat, t0set:(PDF, type(None))=None): + """Like `covariance_matrix` except for an experiment""" loaded_data = experiment.load() + + if t0set: + #Copy data to avoid chaos + loaded_data = type(loaded_data)(loaded_data) + log.debug("Setting T0 predictions for %s" % experiment) + loaded_data.SetT0(t0set.load_t0()) + if fitthcovmat: loaded_thcov = fitthcovmat.load() ds_names = loaded_thcov.index.get_level_values(1) @@ -386,44 +403,27 @@ def experiment_covariance_matrix(experiment: ExperimentSpec, fitthcovmat): sqrtcovmat = loaded_data.get_sqrtcovmat() return covmat, sqrtcovmat -def results(dataset:(DataSetSpec), pdf:PDF, covariance_matrix, t0set:(PDF, type(None))=None): +def results(dataset:(DataSetSpec), pdf:PDF, covariance_matrix): """Tuple of data and theory results for a single pdf. The theory is specified as part of the dataset. An experiment is also allowed. (as a result of the C++ code layout).""" data = dataset.load() - - - if t0set: - #Copy data to avoid chaos - data = type(data)(data) - log.debug("Setting T0 predictions for %s" % dataset) - data.SetT0(t0set.load_t0()) - return (DataResult(data, *covariance_matrix), ThPredictionsResult.from_convolution(pdf, dataset, loaded_data=data)) -def experiment_results(experiment, pdf:PDF, experiment_covariance_matrix, - t0set:(PDF, type(None))=None): +def experiment_results(experiment, pdf:PDF, experiment_covariance_matrix): """Like `results` but for a whole experiment""" - return results(experiment, pdf, experiment_covariance_matrix, t0set) + return results(experiment, pdf, experiment_covariance_matrix) #It's better to duplicate a few lines than to complicate the logic of #``results`` to support this. #TODO: The above comment doesn't make sense after adding T0. Deprecate this -def pdf_results(dataset:(DataSetSpec, ExperimentSpec), pdfs:Sequence, covariance_matrix:tuple, - t0set:(PDF, type(None))): +def pdf_results(dataset:(DataSetSpec, ExperimentSpec), pdfs:Sequence, covariance_matrix:tuple): """Return a list of results, the first for the data and the rest for each of the PDFs.""" data = dataset.load() - - if t0set: - #Copy data to avoid chaos - data = type(data)(data) - log.debug("Setting T0 predictions for %s" % dataset) - data.SetT0(t0set.load_t0()) - th_results = [] for pdf in pdfs: th_result = ThPredictionsResult.from_convolution(pdf, dataset, @@ -438,16 +438,15 @@ def pdf_results(dataset:(DataSetSpec, ExperimentSpec), pdfs:Sequence, covarianc def one_or_more_results(dataset:(DataSetSpec, ExperimentSpec), covariance_matrix: tuple, pdfs:(type(None), Sequence)=None, - pdf:(type(None), PDF)=None, - t0set:(PDF, type(None))=None): + pdf:(type(None), PDF)=None): """Generate a list of results, where the first element is the data values, and the next is either the prediction for pdf or for each of the pdfs. Which of the two is selected intelligently depending on the namespace, when executing as an action.""" if pdf: - return results(dataset, pdf, covariance_matrix, t0set) + return results(dataset, pdf, covariance_matrix) else: - return pdf_results(dataset, pdfs, covariance_matrix, t0set) + return pdf_results(dataset, pdfs, covariance_matrix) raise ValueError("Either 'pdf' or 'pdfs' is required") diff --git a/validphys2/src/validphys/tests/conftest.py b/validphys2/src/validphys/tests/conftest.py index f57bbd25e1..3a84a34314 100644 --- a/validphys2/src/validphys/tests/conftest.py +++ b/validphys2/src/validphys/tests/conftest.py @@ -37,8 +37,8 @@ def data(): def convolution_results_implement(data): pdf, exps = data #no theory covmat here - covs = [results.experiment_covariance_matrix(exp, False) for exp in exps] - return [results.experiment_results(exp, pdf, cov, pdf) for exp, cov in zip(exps, covs)] + covs = [results.experiment_covariance_matrix(exp, False, pdf) for exp in exps] + return [results.experiment_results(exp, pdf, cov) for exp, cov in zip(exps, covs)] @pytest.fixture(scope='module') def convolution_results(data): @@ -48,8 +48,8 @@ def convolution_results(data): def dataset_t0_convolution_results(data): pdf, exps = data ds = [x.datasets[0] for x in exps] - covs = [results.covariance_matrix(x, False) for x in ds] - return [results.results(x, pdf, cov, t0set=pdf) for x, cov in zip(ds, covs)] + covs = [results.covariance_matrix(x, False, pdf) for x in ds] + return [results.results(x, pdf, cov) for x, cov in zip(ds, covs)] @pytest.fixture(scope='module') def single_exp_data(): @@ -65,8 +65,8 @@ def single_exp_data(): @pytest.fixture(scope='module') def dataset_convolution_results(single_exp_data): pdf, exp = single_exp_data - covs = [results.covariance_matrix(ds, False) for ds in exp.datasets] - return [results.results(ds, pdf, cov, pdf) for ds, cov in zip(exp.datasets, covs)] + covs = [results.covariance_matrix(ds, False, pdf) for ds in exp.datasets] + return [results.results(ds, pdf, cov) for ds, cov in zip(exp.datasets, covs)] @pytest.fixture(scope='module') def dataset_chi2data(dataset_convolution_results): From d03edeb9ba28a020b55decc3c79df5cf8849de67 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Mon, 15 Apr 2019 17:45:26 +0100 Subject: [PATCH 07/13] added labels to vp-comparefits tables and plots to indicate th covmat usage --- validphys2/src/validphys/fitdata.py | 4 ++-- validphys2/src/validphys/results.py | 33 +++++++++++++++++++---------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index 64c768c852..7923058879 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -120,7 +120,7 @@ def replica_data(fit, replica_paths): @table -def fit_summary(fit, replica_data, total_experiments_chi2data): +def fit_summary(fit_label, replica_data, total_experiments_chi2data): """ Summary table of fit properties - Central chi-squared - Average chi-squared @@ -157,7 +157,7 @@ def fit_summary(fit, replica_data, total_experiments_chi2data): (r"$<\chi^2>$", f"{VET(np.mean(member_chi2), np.std(member_chi2))}"), (r"$\phi$", f"{VET(phi, phi_err)}"))) - return pd.Series(data, index=data.keys(), name=fit.name) + return pd.Series(data, index=data.keys(), name=fit_label) collected_fit_summaries = collect('fit_summary', ('fits', 'fitcontext')) diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index f43fc3561e..11802648ea 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -358,7 +358,7 @@ def closure_pseudodata_replicas(experiments, pdf, nclosure:int, return df #TODO: Add check here that dataset appears in fitthcovmat (if true) and that cuts match -def covariance_matrix(dataset:DataSetSpec, fitthcovmat, t0set:(PDF, type(None))=None): +def covariance_matrix(dataset:DataSetSpec, fitthcovmat, t0set:(PDF, type(None)) = None): """Returns a tuple of Covariance matrix and sqrt covariance matrix for the given dataset which includes theory contribution from scale variations if `use_theorycovmat` is True and an appropriate fit from which the covariance matrix will be loaded is given @@ -382,7 +382,8 @@ def covariance_matrix(dataset:DataSetSpec, fitthcovmat, t0set:(PDF, type(None))= sqrtcovmat = loaded_data.get_sqrtcovmat() return covmat, sqrtcovmat -def experiment_covariance_matrix(experiment: ExperimentSpec, fitthcovmat, t0set:(PDF, type(None))=None): +def experiment_covariance_matrix( + experiment: ExperimentSpec, fitthcovmat, t0set:(PDF, type(None)) = None): """Like `covariance_matrix` except for an experiment""" loaded_data = experiment.load() @@ -680,10 +681,20 @@ def dataset_chi2_table(chi2_stats, dataset): fits_experiments = collect( 'experiments', ('fits', 'experiments_from_plotting_withcontext',)) +def fit_label(fit, fitthcovmat): + if fitthcovmat: + label = str(fit) + " (exp + th)" + else: + label = str(fit) + return label + +fits_label = collect('fit_label', ('fits',)) + + #TODO: Possibly get rid of the per_point_data parameter and have separate #actions for absolute and relative tables. @table -def fits_experiments_chi2_table(fits, fits_experiments, fits_experiment_chi2_data, +def fits_experiments_chi2_table(fits_label, fits_experiments, fits_experiment_chi2_data, per_point_data:bool=True): """A table with the chi2 for each included experiment in the fits, computed with the theory corresponding to each fit. If points_per_data is @@ -691,7 +702,7 @@ def fits_experiments_chi2_table(fits, fits_experiments, fits_experiment_chi2_dat Otherwise they will be absolute.""" dfs = [] cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') - for fit, experiments, exps_chi2 in zip(fits, fits_experiments, fits_experiment_chi2_data): + for label, experiments, exps_chi2 in zip(fits_label, fits_experiments, fits_experiment_chi2_data): records = [] for experiment, exp_chi2 in zip(experiments, exps_chi2): mean_chi2 = exp_chi2.central_result.mean() @@ -708,7 +719,7 @@ def fits_experiments_chi2_table(fits, fits_experiments, fits_experiment_chi2_dat ) if per_point_data: df['mean_chi2'] /= df['npoints'] - df.columns = pd.MultiIndex.from_product(([str(fit)], cols)) + df.columns = pd.MultiIndex.from_product(([label], cols)) dfs.append(df) res = pd.concat(dfs, axis=1) return res @@ -717,13 +728,13 @@ def fits_experiments_chi2_table(fits, fits_experiments, fits_experiment_chi2_dat 'experiments_phi', ('fits', 'experiments_from_plotting_withcontext')) @table -def fits_experiments_phi_table(fits, fits_experiments, fits_experiments_phi): +def fits_experiments_phi_table(fits_label, fits_experiments, fits_experiments_phi): """For every fit, returns phi and number of data points per experiment, where experiment is a collection of datasets grouped according to the experiment key in the PLOTTING info file """ dfs = [] cols = ('ndata', r'$\phi$') - for fit, experiments, exps_phi in zip(fits, fits_experiments, fits_experiments_phi): + for label, experiments, exps_phi in zip(fits_label, fits_experiments, fits_experiments_phi): records = [] for experiment, (exp_phi, npoints) in zip(experiments, exps_phi): npoints = npoints @@ -737,7 +748,7 @@ def fits_experiments_phi_table(fits, fits_experiments, fits_experiments_phi): columns=('experiment', 'npoints', 'phi'), index = ('experiment', ) ) - df.columns = pd.MultiIndex.from_product(([str(fit)], cols)) + df.columns = pd.MultiIndex.from_product(([label], cols)) dfs.append(df) res = pd.concat(dfs, axis=1) return res @@ -755,7 +766,7 @@ def dataspecs_experiments_chi2_table(dataspecs_speclabel, dataspecs_experiments, @table -def fits_datasets_chi2_table(fits, fits_experiments, fits_chi2_data, +def fits_datasets_chi2_table(fits_label, fits_experiments, fits_chi2_data, per_point_data:bool=True): """A table with the chi2 for each included dataset in the fits, computed with the theory corresponding to the fit. The result are indexed in two @@ -768,7 +779,7 @@ def fits_datasets_chi2_table(fits, fits_experiments, fits_chi2_data, cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') dfs = [] - for fit, experiments in zip(fits, fits_experiments): + for label, experiments in zip(fits_label, fits_experiments): records = [] for experiment in experiments: for dataset, chi2 in zip(experiment.datasets, chi2_it): @@ -788,7 +799,7 @@ def fits_datasets_chi2_table(fits, fits_experiments, fits_chi2_data, ) if per_point_data: df['mean_chi2'] /= df['npoints'] - df.columns = pd.MultiIndex.from_product(([str(fit)], cols)) + df.columns = pd.MultiIndex.from_product(([label], cols)) dfs.append(df) return pd.concat(dfs, axis=1) From 3c6c7bdda0cca825b713ae5bf14cdfd380daad34 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Tue, 16 Apr 2019 14:56:15 +0100 Subject: [PATCH 08/13] added rudimentary checks for loaded th covmat, added vp-comparefits summarise if th covmat used --- validphys2/src/validphys/checks.py | 31 +++++++++++++++++++ .../validphys/comparefittemplates/report.md | 4 +++ validphys2/src/validphys/fitdata.py | 26 ++++++++++++++++ validphys2/src/validphys/results.py | 14 +++++++-- 4 files changed, 72 insertions(+), 3 deletions(-) diff --git a/validphys2/src/validphys/checks.py b/validphys2/src/validphys/checks.py index e6ce3eb531..d2719ec690 100644 --- a/validphys2/src/validphys/checks.py +++ b/validphys2/src/validphys/checks.py @@ -103,6 +103,37 @@ def check_cuts_considered(use_cuts): raise CheckError(f"Cuts must be computed for this action, but they are set to {use_cuts.value}") +@make_argcheck +def check_dataset_cuts_match_theorycovmat(dataset, fitthcovmat): + if fitthcovmat: + ds_index = fitthcovmat.load().index.get_level_values(1) + ncovmat = (ds_index == dataset.name).sum() + + cuts = dataset.cuts + if cuts: + ndata = len(dataset.cuts.load()) + else: + ndata = dataset.commondata.ndata + check(ndata == ncovmat) + + +@make_argcheck +def check_experiment_cuts_match_theorycovmat( + experiment, fitthcovmat): + for dataset in experiment.datasets: + if fitthcovmat: + ds_index = fitthcovmat.load().index.get_level_values(1) + ncovmat = (ds_index == dataset.name).sum() + + cuts = dataset.cuts + if cuts: + ndata = len(dataset.cuts.load()) + else: + ndata = dataset.commondata.ndata + check(ndata == ncovmat) + + + @make_argcheck def check_have_two_pdfs(pdfs): check(len(pdfs) == 2,'Expecting exactly two pdfs.') diff --git a/validphys2/src/validphys/comparefittemplates/report.md b/validphys2/src/validphys/comparefittemplates/report.md index 1f7030d63b..d879eb9f93 100644 --- a/validphys2/src/validphys/comparefittemplates/report.md +++ b/validphys2/src/validphys/comparefittemplates/report.md @@ -4,6 +4,10 @@ Fit summary ------------------ {@ summarise_fits @} +Theory Covariance Summary +------------------------- +{@summarise_theory_covmat_fits@} + Dataset properties ------------------ {@current datasets_properties_table@} diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index 7923058879..2de1573470 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -298,6 +298,32 @@ def print_different_cuts(fits, test_for_same_cuts): return res.getvalue() +def fit_theory_covmat_summary(fit, fitthcovmat): + """returns a table with a single column for the `fit`, with three rows + indicating if the theory covariance matrix was used in the 'sampling' of the pseudodata, + the 'fitting', and the 'validphys statistical estimators' in the current namespace for that fit. + """ + try: + config = fit.as_input()['theorycovmatconfig'] + except KeyError: + config = {'use_thcovmat_in_sampling': False, 'use_thcovmat_in_fitting': False} + sampling = config.get('use_thcovmat_in_sampling', False) + fitting = config.get('use_thcovmat_in_fitting', False) + report = (True if fitthcovmat else False) + df = pd.DataFrame( + [sampling, fitting, report], + columns=[fit.name], + index=['sampling', 'fitting', 'validphys statistical estimators']) + return df + +fits_theory_covmat_summary = collect('fit_theory_covmat_summary', ('fits',)) + +@table +def summarise_theory_covmat_fits(fits_theory_covmat_summary): + """Collects the theory covmat summary for all fits and concatenates them into a single table""" + return pd.concat(fits_theory_covmat_summary, axis=1) + + def _get_fitted_index(pdf, i): """Return the nnfit index for the replcia i""" p = pathlib.Path(pdf.infopath).with_name(f'{pdf.name}_{i:04d}.dat') diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index 11802648ea..0c0605e2dd 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -20,7 +20,9 @@ from reportengine import collect from validphys.checks import (check_cuts_considered, check_pdf_is_montecarlo, - check_speclabels_different, check_two_dataspecs) + check_speclabels_different, check_two_dataspecs, + check_dataset_cuts_match_theorycovmat, + check_experiment_cuts_match_theorycovmat) from validphys.core import DataSetSpec, PDF, ExperimentSpec, ThCovMatSpec from validphys.calcutils import (all_chi2, central_chi2, calc_chi2, calc_phi, bootstrap_values, get_df_block) @@ -358,6 +360,7 @@ def closure_pseudodata_replicas(experiments, pdf, nclosure:int, return df #TODO: Add check here that dataset appears in fitthcovmat (if true) and that cuts match +@check_dataset_cuts_match_theorycovmat def covariance_matrix(dataset:DataSetSpec, fitthcovmat, t0set:(PDF, type(None)) = None): """Returns a tuple of Covariance matrix and sqrt covariance matrix for the given dataset which includes theory contribution from scale variations if `use_theorycovmat` is True and @@ -382,8 +385,9 @@ def covariance_matrix(dataset:DataSetSpec, fitthcovmat, t0set:(PDF, type(None)) sqrtcovmat = loaded_data.get_sqrtcovmat() return covmat, sqrtcovmat +@check_experiment_cuts_match_theorycovmat def experiment_covariance_matrix( - experiment: ExperimentSpec, fitthcovmat, t0set:(PDF, type(None)) = None): + experiment: ExperimentSpec, fitthcovmat, t0set:(PDF, type(None)) = None): """Like `covariance_matrix` except for an experiment""" loaded_data = experiment.load() @@ -405,7 +409,11 @@ def experiment_covariance_matrix( return covmat, sqrtcovmat def results(dataset:(DataSetSpec), pdf:PDF, covariance_matrix): - """Tuple of data and theory results for a single pdf. + """Tuple of data and theory results for a single pdf. The data will have an associated + covariance matrix, which can include a contribution from the theory covariance matrix which + is constructed from scale variation. The inclusion of this covariance matrix by default is used + where available, however this behaviour can be modified with the flag `use_theorycovmat`. + The theory is specified as part of the dataset. An experiment is also allowed. (as a result of the C++ code layout).""" From 685e466f3264f69169a6d247b0cd8b2b4ca62afb Mon Sep 17 00:00:00 2001 From: wilsonm Date: Wed, 24 Apr 2019 14:06:26 +0100 Subject: [PATCH 09/13] change default value of use_thcovmat_if_present to be False --- validphys2/src/validphys/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index f6d7dbeaee..05c9a53643 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -696,11 +696,10 @@ def res(*args, **kwargs): return res def produce_fitthcovmat( - self, use_thcovmat_if_present: bool = True, fit: (str, type(None)) = None): + self, use_thcovmat_if_present: bool = False, fit: (str, type(None)) = None): """If a `fit` is specified and `use_thcovmat_if_present` is `True` then returns the corresponding covariance matrix for the given fit if it exists. If the fit doesn't have a - theory covariance matrix then returns `False`. If no fit is specified then returns the user - must manually set `use_thcovmat_if_present` to be False, or provide an appropriate fit. + theory covariance matrix then returns `False`. """ if not isinstance(use_thcovmat_if_present, bool): raise ConfigError("use_thcovmat_if_present should be a boolean, by default it is True") From baf5865b217f0ef67d7cc61346cf0c4a739125c1 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 26 Apr 2019 16:29:20 +0100 Subject: [PATCH 10/13] changed produce_fitthcovmat to be more readable including extra variable names, outputs either None or a ThCovMatSpec --- validphys2/src/validphys/config.py | 16 ++++++++------ validphys2/src/validphys/fitdata.py | 5 +---- validphys2/src/validphys/results.py | 33 +++++++++++++++++------------ 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index 05c9a53643..af35cbbaf9 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -702,7 +702,7 @@ def produce_fitthcovmat( theory covariance matrix then returns `False`. """ if not isinstance(use_thcovmat_if_present, bool): - raise ConfigError("use_thcovmat_if_present should be a boolean, by default it is True") + raise ConfigError("use_thcovmat_if_present should be a boolean, by default it is False") if use_thcovmat_if_present and not fit: raise ConfigError("`use_thcovmat_if_present` was true but no `fit` was specified.") @@ -717,17 +717,19 @@ def produce_fitthcovmat( "`use_thcovmat_in_fitting` didn't exist in the runcard for " f"{fit.name}. Theory covariance matrix will not be used " "in any statistical estimators.") - use_thcovmat_if_present = False + fit_theory_covmat = None #Now set as expected path and check it exists if use_thcovmat_if_present: - use_thcovmat_if_present = ( + covmat_path = ( fit.path/'tables'/'datacuts_theory_theorycovmatconfig_theory_covmat.csv') - if not os.path.exists(use_thcovmat_if_present): + if not os.path.exists(covmat_path): raise ConfigError( "Fit appeared to use theory covmat in fit but the file was not at the " - f"usual location: {use_thcovmat_if_present}.") - use_thcovmat_if_present = ThCovMatSpec(use_thcovmat_if_present) - return use_thcovmat_if_present + f"usual location: {covmat_path}.") + fit_theory_covmat = ThCovMatSpec(covmat_path) + else: + fit_theory_covmat = None + return fit_theory_covmat def parse_speclabel(self, label:(str, type(None))): """A label for a dataspec. To be used in some plots""" diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index 2de1573470..712aa2a738 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -165,9 +165,6 @@ def fit_summary(fit_label, replica_data, total_experiments_chi2data): def summarise_fits(collected_fit_summaries): """ Produces a table of basic comparisons between fits, includes all the fields used in fit_summary """ - log.warning("Using `collected_fit_summaries` which collects over `experiments` defined in fit " - "can affect construction of total covariance matrix, this will be deprecated " - "and should be changed to collect over `data` when that keyword is implemented") return pd.concat(collected_fit_summaries, axis=1) @@ -309,7 +306,7 @@ def fit_theory_covmat_summary(fit, fitthcovmat): config = {'use_thcovmat_in_sampling': False, 'use_thcovmat_in_fitting': False} sampling = config.get('use_thcovmat_in_sampling', False) fitting = config.get('use_thcovmat_in_fitting', False) - report = (True if fitthcovmat else False) + report = bool(fitthcovmat) df = pd.DataFrame( [sampling, fitting, report], columns=[fit.name], diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index 0c0605e2dd..884c5812a1 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -689,28 +689,33 @@ def dataset_chi2_table(chi2_stats, dataset): fits_experiments = collect( 'experiments', ('fits', 'experiments_from_plotting_withcontext',)) -def fit_label(fit, fitthcovmat): +def fit_name_with_covmat_label(fit, fitthcovmat): + """If theory covariance matrix is being used to calculate statistical estimators for the `fit` + then appends (exp + th) onto the fit name for use in legends and column headers to help the user + see what covariance matrix was used to produce the plot or table they are looking at. + """ if fitthcovmat: label = str(fit) + " (exp + th)" else: label = str(fit) return label -fits_label = collect('fit_label', ('fits',)) +fits_name_with_covmat_label = collect('fit_name_with_covmat_label', ('fits',)) #TODO: Possibly get rid of the per_point_data parameter and have separate #actions for absolute and relative tables. @table -def fits_experiments_chi2_table(fits_label, fits_experiments, fits_experiment_chi2_data, - per_point_data:bool=True): +def fits_experiments_chi2_table(fits_name_with_covmat_label, fits_experiments, + fits_experiment_chi2_data, per_point_data:bool=True): """A table with the chi2 for each included experiment in the fits, computed with the theory corresponding to each fit. If points_per_data is True, the chi² will be shown divided by ndata. Otherwise they will be absolute.""" dfs = [] cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') - for label, experiments, exps_chi2 in zip(fits_label, fits_experiments, fits_experiment_chi2_data): + for label, experiments, exps_chi2 in zip( + fits_name_with_covmat_label, fits_experiments, fits_experiment_chi2_data): records = [] for experiment, exp_chi2 in zip(experiments, exps_chi2): mean_chi2 = exp_chi2.central_result.mean() @@ -736,13 +741,14 @@ def fits_experiments_chi2_table(fits_label, fits_experiments, fits_experiment_ch 'experiments_phi', ('fits', 'experiments_from_plotting_withcontext')) @table -def fits_experiments_phi_table(fits_label, fits_experiments, fits_experiments_phi): +def fits_experiments_phi_table(fits_name_with_covmat_label, fits_experiments, fits_experiments_phi): """For every fit, returns phi and number of data points per experiment, where experiment is a collection of datasets grouped according to the experiment key in the PLOTTING info file """ dfs = [] cols = ('ndata', r'$\phi$') - for label, experiments, exps_phi in zip(fits_label, fits_experiments, fits_experiments_phi): + for label, experiments, exps_phi in zip( + fits_name_with_covmat_label, fits_experiments, fits_experiments_phi): records = [] for experiment, (exp_phi, npoints) in zip(experiments, exps_phi): npoints = npoints @@ -774,7 +780,7 @@ def dataspecs_experiments_chi2_table(dataspecs_speclabel, dataspecs_experiments, @table -def fits_datasets_chi2_table(fits_label, fits_experiments, fits_chi2_data, +def fits_datasets_chi2_table(fits_name_with_covmat_label, fits_experiments, fits_chi2_data, per_point_data:bool=True): """A table with the chi2 for each included dataset in the fits, computed with the theory corresponding to the fit. The result are indexed in two @@ -787,7 +793,7 @@ def fits_datasets_chi2_table(fits_label, fits_experiments, fits_chi2_data, cols = ('ndata', r'$\chi^2/ndata$') if per_point_data else ('ndata', r'$\chi^2$') dfs = [] - for label, experiments in zip(fits_label, fits_experiments): + for label, experiments in zip(fits_name_with_covmat_label, fits_experiments): records = [] for experiment in experiments: for dataset, chi2 in zip(experiment.datasets, chi2_it): @@ -844,11 +850,10 @@ def fits_chi2_table( for lv in lvs: dfs.append(pd.concat((edf.loc[lv],ddf.loc[lv]), copy=False, axis=0)) if show_total: - log.warning("Using `fits_total_chi2_data` which collects over `experiments` defined in fit " - "can affect construction of total covariance matrix, this will be deprecated " - "and should be changed to collect over `data` when that keyword is implemented") - total_points = np.array([total_chi2_data.ndata for total_chi2_data in fits_total_chi2_data]) - total_chi = np.array([total_chi2_data.central_result for total_chi2_data in fits_total_chi2_data]) + total_points = np.array( + [total_chi2_data.ndata for total_chi2_data in fits_total_chi2_data]) + total_chi = np.array( + [total_chi2_data.central_result for total_chi2_data in fits_total_chi2_data]) total_chi /= total_points row = np.zeros(len(total_points)*2) row[::2] = total_points From 1d2dc7a1af1d88daaafbd0b6070ce5cd5e258a78 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Fri, 26 Apr 2019 17:44:51 +0100 Subject: [PATCH 11/13] added documentation for running fit and report with th covmat adapted from https://www.wiki.ed.ac.uk/download/attachments/58984316/Theory_covariance_module_documentation.pdf?version=1&modificationDate=1553184077000&api=v2 --- doc/validphys2/guide.md | 222 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) diff --git a/doc/validphys2/guide.md b/doc/validphys2/guide.md index b19109d8ea..5f847f29f6 100644 --- a/doc/validphys2/guide.md +++ b/doc/validphys2/guide.md @@ -1942,6 +1942,228 @@ In addition, errors will be raised if the input directory is not a valid fit (fo If the user wishes to add their own, non-standard files, then it is advisable to avoid using the fit name in these files as the `fitrename` command will also rename these files. +### Fits with a Theory Covariance Matrix + +Fits can be ran with a contribution to the covarince matrix obtained from performing scale +variations. `validphys` also has flags which control the how whether the covariance matrix used to +calculate statistical estimators should include a contribution from the theory covariance matrix. +Getting the various flags that control these behaviours correct in both fit and `validphys` runcards +is critical to getting sensible results. Examples with explanation will be provided here to +demonstrate how to run a fit with a theory covariance matrix and then use the various `validphys` +analysis tools on the fit. + +#### Running a fit with Theory Covariance Matrix + +In order to run a fit with a theory covariance the user must first specify that `datasets` are all +part of the same `experiment`. An example of how this is done in practise is provided with the +`experiments` section of a DIS only fit runcard: + +``` +experiments: +- experiment: BIGEXP + datasets: + - {dataset: NMCPD, frac: 0.5} + - {dataset: NMC, frac: 0.5} + - {dataset: SLACP, frac: 0.5} + - {dataset: SLACD, frac: 0.5} + - {dataset: BCDMSP, frac: 0.5} + - {dataset: BCDMSD, frac: 0.5} + - {dataset: CHORUSNU, frac: 0.5} + - {dataset: CHORUSNB, frac: 0.5} + - {dataset: NTVNUDMN, frac: 0.5} + - {dataset: NTVNBDMN, frac: 0.5} + - {dataset: HERACOMBNCEM, frac: 0.5} + - {dataset: HERACOMBNCEP460, frac: 0.5} + - {dataset: HERACOMBNCEP575, frac: 0.5} + - {dataset: HERACOMBNCEP820, frac: 0.5} + - {dataset: HERACOMBNCEP920, frac: 0.5} + - {dataset: HERACOMBCCEM, frac: 0.5} + - {dataset: HERACOMBCCEP, frac: 0.5} + - {dataset: HERAF2CHARM, frac: 0.5} +``` + +The `datasets` must be part of the same single `experiment` for the theory covariance matrix which +is generated when the user runs `vp-setupfit` is compatible with the fitting infrastructure. + +The next step is to specify the `theorycovmatconfig`. This namespace controls what kind of scale +variation is used to generate the theory covariance matrix, and whether the theory covariance matrix +will be used in the sampling of the pseudodata, the fitting of the data or both. + +The different prescriptions for scale variation are 3-point, 5-point, 5bar-point, 7-point and +9-point, depending on which presciption the user decides to use, they must provide the correct +number and combination of `theoryids`. In addition to this if 5 or 5bar is being used then the user +must specify which of these prescriptions to use with the `fivetheories` flag. There are also two +options for the 7-point presciption, the default is 'Gavin's' prescription but the user can also +specify `seventheories: original`. + +The various configuration options might seem overwhelming, so for each of the presciptions the +appropriate `theoryids` and additional flags required are provided, and can be pasted into a report +runcard. + +--------------------------------------------------------------------- + +##### 3-point + +``` +theorycovmatconfig: + theoryids: + - 163 + - 180 + - 173 + +``` + +##### 5-point + +``` +theorycovmatconfig: + theoryids: + - 163 + - 177 + - 176 + - 179 + - 174 + fivetheories: nobar + +``` + +##### 5bar-point + +``` +theorycovmatconfig: + theoryids: + - 163 + - 180 + - 173 + - 175 + - 178 + fivetheories: bar + +``` + +##### 7-point original + +``` +theorycovmatconfig: + theoryids: + - 163 + - 177 + - 176 + - 179 + - 174 + - 180 + - 173 + seventheories: original +``` + +##### 7-point Gavin (default) + +``` +theorycovmatconfig: + theoryids: + - 163 + - 177 + - 176 + - 179 + - 174 + - 180 + - 173 + +``` + +##### 9-point + +``` +theorycovmatconfig: + theoryids: + - 163 + - 177 + - 176 + - 179 + - 174 + - 180 + - 173 + - 175 + - 178 +``` + +--------------------------------------------------------------------- + +Once the user has correctly specified the `theoryids` and additional flags for their chosen +prescription then the user must specify which PDF will be used to generate the predictions required +to construct the theory covariance matrix and where the theory covariance is to be used. The theory +covariance can be used to sample the pseudodata by setting `use_thcovmat_in_sampling: true`, +likewise the theory covariance can be included in covariance matrix used in the fit by specifying +`use_thcovmat_in_fitting: true`. + +Combining all of the above information, if one wanted to run a fit using the theory covariance, +calculated using the 9-point prescription, in both the fitting and sampling with +`NNPDF31_nlo_as_0118` used to generate the covariance matrix then the complete `theorycovmatconfig` +would be: + +``` +theorycovmatconfig: + theoryids: + - 163 + - 177 + - 176 + - 179 + - 174 + - 180 + - 173 + - 175 + - 178 + pdf: NNPDF31_nlo_as_0118 + use_thcovmat_in_fitting: true + use_thcovmat_in_sampling: true +``` + +#### Using `validphys` statistical estimators with theory covariance + +Once a fit has been ran with the theory covariance, it is necessary to use the theory covariance +matrix in estimators such as calculating the chi² in order to get meaningful values. This behaviour +is controlled by the flag `use_thcovmat_if_present`, which by default the flag is set to `False`. + +If the user specifies `use_thcovmat_if_present: True` then they must also specify a corresponding +`fit`. The configuration file for that `fit` will be read. If `use_thcovmat_in_fitting: true` then +validphys will locate the theory covariance matrix used during the fit and add it to the +experimental covariance matrix, for use in statistical estimators such as chi². A simple example of +this would be + +``` +dataset_input: {dataset: HERAF2CHARM} + +use_thcovmat_if_present: True + +fit: 190310-tg-nlo-global-7pts + +use_cuts: "fromfit" + +pdf: + from_: fit + +theory: + from_: fit + +theoryid: + from_: theory + +actions_: + - dataset_chi2_table +``` + +It should be noted that any `dataset_input` specified in the same runcard that +`use_thcovmat_if_present: True` must have been fitted in the corresponding `fit`. If the +corresponding fit has `use_thcovmat_if_present: False` then the user will be warned and there will +be no contribution from the theory covariance matrix used in calculating statistical estimators for +that runcard. + +Finally, the `use_thcovmat_if_present` flag can be specified at runtime when using the +`vp-comparefits` application. The user can either specify the commandline flag `--theory_cov` which +will set `use_thcovmat_if_present: True` or if the user instead uses the interactive mode +`vp-comparefits -i` then the user will be prompted to select whether or not to use the theory +covariance matrix, if available, in the report. + Parallel mode ------------- From e7a9abb62ec4faddbdb0592c24f7b041fda0620a Mon Sep 17 00:00:00 2001 From: wilsonmr Date: Mon, 29 Apr 2019 15:15:38 +0100 Subject: [PATCH 12/13] updated guide, changed boolean flags for vp-comparefits to work, small fixes --- doc/validphys2/guide.md | 24 +++++++------ validphys2/src/validphys/config.py | 24 +++++++------ validphys2/src/validphys/fitdata.py | 4 +-- .../src/validphys/scripts/vp_comparefits.py | 35 ++++++++++++++----- 4 files changed, 54 insertions(+), 33 deletions(-) diff --git a/doc/validphys2/guide.md b/doc/validphys2/guide.md index 5f847f29f6..8a1458f80e 100644 --- a/doc/validphys2/guide.md +++ b/doc/validphys2/guide.md @@ -1983,10 +1983,10 @@ experiments: ``` The `datasets` must be part of the same single `experiment` for the theory covariance matrix which -is generated when the user runs `vp-setupfit` is compatible with the fitting infrastructure. +is generated when the user runs `vp-setupfit` to be compatible with the fitting infrastructure. -The next step is to specify the `theorycovmatconfig`. This namespace controls what kind of scale -variation is used to generate the theory covariance matrix, and whether the theory covariance matrix +The next step is to specify the `theorycovmatconfig`. This namespace controls which point prescription +is used to generate the theory covariance matrix, and whether the theory covariance matrix will be used in the sampling of the pseudodata, the fitting of the data or both. The different prescriptions for scale variation are 3-point, 5-point, 5bar-point, 7-point and @@ -1997,7 +1997,7 @@ options for the 7-point presciption, the default is 'Gavin's' prescription but t specify `seventheories: original`. The various configuration options might seem overwhelming, so for each of the presciptions the -appropriate `theoryids` and additional flags required are provided, and can be pasted into a report +appropriate `theoryids` and additional flags required are provided below, ready to be pasted into a report runcard. --------------------------------------------------------------------- @@ -2090,8 +2090,8 @@ theorycovmatconfig: --------------------------------------------------------------------- Once the user has correctly specified the `theoryids` and additional flags for their chosen -prescription then the user must specify which PDF will be used to generate the predictions required -to construct the theory covariance matrix and where the theory covariance is to be used. The theory +prescription then the user must specify which PDF will be used to generate the theory 'points' required +to construct the theory covariance matrix. The user must additionally specify where the theory covariance is to be used. The theory covariance can be used to sample the pseudodata by setting `use_thcovmat_in_sampling: true`, likewise the theory covariance can be included in covariance matrix used in the fit by specifying `use_thcovmat_in_fitting: true`. @@ -2125,7 +2125,7 @@ matrix in estimators such as calculating the chi² in order to get meaningful va is controlled by the flag `use_thcovmat_if_present`, which by default the flag is set to `False`. If the user specifies `use_thcovmat_if_present: True` then they must also specify a corresponding -`fit`. The configuration file for that `fit` will be read. If `use_thcovmat_in_fitting: true` then +`fit`. The configuration file for that `fit` will be read. If `use_thcovmat_in_fitting: True` then validphys will locate the theory covariance matrix used during the fit and add it to the experimental covariance matrix, for use in statistical estimators such as chi². A simple example of this would be @@ -2159,10 +2159,12 @@ be no contribution from the theory covariance matrix used in calculating statist that runcard. Finally, the `use_thcovmat_if_present` flag can be specified at runtime when using the -`vp-comparefits` application. The user can either specify the commandline flag `--theory_cov` which -will set `use_thcovmat_if_present: True` or if the user instead uses the interactive mode -`vp-comparefits -i` then the user will be prompted to select whether or not to use the theory -covariance matrix, if available, in the report. +`vp-comparefits` application. The user **must** either specify the commandline flag `--thcovmat_if_present` +or `--no-thcovmat_if_present` which set `use_thcovmat_if_present` to `True` or `False` respectively. + +If the user uses the interactive mode, `vp-comparefits -i`, then they will be prompted to select +whether or not to use the theory covariance matrix, if available, in the report if they have +not already specified on the command line. Parallel mode ------------- diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index af35cbbaf9..57f9a234e7 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -709,7 +709,7 @@ def produce_fitthcovmat( if use_thcovmat_if_present and fit: try: - use_thcovmat_if_present = fit.as_input()[ + thcovmat_present = fit.as_input()[ 'theorycovmatconfig']['use_thcovmat_in_fitting'] except KeyError: #assume covmat wasn't used and fill in key accordingly but warn user @@ -717,16 +717,18 @@ def produce_fitthcovmat( "`use_thcovmat_in_fitting` didn't exist in the runcard for " f"{fit.name}. Theory covariance matrix will not be used " "in any statistical estimators.") - fit_theory_covmat = None - #Now set as expected path and check it exists - if use_thcovmat_if_present: - covmat_path = ( - fit.path/'tables'/'datacuts_theory_theorycovmatconfig_theory_covmat.csv') - if not os.path.exists(covmat_path): - raise ConfigError( - "Fit appeared to use theory covmat in fit but the file was not at the " - f"usual location: {covmat_path}.") - fit_theory_covmat = ThCovMatSpec(covmat_path) + thcovmat_present = False + + + if use_thcovmat_if_present and thcovmat_present: + # Expected path of covmat hardcoded + covmat_path = ( + fit.path/'tables'/'datacuts_theory_theorycovmatconfig_theory_covmat.csv') + if not os.path.exists(covmat_path): + raise ConfigError( + "Fit appeared to use theory covmat in fit but the file was not at the " + f"usual location: {covmat_path}.") + fit_theory_covmat = ThCovMatSpec(covmat_path) else: fit_theory_covmat = None return fit_theory_covmat diff --git a/validphys2/src/validphys/fitdata.py b/validphys2/src/validphys/fitdata.py index 712aa2a738..dded113ae3 100644 --- a/validphys2/src/validphys/fitdata.py +++ b/validphys2/src/validphys/fitdata.py @@ -120,7 +120,7 @@ def replica_data(fit, replica_paths): @table -def fit_summary(fit_label, replica_data, total_experiments_chi2data): +def fit_summary(fit_name_with_covmat_label, replica_data, total_experiments_chi2data): """ Summary table of fit properties - Central chi-squared - Average chi-squared @@ -157,7 +157,7 @@ def fit_summary(fit_label, replica_data, total_experiments_chi2data): (r"$<\chi^2>$", f"{VET(np.mean(member_chi2), np.std(member_chi2))}"), (r"$\phi$", f"{VET(phi, phi_err)}"))) - return pd.Series(data, index=data.keys(), name=fit_label) + return pd.Series(data, index=data.keys(), name=fit_name_with_covmat_label) collected_fit_summaries = collect('fit_summary', ('fits', 'fitcontext')) diff --git a/validphys2/src/validphys/scripts/vp_comparefits.py b/validphys2/src/validphys/scripts/vp_comparefits.py index 07462da8d0..0ff1671a06 100644 --- a/validphys2/src/validphys/scripts/vp_comparefits.py +++ b/validphys2/src/validphys/scripts/vp_comparefits.py @@ -67,14 +67,31 @@ def add_positional_arguments(self, parser): '--closure', help="Use the closure comparison template.", action='store_true') +# parser.add_argument( +# '--use_thcovmat_if_present', +# help="Use theory cov mat for calculating statistical estimators.", +# action='store_true') parser.add_argument( - '--theory_cov', - help="Use theory cov mat for calculating statistical estimators.") + '--thcovmat_if_present', + dest='thcovmat_if_present', + action='store_true', + help="Use theory cov mat for calculating statistical estimators if available.") + parser.add_argument( + '--no-thcovmat_if_present', + dest='thcovmat_if_present', + action='store_false', + help="Do not use theory cov mat for calculating statistical estimators.") + parser.set_defaults(thcovmat_if_present=None) def try_complete_args(self): args = self.args - argnames = ('base_fit', 'reference_fit', 'theory_cov', 'title', 'author', 'keywords') - bad = [argname for argname in argnames if not args[argname]] + argnames = ( + 'base_fit', 'reference_fit', 'title', 'author', 'keywords') + boolnames = ( + 'thcovmat_if_present',) + badargs = [argname for argname in argnames if not args[argname]] + badbools = [bname for bname in boolnames if args[bname] is None] + bad = badargs + badbools if bad and not args['interactive']: sys.exit(f"The following arguments are required: {bad}") try: @@ -84,7 +101,7 @@ def try_complete_args(self): raise KeyboardInterrupt() texts = '\n'.join( f' {argname.replace("_", " ").capitalize()}: {args[argname]}' - for argname in argnames) + for argname in [*argnames, *boolnames]) log.info(f"Starting NNPDF fit comparison:\n{texts}") def interactive_base_fit(self): @@ -126,11 +143,11 @@ def interactive_keywords(self): complete_in_thread=True) return [k.strip() for k in kwinp.split(',') if k] - def interactive_theory_cov(self): + def interactive_thcovmat_if_present(self): """Interactively fill in the `use_thcovmat_if_present` runcard flag. Which is True by default """ - message = ("Do you want to use the fitted covariance matrix (including theory covariance\n" - "matrix) to calculate the statistical estimators? ") + message = ("Do you want to use the theory covariance matrix, if available,\n" + "to calculate the statistical estimators? ") return confirm(message, default=True) def get_commandline_arguments(self, cmdline=None): @@ -178,7 +195,7 @@ def complete_mapping(self): }, 'speclabel': 'Reference Fit' } - autosettings['use_thcovmat_if_present'] = args['theory_cov'] + autosettings['use_thcovmat_if_present'] = args['thcovmat_if_present'] return autosettings From a7ecf11c69eb1302327072688a236a4a14ad5999 Mon Sep 17 00:00:00 2001 From: wilsonm Date: Sat, 4 May 2019 22:55:59 +0100 Subject: [PATCH 13/13] changed production rule names, small fixes to formatting of documentation --- doc/validphys2/guide.md | 133 +++++++++++++++------------- validphys2/src/validphys/config.py | 16 ++-- validphys2/src/validphys/results.py | 8 +- 3 files changed, 84 insertions(+), 73 deletions(-) diff --git a/doc/validphys2/guide.md b/doc/validphys2/guide.md index 8a1458f80e..dcc71b821f 100644 --- a/doc/validphys2/guide.md +++ b/doc/validphys2/guide.md @@ -1944,21 +1944,23 @@ files as the `fitrename` command will also rename these files. ### Fits with a Theory Covariance Matrix -Fits can be ran with a contribution to the covarince matrix obtained from performing scale -variations. `validphys` also has flags which control the how whether the covariance matrix used to -calculate statistical estimators should include a contribution from the theory covariance matrix. -Getting the various flags that control these behaviours correct in both fit and `validphys` runcards -is critical to getting sensible results. Examples with explanation will be provided here to -demonstrate how to run a fit with a theory covariance matrix and then use the various `validphys` -analysis tools on the fit. +Fits can be ran with a contribution to the covarince matrix obtained from +performing scale variations. `validphys` also has flags which control the how +whether the covariance matrix used to calculate statistical estimators should +include a contribution from the theory covariance matrix. Getting the various +flags that control these behaviours correct in both fit and `validphys` runcards +is critical to getting sensible results. Examples with explanation will be +provided here to demonstrate how to run a fit with a theory covariance matrix +and then use the various `validphys` analysis tools on the fit. #### Running a fit with Theory Covariance Matrix -In order to run a fit with a theory covariance the user must first specify that `datasets` are all -part of the same `experiment`. An example of how this is done in practise is provided with the -`experiments` section of a DIS only fit runcard: +In order to run a fit with a theory covariance the user must first specify that +`datasets` are all part of the same `experiment`. An example of how this is done +in practise is provided with the `experiments` section of a DIS only fit +runcard: -``` +```yaml experiments: - experiment: BIGEXP datasets: @@ -1982,29 +1984,32 @@ experiments: - {dataset: HERAF2CHARM, frac: 0.5} ``` -The `datasets` must be part of the same single `experiment` for the theory covariance matrix which -is generated when the user runs `vp-setupfit` to be compatible with the fitting infrastructure. +The `datasets` must be part of the same single `experiment` for the theory +covariance matrix which is generated when the user runs `vp-setupfit` to be +compatible with the fitting infrastructure. -The next step is to specify the `theorycovmatconfig`. This namespace controls which point prescription -is used to generate the theory covariance matrix, and whether the theory covariance matrix -will be used in the sampling of the pseudodata, the fitting of the data or both. +The next step is to specify the `theorycovmatconfig`. This namespace controls +which point prescription is used to generate the theory covariance matrix, and +whether the theory covariance matrix will be used in the sampling of the +pseudodata, the fitting of the data or both. -The different prescriptions for scale variation are 3-point, 5-point, 5bar-point, 7-point and -9-point, depending on which presciption the user decides to use, they must provide the correct -number and combination of `theoryids`. In addition to this if 5 or 5bar is being used then the user -must specify which of these prescriptions to use with the `fivetheories` flag. There are also two -options for the 7-point presciption, the default is 'Gavin's' prescription but the user can also -specify `seventheories: original`. +The different prescriptions for scale variation are 3-point, 5-point, +5bar-point, 7-point and 9-point, depending on which presciption the user decides +to use, they must provide the correct number and combination of `theoryids`. In +addition to this if 5 or 5bar is being used then the user must specify which of +these prescriptions to use with the `fivetheories` flag. There are also two +options for the 7-point presciption, the default is 'Gavin's' prescription but +the user can also specify `seventheories: original`. -The various configuration options might seem overwhelming, so for each of the presciptions the -appropriate `theoryids` and additional flags required are provided below, ready to be pasted into a report -runcard. +The various configuration options might seem overwhelming, so for each of the +presciptions the appropriate `theoryids` and additional flags required are +provided below, ready to be pasted into a report runcard. --------------------------------------------------------------------- ##### 3-point -``` +```yaml theorycovmatconfig: theoryids: - 163 @@ -2015,7 +2020,7 @@ theorycovmatconfig: ##### 5-point -``` +```yaml theorycovmatconfig: theoryids: - 163 @@ -2029,7 +2034,7 @@ theorycovmatconfig: ##### 5bar-point -``` +```yaml theorycovmatconfig: theoryids: - 163 @@ -2043,7 +2048,7 @@ theorycovmatconfig: ##### 7-point original -``` +```yaml theorycovmatconfig: theoryids: - 163 @@ -2058,7 +2063,7 @@ theorycovmatconfig: ##### 7-point Gavin (default) -``` +```yaml theorycovmatconfig: theoryids: - 163 @@ -2073,7 +2078,7 @@ theorycovmatconfig: ##### 9-point -``` +```yaml theorycovmatconfig: theoryids: - 163 @@ -2089,19 +2094,21 @@ theorycovmatconfig: --------------------------------------------------------------------- -Once the user has correctly specified the `theoryids` and additional flags for their chosen -prescription then the user must specify which PDF will be used to generate the theory 'points' required -to construct the theory covariance matrix. The user must additionally specify where the theory covariance is to be used. The theory -covariance can be used to sample the pseudodata by setting `use_thcovmat_in_sampling: true`, -likewise the theory covariance can be included in covariance matrix used in the fit by specifying +Once the user has correctly specified the `theoryids` and additional flags for +their chosen prescription then the user must specify which PDF will be used to +generate the theory 'points' required to construct the theory covariance matrix. +The user must additionally specify where the theory covariance is to be used. +The theory covariance can be used to sample the pseudodata by setting +`use_thcovmat_in_sampling: true`, likewise the theory covariance can be included +in covariance matrix used in the fit by specifying `use_thcovmat_in_fitting: true`. -Combining all of the above information, if one wanted to run a fit using the theory covariance, -calculated using the 9-point prescription, in both the fitting and sampling with -`NNPDF31_nlo_as_0118` used to generate the covariance matrix then the complete `theorycovmatconfig` -would be: +Combining all of the above information, if one wanted to run a fit using the +theory covariance, calculated using the 9-point prescription, in both the +fitting and sampling with `NNPDF31_nlo_as_0118` used to generate the +covariance matrix then the complete `theorycovmatconfig` would be: -``` +```yaml theorycovmatconfig: theoryids: - 163 @@ -2120,17 +2127,19 @@ theorycovmatconfig: #### Using `validphys` statistical estimators with theory covariance -Once a fit has been ran with the theory covariance, it is necessary to use the theory covariance -matrix in estimators such as calculating the chi² in order to get meaningful values. This behaviour -is controlled by the flag `use_thcovmat_if_present`, which by default the flag is set to `False`. +Once a fit has been ran with the theory covariance, it is necessary to use the +theory covariance matrix in estimators such as calculating the chi² in order to +get meaningful values. This behaviour is controlled by the flag +`use_thcovmat_if_present`, which by default the flag is set to `False`. -If the user specifies `use_thcovmat_if_present: True` then they must also specify a corresponding -`fit`. The configuration file for that `fit` will be read. If `use_thcovmat_in_fitting: True` then -validphys will locate the theory covariance matrix used during the fit and add it to the -experimental covariance matrix, for use in statistical estimators such as chi². A simple example of -this would be +If the user specifies `use_thcovmat_if_present: True` then they must also +specify a corresponding `fit`. The configuration file for that `fit` will be +read. If `use_thcovmat_in_fitting: True` then validphys will locate the theory +covariance matrix used during the fit and add it to the experimental +covariance matrix, for use in statistical estimators such as chi². A simple +example of this would be -``` +```yaml dataset_input: {dataset: HERAF2CHARM} use_thcovmat_if_present: True @@ -2153,18 +2162,18 @@ actions_: ``` It should be noted that any `dataset_input` specified in the same runcard that -`use_thcovmat_if_present: True` must have been fitted in the corresponding `fit`. If the -corresponding fit has `use_thcovmat_if_present: False` then the user will be warned and there will -be no contribution from the theory covariance matrix used in calculating statistical estimators for -that runcard. - -Finally, the `use_thcovmat_if_present` flag can be specified at runtime when using the -`vp-comparefits` application. The user **must** either specify the commandline flag `--thcovmat_if_present` -or `--no-thcovmat_if_present` which set `use_thcovmat_if_present` to `True` or `False` respectively. - -If the user uses the interactive mode, `vp-comparefits -i`, then they will be prompted to select -whether or not to use the theory covariance matrix, if available, in the report if they have -not already specified on the command line. +`use_thcovmat_if_present: True` must have been fitted in the corresponding +`fit`. If the corresponding fit has `use_thcovmat_if_present: False` then the +user will be warned and there will be no contribution from the theory covariance +matrix used in calculating statistical estimators for that runcard. + +When using the `vp-comparefits` application, the user **must** either specify +the commandline flag `--thcovmat_if_present` or `--no-thcovmat_if_present` +which set `use_thcovmat_if_present` to `True` or `False` respectively. + +If the user uses the interactive mode, `vp-comparefits -i`, then they will be +prompted to select whether or not to use the theory covariance matrix, if +available, in the report if they have not already specified on the command line. Parallel mode ------------- diff --git a/validphys2/src/validphys/config.py b/validphys2/src/validphys/config.py index 57f9a234e7..b9f6ce2887 100644 --- a/validphys2/src/validphys/config.py +++ b/validphys2/src/validphys/config.py @@ -749,9 +749,10 @@ def parse_fitdeclaration(self, label:str): """ return label - def produce_experiments_from_plotting(self, fit): - """Used to produce experiments from the fit, where each experiment is a group of datasets - according to the experiment key in the plotting info file + def produce_fit_data_groupby_experiment(self, fit): + """Used to produce data from the fit grouped into experiments, + where each experiment is a group of datasets according to the experiment + key in the plotting info file. """ #TODO: consider this an implimentation detail from reportengine.namespaces import NSList @@ -776,15 +777,16 @@ def produce_experiments_from_plotting(self, fit): experiments = NSList(exps, nskey='experiment') return {'experiments': experiments} - def produce_experiments_from_plotting_withcontext(self, fit): - """produces experiments similarly to `experiments_from_plotting` but also sets fitcontext - (pdf and theoryid) + def produce_fit_context_groupby_experiment(self, fit): + """produces experiments similarly to `fit_data_groupby_experiment` + but also sets fitcontext (pdf and theoryid) """ _, pdf = self.parse_from_('fit', 'pdf', write=False) _, theory = self.parse_from_('fit', 'theory', write=False) thid = theory['theoryid'] with self.set_context(ns=self._curr_ns.new_child({'theoryid':thid})): - experiments = self.produce_experiments_from_plotting(fit)['experiments'] + experiments = self.produce_fit_data_groupby_experiment( + fit)['experiments'] return {'pdf': pdf, 'theoryid':thid, 'experiments': experiments} diff --git a/validphys2/src/validphys/results.py b/validphys2/src/validphys/results.py index 884c5812a1..23c3a8d8aa 100644 --- a/validphys2/src/validphys/results.py +++ b/validphys2/src/validphys/results.py @@ -359,7 +359,7 @@ def closure_pseudodata_replicas(experiments, pdf, nclosure:int, return df -#TODO: Add check here that dataset appears in fitthcovmat (if true) and that cuts match + @check_dataset_cuts_match_theorycovmat def covariance_matrix(dataset:DataSetSpec, fitthcovmat, t0set:(PDF, type(None)) = None): """Returns a tuple of Covariance matrix and sqrt covariance matrix for the given dataset @@ -685,9 +685,9 @@ def dataset_chi2_table(chi2_stats, dataset): return pd.DataFrame(chi2_stats, index=[dataset.name]) fits_experiment_chi2_data = collect( - 'experiments_chi2', ('fits', 'experiments_from_plotting_withcontext',)) + 'experiments_chi2', ('fits', 'fit_context_groupby_experiment',)) fits_experiments = collect( - 'experiments', ('fits', 'experiments_from_plotting_withcontext',)) + 'experiments', ('fits', 'fit_context_groupby_experiment',)) def fit_name_with_covmat_label(fit, fitthcovmat): """If theory covariance matrix is being used to calculate statistical estimators for the `fit` @@ -738,7 +738,7 @@ def fits_experiments_chi2_table(fits_name_with_covmat_label, fits_experiments, return res fits_experiments_phi = collect( - 'experiments_phi', ('fits', 'experiments_from_plotting_withcontext')) + 'experiments_phi', ('fits', 'fit_context_groupby_experiment')) @table def fits_experiments_phi_table(fits_name_with_covmat_label, fits_experiments, fits_experiments_phi):