[New commondata format] Jets

buildmaster/ATLAS_1JET_8TEV_R06/data.yaml

@@ -0,0 +1,172 @@
+data_central:
+- 36994.0
+- 15405.6
+- 7009.4
+- 3319.54
+- 1627.69
+- 822.52
+- 421.667
+- 223.264
+- 123.88
+- 69.53
+- 40.254
+- 23.062
+- 13.626
+- 8.337
+- 5.1569
+- 3.1134
+- 1.8922
+- 1.1635
+- 0.7111
+- 0.44279
+- 0.27428
+- 0.17043
+- 0.10614
+- 0.06661
+- 0.041322
+- 0.025443
+- 0.015691
+- 0.009639
+- 0.005995
+- 0.003448
+- 0.0014775
+- 0.0003872
+- 5.009e-05
+- 1.42e-06
+- 35232.5
+- 14221.5
+- 6519.5
+- 3066.07
+- 1499.1
+- 757.76
+- 392.639
+- 206.944
+- 114.55
+- 63.287
+- 36.287
+- 20.861
+- 12.404
+- 7.5669
+- 4.5112
+- 2.7158
+- 1.6382
+- 0.9953
+- 0.6045
+- 0.37186
+- 0.22762
+- 0.13906
+- 0.085314
+- 0.051916
+- 0.031895
+- 0.018985
+- 0.011346
+- 0.006725
+- 0.003879
+- 0.0020569
+- 0.0008497
+- 0.0001838
+- 6.58e-06
+- 30557.3
+- 12452.7
+- 5745.3
+- 2630.59
+- 1263.8
+- 638.88
+- 319.543
+- 165.698
+- 90.26
+- 49.844
+- 28.205
+- 15.919
+- 9.281
+- 5.5009
+- 3.2102
+- 1.8731
+- 1.1101
+- 0.6558
+- 0.38074
+- 0.22072
+- 0.12796
+- 0.07427
+- 0.042121
+- 0.023943
+- 0.013329
+- 0.007287
+- 0.003872
+- 0.0019997
+- 0.0010205
+- 0.0004429
+- 0.0001189
+- 5.37e-06
+- 24622.2
+- 9795.9
+- 4208.2
+- 1970.38
+- 949.4
+- 466.875
+- 229.994
+- 116.89
+- 62.17
+- 33.357
+- 18.199
+- 9.772
+- 5.426
+- 3.0905
+- 1.7167
+- 0.90963
+- 0.5003
+- 0.26684
+- 0.13761
+- 0.07125
+- 0.03547
+- 0.017315
+- 0.0082752
+- 0.003744
+- 0.0016581
+- 0.0006909
+- 0.0002583
+- 0.00011
+- 3.58e-05
+- 3.04e-06
+- 20104.9
+- 7668.9
+- 3167.6
+- 1414.54
+- 653.61
+- 311.066
+- 145.509
+- 69.5
+- 34.69
+- 16.754
+- 8.301
+- 4.009
+- 1.952
+- 0.9391
+- 0.4353
+- 0.1905
+- 0.08173
+- 0.032263
+- 0.012586
+- 0.004426
+- 0.001531
+- 0.0004622
+- 0.0001046
+- 1.348e-05
+- 13644.5
+- 5071.1
+- 2024.3
+- 824.82
+- 353.16
+- 143.567
+- 57.724
+- 22.944
+- 9.31
+- 3.705
+- 1.415
+- 0.5147
+- 0.1678
+- 0.04826
+- 0.01333
+- 0.002652
+- 0.000539
+- 4.6e-05

buildmaster/ATLAS_1JET_8TEV_R06/filter.py

@@ -0,0 +1,170 @@
+import yaml
+import numpy as np
+import pandas as pd
+from filter_utils import get_data_values, get_kinematics, fill_df, decompose_covmat
+
+# ignore pandas warning
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+def filter_ATLAS_1JET_8TEV_data_kinetic():
+    """
+    write kinematics and central data values
+    in kinematics.yaml and data.yaml files
+    respectively.
+    """
+
+    with open("metadata.yaml", "r") as file:
+        metadata = yaml.safe_load(file)
+
+    version = metadata["hepdata"]["version"]
+    tables = metadata["hepdata"]["tables"]
+
+    # get kinematics from hepdata tables
+    kin = get_kinematics(tables, version)
+
+    # get central values from hepdata tables
+    data_central = get_data_values(tables, version)
+
+    data_central_yaml = {"data_central": data_central}
+    kinematics_yaml = {"bins": kin}
+
+    # write central values and kinematics to yaml file
+    with open("data.yaml", "w") as file:
+        yaml.dump(data_central_yaml, file, sort_keys=False)
+
+    with open("kinematics.yaml", "w") as file:
+        yaml.dump(kinematics_yaml, file, sort_keys=False)
+
+
+def filter_ATLAS_1JET_8TEV_uncertainties():
+    """
+    write uncertainties to uncertainties.yaml
+    file.
+    There are three types of uncertainties:
+
+    1. Statistical Uncertainties: CORR
+       -> correlated over the full dataset
+
+    2. Artificial Uncertainties: CORR, these
+       are obtained by following the steps:
+
+       - Construct an Error matrix in which
+         each part of an asymmetric error is considered
+         as as separate error (hence dividing by sqrt(2))
+         see also filter_utils/process_error and
+         filter_utils/HEP_table_to_df
+
+       - Construct covariance matrix from the Error matrix
+
+       - Decompose covariance matrix so as to get ndat art unc
+
+    3. Luminosity Uncertainty: ATLASLUMI12
+       this uncertainty is correlated with all
+       the other ATLASLUMI12 datasets
+    """
+
+    with open("metadata.yaml", "r") as file:
+        metadata = yaml.safe_load(file)
+
+    version = metadata["hepdata"]["version"]
+    tables = metadata["hepdata"]["tables"]
+
+    # get df of uncertainties
+    dfs = []
+    for table in tables:
+        # uncertainties dataframe
+        df = fill_df(table, version)
+        dfs.append(df)
+
+    df_unc = pd.concat([df for df in dfs], axis=0)
+
+    # statistical errors fully uncorrelated
+    stat_errors = df_unc["stat"].to_numpy()
+
+    # luminosity errors
+    lum_errors = df_unc["syst_lumi"].to_numpy()
+
+    # note: in the old implementation some sys are decorrelated.
+    # however the systype file used for the covmat construction is
+    # fully correlated (see difference between systype file in buildmaster and nnpdfcpp).
+
+    A_corr = df_unc.drop(["stat", "syst_lumi"], axis=1).to_numpy() / np.sqrt(2.0)
+    cov_corr = np.einsum("ij,kj->ik", A_corr, A_corr)
+    A_art_corr = decompose_covmat(covmat=cov_corr)
+
+    # error definition
+    error_definition = {
+        f"art_sys_corr_{i}": {
+            "description": f"artificial systematic {i}",
+            "treatment": "ADD",
+            "type": "CORR",
+        }
+        for i in range(1, A_art_corr.shape[0] + 1)
+    }
+
+    error_definition["luminosity_uncertainty"] = {
+        "description": "luminosity uncertainty",
+        "treatment": "ADD",
+        "type": "ATLASLUMI12",
+    }
+
+    error_definition["statistical_uncertainty"] = {
+        "description": "statistical uncertainty",
+        "treatment": "ADD",
+        "type": "UNCORR",
+    }
+
+    # store error in dict
+    error = []
+    for n in range(A_art_corr.shape[0]):
+        error_value = {}
+        for m in range(A_art_corr.shape[1]):
+            error_value[f"art_sys_corr_{m+1}"] = float(A_art_corr[n, m])
+
+        error_value["luminosity_uncertainty"] = float(lum_errors[n])
+        error_value["statistical_uncertainty"] = float(stat_errors[n])
+        error.append(error_value)
+
+    uncertainties_yaml = {"definition": error_definition, "bins": error}
+
+    # write uncertainties to file
+    with open(f"uncertainties.yaml", "w") as file:
+        yaml.dump(uncertainties_yaml, file, sort_keys=False)
+
+    # @@@@@@@@@@@ code below for testing only, should be removed at some point @@@@@@@@@@@@#
+    cov_lum = np.einsum("i,j->ij", lum_errors, lum_errors)
+    cov_stat = np.diag(stat_errors**2)
+
+    covmat = cov_corr + cov_stat + cov_lum
+
+    return np.real(covmat)
+
+
+if __name__ == "__main__":
+    # # write kinematics and central data values
+    # filter_ATLAS_1JET_8TEV_data_kinetic()
+
+    # # write uncertainties file
+    # filter_ATLAS_1JET_8TEV_uncertainties()
+
+    # code below for testing only. Should be removed at some point
+    covmat = filter_ATLAS_1JET_8TEV_uncertainties()
+
+    from validphys.api import API
+
+
+    setname = "ATLAS_1JET_8TEV_R06"
+    dsinps = [
+        {"dataset": setname},
+    ]
+    inp = dict(dataset_inputs=dsinps, theoryid=200, use_cuts="internal")
+    cov = API.dataset_inputs_covmat_from_systematics(**inp)
+
+
+    ones = cov / covmat
+    print(ones)
+    print(np.max(ones), np.min(ones))
+    print(np.allclose(ones, np.ones(cov.shape), rtol=1e-5))