From 006280c4c8ce1c76d46cf614061d096056d43af3 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 1 Mar 2022 12:52:53 +0100 Subject: [PATCH 01/36] Move towards datasets --- data/operator_cards/.gitignore | 1 + data/ymldb/.gitignore | 1 + data/ymldb/load.sh | 2 ++ pineko.toml | 3 +++ src/pineko/cli/__init__.py | 2 +- src/pineko/cli/theory_opcards.py | 17 +++++++++++++++++ src/pineko/configs.py | 4 ++-- 7 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 data/operator_cards/.gitignore create mode 100644 data/ymldb/.gitignore create mode 100644 data/ymldb/load.sh create mode 100644 src/pineko/cli/theory_opcards.py diff --git a/data/operator_cards/.gitignore b/data/operator_cards/.gitignore new file mode 100644 index 00000000..1e82fc7d --- /dev/null +++ b/data/operator_cards/.gitignore @@ -0,0 +1 @@ +*.yaml diff --git a/data/ymldb/.gitignore b/data/ymldb/.gitignore new file mode 100644 index 00000000..1e82fc7d --- /dev/null +++ b/data/ymldb/.gitignore @@ -0,0 +1 @@ +*.yaml diff --git a/data/ymldb/load.sh b/data/ymldb/load.sh new file mode 100644 index 00000000..596f8eca --- /dev/null +++ b/data/ymldb/load.sh @@ -0,0 +1,2 @@ +# copy from remote +# scp dom.mi.infn.it:/media/FK/fktables/data/fktocommon/*.yaml . diff --git a/pineko.toml b/pineko.toml index e69de29b..519169ba 100644 --- a/pineko.toml +++ b/pineko.toml @@ -0,0 +1,3 @@ +[paths] +ymldb = "data/ymldb" +opcards = "data/operator_cards" diff --git a/src/pineko/cli/__init__.py b/src/pineko/cli/__init__.py index d65ff176..ff85cb6d 100644 --- a/src/pineko/cli/__init__.py +++ b/src/pineko/cli/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -from . import check, compare, convolute, opcard +from . import check, compare, convolute, opcard, theory_opcards from ._base import command diff --git a/src/pineko/cli/theory_opcards.py b/src/pineko/cli/theory_opcards.py new file mode 100644 index 00000000..2d254833 --- /dev/null +++ b/src/pineko/cli/theory_opcards.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +import click +import pineappl +import rich +import yaml + +from .. import evolve +from ._base import command + + +@command.command("theory_opcards") +@click.argument("datasets", type=click.STRING, nargs=-1) +def subcommand(datasets): + """Write EKO card for all FK tables in all datasets. + """ + print(datasets) + diff --git a/src/pineko/configs.py b/src/pineko/configs.py index 2ae296d4..a19b3b12 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -83,9 +83,9 @@ def defaults(base_configs): def add_paths(configs): - for key, default in dict(grids="grids", ekos="ekos").items(): + for key in ["ymldb", "opcards"]: if key not in configs.paths: - configs.paths[key] = configs.paths.root / default + raise ValueError(f"Configuration is missing a '{key}' key") elif pathlib.Path(configs.paths[key]).anchor == "": configs.paths[key] = configs.paths.root / configs.paths[key] else: From 1e9cb65c65bce6693e04170fcfca78a79e9f5fad Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 1 Mar 2022 15:40:05 +0100 Subject: [PATCH 02/36] Expose opcard generation for datasets --- data/grids/200/.gitignore | 1 + data/grids/common/.gitignore | 1 + data/grids/load.sh | 8 ++++++++ pineko.toml | 3 +++ src/pineko/cli/opcard.py | 5 +---- src/pineko/cli/theory_opcards.py | 27 +++++++++++++++++-------- src/pineko/configs.py | 2 +- src/pineko/evolve.py | 34 ++++++++++++++++++++++++++++++++ 8 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 data/grids/200/.gitignore create mode 100644 data/grids/common/.gitignore create mode 100644 data/grids/load.sh diff --git a/data/grids/200/.gitignore b/data/grids/200/.gitignore new file mode 100644 index 00000000..9727d870 --- /dev/null +++ b/data/grids/200/.gitignore @@ -0,0 +1 @@ +*.pineappl.lz4 diff --git a/data/grids/common/.gitignore b/data/grids/common/.gitignore new file mode 100644 index 00000000..9727d870 --- /dev/null +++ b/data/grids/common/.gitignore @@ -0,0 +1 @@ +*.pineappl.lz4 diff --git a/data/grids/load.sh b/data/grids/load.sh new file mode 100644 index 00000000..9abf6ec3 --- /dev/null +++ b/data/grids/load.sh @@ -0,0 +1,8 @@ +# copy from remote + +# common example +# scp dom.mi.infn.it:/media/FK/fktables/data/grids/200-LHCB_DY_13TEV_DIELECTRON.pineappl.lz4 ./common/LHCB_DY_13TEV_DIELECTRON.pineappl.lz4 + +# theory example +# scp dom.mi.infn.it:/media/FK/fktables/data/grids/200-SLAC_NC_EM_D_F2.pineappl.lz4 ./200/SLAC_NC_EM_D_F2.pineappl.lz4 +# scp dom.mi.infn.it:/media/FK/fktables/data/grids/200-SLAC_NC_EM_P_F2.pineappl.lz4 ./200/SLAC_NC_EM_P_F2.pineappl.lz4 diff --git a/pineko.toml b/pineko.toml index 519169ba..37be712f 100644 --- a/pineko.toml +++ b/pineko.toml @@ -1,3 +1,6 @@ [paths] ymldb = "data/ymldb" +grids = "data/grids" +grids_common = "data/grids/common" opcards = "data/operator_cards" +opcard_template = "data/operator_cards/_template.yaml" diff --git a/src/pineko/cli/opcard.py b/src/pineko/cli/opcard.py index fccc9a95..66f8d79c 100644 --- a/src/pineko/cli/opcard.py +++ b/src/pineko/cli/opcard.py @@ -18,8 +18,5 @@ def subcommand(pineappl_path, default_card_path, opcard_path): Writes a copy of the default card from DEFAULT_CARD to OPCARD with the adjusted x grid and Q2 grid read from PINEAPPL. """ - pineappl_grid = pineappl.grid.Grid.read(pineappl_path) - with open(default_card_path, "r", encoding="UTF-8") as f: - default_card = yaml.safe_load(f) - _x_grid,q2_grid = evolve.write_operator_card(pineappl_grid, default_card, opcard_path) + _x_grid,q2_grid = evolve.write_operator_card_from_file(pineappl_path, default_card_path, opcard_path) rich.print(f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}") diff --git a/src/pineko/cli/theory_opcards.py b/src/pineko/cli/theory_opcards.py index 2d254833..4294dedd 100644 --- a/src/pineko/cli/theory_opcards.py +++ b/src/pineko/cli/theory_opcards.py @@ -1,17 +1,28 @@ # -*- coding: utf-8 -*- import click -import pineappl import rich -import yaml -from .. import evolve +from .. import parser, configs, evolve from ._base import command @command.command("theory_opcards") +@click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -def subcommand(datasets): - """Write EKO card for all FK tables in all datasets. - """ - print(datasets) - +def subcommand(theory_id, datasets): + """Write EKO card for all FK tables in all datasets.""" + paths = configs.configs.paths + for ds in datasets: + rich.print(f"Analyze {ds}") + try: + _info, grids = parser.get_yaml_information(paths.ymldb / f"{ds}.yaml", paths.grids / str(theory_id)) + except FileNotFoundError: + _info, grids = parser.get_yaml_information(paths.ymldb / f"{ds}.yaml", paths.grids_common) + # the list is still nested, so flatten + grids = [grid for opgrids in grids for grid in opgrids] + for grid in grids: + name = grid.stem.rsplit(".",1)[0] + opcard_path = paths.opcards / f"{name}.yaml" + x_grid, q2_grid = evolve.write_operator_card_from_file(grid, paths.opcard_template,opcard_path) + rich.print(f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}") + print() diff --git a/src/pineko/configs.py b/src/pineko/configs.py index a19b3b12..bff75244 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -83,7 +83,7 @@ def defaults(base_configs): def add_paths(configs): - for key in ["ymldb", "opcards"]: + for key in ["ymldb", "opcards", "grids", "grids_common", "opcard_template"]: if key not in configs.paths: raise ValueError(f"Configuration is missing a '{key}' key") elif pathlib.Path(configs.paths[key]).anchor == "": diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index 7c517e98..39fefee7 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -13,6 +13,33 @@ from . import check, comparator +def write_operator_card_from_file(pineappl_path, default_card_path, card_path): + """Generate operator card for a grid. + + Parameters + ---------- + pineappl_path : str or os.PathLike + path to grid to evolve + default_card : str or os.PathLike + base operator card + card_path : str or os.PathLike + target path + + Returns + ------- + x_grid : np.ndarray + written x grid + q2_grid : np.ndarray + written Q2 grid + """ + # raise in python rather then rust + if not pathlib.Path(pineappl_path).exists(): + raise FileNotFoundError(pineappl_path) + pineappl_grid = pineappl.grid.Grid.read(pineappl_path) + with open(default_card_path, "r", encoding="UTF-8") as f: + default_card = yaml.safe_load(f) + return write_operator_card(pineappl_grid, default_card, card_path) + def write_operator_card(pineappl_grid, default_card, card_path): """Generate operator card for this grid. @@ -24,6 +51,13 @@ def write_operator_card(pineappl_grid, default_card, card_path): base operator card card_path : str or os.PathLike target path + + Returns + ------- + x_grid : np.ndarray + written x grid + q2_grid : np.ndarray + written Q2 grid """ operators_card = copy.deepcopy(default_card) x_grid, _pids, q2_grid = pineappl_grid.axes() From 5ff8e64597be1ec10f486719227a4cb68dc7e535 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 1 Mar 2022 17:29:12 +0100 Subject: [PATCH 03/36] Add theory_ekos --- data/ekos/.gitignore | 1 + data/{grids/200 => fktables}/.gitignore | 0 data/grids/.gitignore | 1 + data/theory_cards/.gitignore | 1 + pineko.toml | 5 ++- src/pineko/cli/__init__.py | 2 +- src/pineko/cli/theory_ekos.py | 33 +++++++++++++++++ src/pineko/cli/theory_opcards.py | 22 ++++++------ src/pineko/configs.py | 16 +++++++-- src/pineko/parser.py | 47 ++++++++++++++++++++++--- 10 files changed, 107 insertions(+), 21 deletions(-) create mode 100644 data/ekos/.gitignore rename data/{grids/200 => fktables}/.gitignore (100%) create mode 100644 data/grids/.gitignore create mode 100644 data/theory_cards/.gitignore create mode 100644 src/pineko/cli/theory_ekos.py diff --git a/data/ekos/.gitignore b/data/ekos/.gitignore new file mode 100644 index 00000000..d874ad67 --- /dev/null +++ b/data/ekos/.gitignore @@ -0,0 +1 @@ +*.tar diff --git a/data/grids/200/.gitignore b/data/fktables/.gitignore similarity index 100% rename from data/grids/200/.gitignore rename to data/fktables/.gitignore diff --git a/data/grids/.gitignore b/data/grids/.gitignore new file mode 100644 index 00000000..9727d870 --- /dev/null +++ b/data/grids/.gitignore @@ -0,0 +1 @@ +*.pineappl.lz4 diff --git a/data/theory_cards/.gitignore b/data/theory_cards/.gitignore new file mode 100644 index 00000000..1e82fc7d --- /dev/null +++ b/data/theory_cards/.gitignore @@ -0,0 +1 @@ +*.yaml diff --git a/pineko.toml b/pineko.toml index 37be712f..c5fe073d 100644 --- a/pineko.toml +++ b/pineko.toml @@ -2,5 +2,8 @@ ymldb = "data/ymldb" grids = "data/grids" grids_common = "data/grids/common" -opcards = "data/operator_cards" +operator_cards = "data/operator_cards" +theory_cards = "data/theory_cards" opcard_template = "data/operator_cards/_template.yaml" +ekos = "data/ekos" +fktables = "data/fktables" diff --git a/src/pineko/cli/__init__.py b/src/pineko/cli/__init__.py index ff85cb6d..b90a367d 100644 --- a/src/pineko/cli/__init__.py +++ b/src/pineko/cli/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -from . import check, compare, convolute, opcard, theory_opcards +from . import check, compare, convolute, opcard, theory_ekos, theory_opcards from ._base import command diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py new file mode 100644 index 00000000..f8207210 --- /dev/null +++ b/src/pineko/cli/theory_ekos.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +import click +import eko +import rich +import yaml + +from .. import configs, parser +from ._base import command + + +@command.command("theory_ekos") +@click.argument("theory_id", type=click.INT) +@click.argument("datasets", type=click.STRING, nargs=-1) +def subcommand(theory_id, datasets): + """Compute EKOs for all FK tables in all datasets.""" + paths = configs.configs.paths + tcard_path = paths.theory_cards / f"{theory_id}.yaml" + with open(tcard_path, encoding="utf-8") as f: + theory_card = yaml.safe_load(f) + eko_path = paths.ekos / str(theory_id) + eko_path.mkdir(exist_ok=True) + for ds in datasets: + rich.print(f"Analyze {ds}") + grids = parser.load_grids(theory_id, ds) + for name, grid in grids.items(): + opcard_path = paths.operator_cards / f"{name}.yaml" + with open(opcard_path, encoding="utf-8") as f: + operators_card = yaml.safe_load(f) + eko_filename = eko_path / f"{name}.tar" + ops = eko.run_dglap(theory_card=theory_card, operators_card=operators_card) + ops.dump_tar(eko_filename) + rich.print(f"[green]Success:[/] Write EKO to {eko_filename}") + print() diff --git a/src/pineko/cli/theory_opcards.py b/src/pineko/cli/theory_opcards.py index 4294dedd..9e3305e4 100644 --- a/src/pineko/cli/theory_opcards.py +++ b/src/pineko/cli/theory_opcards.py @@ -2,7 +2,7 @@ import click import rich -from .. import parser, configs, evolve +from .. import configs, evolve, parser from ._base import command @@ -14,15 +14,13 @@ def subcommand(theory_id, datasets): paths = configs.configs.paths for ds in datasets: rich.print(f"Analyze {ds}") - try: - _info, grids = parser.get_yaml_information(paths.ymldb / f"{ds}.yaml", paths.grids / str(theory_id)) - except FileNotFoundError: - _info, grids = parser.get_yaml_information(paths.ymldb / f"{ds}.yaml", paths.grids_common) - # the list is still nested, so flatten - grids = [grid for opgrids in grids for grid in opgrids] - for grid in grids: - name = grid.stem.rsplit(".",1)[0] - opcard_path = paths.opcards / f"{name}.yaml" - x_grid, q2_grid = evolve.write_operator_card_from_file(grid, paths.opcard_template,opcard_path) - rich.print(f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}") + grids = parser.load_grids(theory_id, ds) + for name, grid in grids.items(): + opcard_path = paths.operator_cards / f"{name}.yaml" + x_grid, q2_grid = evolve.write_operator_card_from_file( + grid, paths.opcard_template, opcard_path + ) + rich.print( + f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}" + ) print() diff --git a/src/pineko/configs.py b/src/pineko/configs.py index bff75244..779a6b63 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -1,10 +1,10 @@ # -*- coding: utf-8 -*- import pathlib import typing -import appdirs -import tomli +import appdirs import rich +import tomli name = "pineko.toml" "Name of the config while (wherever it is placed)" @@ -83,7 +83,16 @@ def defaults(base_configs): def add_paths(configs): - for key in ["ymldb", "opcards", "grids", "grids_common", "opcard_template"]: + for key in [ + "ymldb", + "operator_cards", + "grids", + "grids_common", + "opcard_template", + "theory_cards", + "fktables", + "ekos", + ]: if key not in configs.paths: raise ValueError(f"Configuration is missing a '{key}' key") elif pathlib.Path(configs.paths[key]).anchor == "": @@ -93,6 +102,7 @@ def add_paths(configs): return configs + def detect(path=None): paths = [] diff --git a/src/pineko/parser.py b/src/pineko/parser.py index 0c76f909..9cd14831 100644 --- a/src/pineko/parser.py +++ b/src/pineko/parser.py @@ -1,10 +1,14 @@ -import yaml -# ATTENTION: this is a partial copy from +# -*- coding: utf-8 -*- +# ATTENTION: this is a partial copy from # https://github.com/NNPDF/nnpdf/blob/ec73c9c5d3765c8b600e3015d3f5d6238dd89400/validphys2/src/validphys/fkparser.py +import yaml + +from . import configs ext = "pineappl.lz4" + class PineAPPLEquivalentNotKnown(Exception): pass @@ -13,9 +17,13 @@ class YamlFileNotFound(FileNotFoundError): pass +class GridFileNotFound(FileNotFoundError): + pass + + def _load_yaml(yaml_file): """Load a dataset.yaml file. - + Parameters ---------- yaml_file : Path @@ -70,7 +78,7 @@ def get_yaml_information(yaml_file, grids_folder, check_pineappl=False): for member in operand: p = grids_folder / f"{member}.{ext}" if not p.exists(): - raise FileNotFoundError(f"Failed to find {p}") + raise GridFileNotFound(f"Failed to find {p}") tmp.append(p) ret.append(tmp) @@ -82,3 +90,34 @@ def get_yaml_information(yaml_file, grids_folder, check_pineappl=False): yaml_content["operation_function"] = yaml_content["operation"] return yaml_content, ret + + +def load_grids(theory_id, ds): + """Load all grids (i.e. process scale) of a dataset. + + Parameters + ---------- + theory_id : int + theory id + ds : str + dataset name + + Returns + ------- + grids : dict + mapping basename to path + """ + paths = configs.configs.paths + try: + _info, grids = get_yaml_information( + paths.ymldb / f"{ds}.yaml", paths.grids / str(theory_id) + ) + except GridFileNotFound: + _info, grids = get_yaml_information( + paths.ymldb / f"{ds}.yaml", paths.grids_common + ) + # the list is still nested, so flatten + grids = [grid for opgrids in grids for grid in opgrids] + # then turn into a map name -> path + grids = {grid.stem.rsplit(".", 1)[0]: grid for grid in grids} + return grids From fc6ed748e017ec2d4879ac0162a8661c94b891d6 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 1 Mar 2022 17:51:46 +0100 Subject: [PATCH 04/36] Attempt on logging --- logs/.gitignore | 1 + pineko.toml | 3 +++ src/pineko/cli/theory_ekos.py | 17 ++++++++++++++++- src/pineko/configs.py | 16 +++++++++++++++- 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 logs/.gitignore diff --git a/logs/.gitignore b/logs/.gitignore new file mode 100644 index 00000000..3a36054c --- /dev/null +++ b/logs/.gitignore @@ -0,0 +1 @@ +*.logs diff --git a/pineko.toml b/pineko.toml index c5fe073d..e554a0a5 100644 --- a/pineko.toml +++ b/pineko.toml @@ -7,3 +7,6 @@ theory_cards = "data/theory_cards" opcard_template = "data/operator_cards/_template.yaml" ekos = "data/ekos" fktables = "data/fktables" + +[path.logs] +ekos = "logs/eko" diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py index f8207210..bc77d5a5 100644 --- a/src/pineko/cli/theory_ekos.py +++ b/src/pineko/cli/theory_ekos.py @@ -3,6 +3,7 @@ import eko import rich import yaml +import logging from .. import configs, parser from ._base import command @@ -11,22 +12,36 @@ @command.command("theory_ekos") @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -def subcommand(theory_id, datasets): +@click.option('--logs', is_flag=True, help="dump logs") +def subcommand(theory_id, datasets, logs): """Compute EKOs for all FK tables in all datasets.""" + # setup data paths = configs.configs.paths tcard_path = paths.theory_cards / f"{theory_id}.yaml" with open(tcard_path, encoding="utf-8") as f: theory_card = yaml.safe_load(f) eko_path = paths.ekos / str(theory_id) eko_path.mkdir(exist_ok=True) + # iterate datasets for ds in datasets: rich.print(f"Analyze {ds}") + # iterate grids grids = parser.load_grids(theory_id, ds) for name, grid in grids.items(): opcard_path = paths.operator_cards / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: operators_card = yaml.safe_load(f) eko_filename = eko_path / f"{name}.tar" + # activate logging + if logs and paths.logs.eko: + log_path = paths.logs.eko / f"{theory_id}-{ds}-{name}.log" + logStdout = logging.FileHandler(log_path) + logStdout.setLevel(logging.INFO) + logStdout.setFormatter(logging.Formatter("%(message)s")) + logging.getLogger("eko").handlers = [] + logging.getLogger("eko").addHandler(logStdout) + logging.getLogger("eko").setLevel(logging.INFO) + # do it! ops = eko.run_dglap(theory_card=theory_card, operators_card=operators_card) ops.dump_tar(eko_filename) rich.print(f"[green]Success:[/] Write EKO to {eko_filename}") diff --git a/src/pineko/configs.py b/src/pineko/configs.py index 779a6b63..2b8e8b54 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import pathlib +import pdb import typing import appdirs @@ -94,12 +95,25 @@ def add_paths(configs): "ekos", ]: if key not in configs.paths: - raise ValueError(f"Configuration is missing a '{key}' key") + raise ValueError(f"Configuration is missing a 'paths.{key}' key") elif pathlib.Path(configs.paths[key]).anchor == "": configs.paths[key] = configs.paths.root / configs.paths[key] else: configs.paths[key] = pathlib.Path(configs.paths[key]) + if "logs" not in configs.paths: + configs.paths["logs"] = Configurations() + + for key in [ + "eko", + ]: + if key not in configs.paths.logs: + configs.paths.logs[key] = None + elif pathlib.Path(configs.paths[key]).anchor == "": + configs.paths.logs[key] = configs.paths.root / configs.paths[key] + else: + configs.paths.logs[key] = pathlib.Path(configs.paths[key]) + return configs From 6ee0fb18b045de425bd6ad72ca1e002d2352eb41 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 3 Mar 2022 16:32:22 +0100 Subject: [PATCH 05/36] Drop Configurations class --- logs/.gitignore | 2 +- src/pineko/cli/theory_ekos.py | 12 ++--- src/pineko/cli/theory_opcards.py | 6 +-- src/pineko/configs.py | 75 +++++++------------------------- src/pineko/evolve.py | 2 +- src/pineko/parser.py | 6 +-- 6 files changed, 29 insertions(+), 74 deletions(-) diff --git a/logs/.gitignore b/logs/.gitignore index 3a36054c..397b4a76 100644 --- a/logs/.gitignore +++ b/logs/.gitignore @@ -1 +1 @@ -*.logs +*.log diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py index bc77d5a5..cbe193f5 100644 --- a/src/pineko/cli/theory_ekos.py +++ b/src/pineko/cli/theory_ekos.py @@ -16,11 +16,11 @@ def subcommand(theory_id, datasets, logs): """Compute EKOs for all FK tables in all datasets.""" # setup data - paths = configs.configs.paths - tcard_path = paths.theory_cards / f"{theory_id}.yaml" + paths = configs.configs["paths"] + tcard_path = paths["theory_cards"] / f"{theory_id}.yaml" with open(tcard_path, encoding="utf-8") as f: theory_card = yaml.safe_load(f) - eko_path = paths.ekos / str(theory_id) + eko_path = paths["ekos"] / str(theory_id) eko_path.mkdir(exist_ok=True) # iterate datasets for ds in datasets: @@ -28,13 +28,13 @@ def subcommand(theory_id, datasets, logs): # iterate grids grids = parser.load_grids(theory_id, ds) for name, grid in grids.items(): - opcard_path = paths.operator_cards / f"{name}.yaml" + opcard_path = paths["operator_cards"] / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: operators_card = yaml.safe_load(f) eko_filename = eko_path / f"{name}.tar" # activate logging - if logs and paths.logs.eko: - log_path = paths.logs.eko / f"{theory_id}-{ds}-{name}.log" + if logs and paths["logs"]["eko"]: + log_path = paths["logs"]["eko"] / f"{theory_id}-{ds}-{name}.log" logStdout = logging.FileHandler(log_path) logStdout.setLevel(logging.INFO) logStdout.setFormatter(logging.Formatter("%(message)s")) diff --git a/src/pineko/cli/theory_opcards.py b/src/pineko/cli/theory_opcards.py index 9e3305e4..315ad6c2 100644 --- a/src/pineko/cli/theory_opcards.py +++ b/src/pineko/cli/theory_opcards.py @@ -11,14 +11,14 @@ @click.argument("datasets", type=click.STRING, nargs=-1) def subcommand(theory_id, datasets): """Write EKO card for all FK tables in all datasets.""" - paths = configs.configs.paths + paths = configs.configs["paths"] for ds in datasets: rich.print(f"Analyze {ds}") grids = parser.load_grids(theory_id, ds) for name, grid in grids.items(): - opcard_path = paths.operator_cards / f"{name}.yaml" + opcard_path = paths["operator_cards"] / f"{name}.yaml" x_grid, q2_grid = evolve.write_operator_card_from_file( - grid, paths.opcard_template, opcard_path + grid, paths["opcard_template"], opcard_path ) rich.print( f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}" diff --git a/src/pineko/configs.py b/src/pineko/configs.py index 2b8e8b54..f5d95147 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -1,61 +1,16 @@ # -*- coding: utf-8 -*- import pathlib -import pdb -import typing +import copy import appdirs -import rich import tomli name = "pineko.toml" "Name of the config while (wherever it is placed)" -class Configurations: - def __init__(self, dictionary=None): - if isinstance(dictionary, Configurations): - self._dict = dictionary._dict - elif dictionary is None: - self._dict = {} - else: - self._dict = dictionary - - def __repr__(self): - return self._dict.__repr__() - - def __getattribute__(self, name) -> typing.Any: - if name[0] == "_": - return super().__getattribute__(name) - - value = self._dict[name] - if isinstance(value, dict): - value = Configurations(value) - return value - - def __getitem__(self, key): - return self.__getattribute__(key) - - def __setattribute__(self, name, value): - self._dict[name] = value - - def __setitem__(self, key, value): - if key[0] == "_": - raise LookupError( - "Elements with leading '_' can not be retrieved later, so you" - f" can not set (attempted: '{key}')" - ) - - self._dict[key] = value - - def __contains__(self, item): - return item in self._dict - - def _pprint(self): - rich.print(self._dict) - - # better to declare immediately the correct type -configs = Configurations() +configs = {} "Holds loaded configurations" @@ -76,11 +31,11 @@ def defaults(base_configs): ---- The general rule is to never replace user provided input. """ - configs = Configurations(base_configs) + configs = copy.deepcopy(base_configs) configs = add_paths(configs) - return Configurations(configs) + return configs def add_paths(configs): @@ -94,25 +49,25 @@ def add_paths(configs): "fktables", "ekos", ]: - if key not in configs.paths: + if key not in configs["paths"]: raise ValueError(f"Configuration is missing a 'paths.{key}' key") - elif pathlib.Path(configs.paths[key]).anchor == "": - configs.paths[key] = configs.paths.root / configs.paths[key] + elif pathlib.Path(configs["paths"][key]).anchor == "": + configs["paths"][key] = configs["paths"]["root"] / configs["paths"][key] else: - configs.paths[key] = pathlib.Path(configs.paths[key]) + configs["paths"][key] = pathlib.Path(configs["paths"][key]) - if "logs" not in configs.paths: - configs.paths["logs"] = Configurations() + if "logs" not in configs["paths"]: + configs["paths"]["logs"] = {} for key in [ "eko", ]: - if key not in configs.paths.logs: - configs.paths.logs[key] = None - elif pathlib.Path(configs.paths[key]).anchor == "": - configs.paths.logs[key] = configs.paths.root / configs.paths[key] + if key not in configs["paths"]["logs"]: + configs["paths"]["logs"][key] = None + elif pathlib.Path(configs["paths"][key]).anchor == "": + configs["paths"]["logs"][key] = configs["paths"]["root"] / configs["paths"][key] else: - configs.paths.logs[key] = pathlib.Path(configs.paths[key]) + configs["paths"]["logs"][key] = pathlib.Path(configs["paths"][key]) return configs diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index 39fefee7..89ca8912 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -71,7 +71,7 @@ def evolve_grid( pineappl_path, eko_path, fktable_path, max_as, max_al, comparison_pdf=None ): """ - Invoke steps from file paths. + Convolute grid with EKO from file paths. Parameters ---------- diff --git a/src/pineko/parser.py b/src/pineko/parser.py index 9cd14831..810974d7 100644 --- a/src/pineko/parser.py +++ b/src/pineko/parser.py @@ -107,14 +107,14 @@ def load_grids(theory_id, ds): grids : dict mapping basename to path """ - paths = configs.configs.paths + paths = configs.configs["paths"] try: _info, grids = get_yaml_information( - paths.ymldb / f"{ds}.yaml", paths.grids / str(theory_id) + paths["ymldb"] / f"{ds}.yaml", paths["grids"] / str(theory_id) ) except GridFileNotFound: _info, grids = get_yaml_information( - paths.ymldb / f"{ds}.yaml", paths.grids_common + paths["ymldb"] / f"{ds}.yaml", paths["grids_common"] ) # the list is still nested, so flatten grids = [grid for opgrids in grids for grid in opgrids] From 4eb794713b924b0297cec46e8d3a1d6406055abd Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 3 Mar 2022 16:41:50 +0100 Subject: [PATCH 06/36] Fix eko logging --- pineko.toml | 4 ++-- src/pineko/cli/theory_ekos.py | 2 +- src/pineko/configs.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pineko.toml b/pineko.toml index e554a0a5..8df9fc0a 100644 --- a/pineko.toml +++ b/pineko.toml @@ -8,5 +8,5 @@ opcard_template = "data/operator_cards/_template.yaml" ekos = "data/ekos" fktables = "data/fktables" -[path.logs] -ekos = "logs/eko" +[paths.logs] +eko = "logs/eko" diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py index cbe193f5..ce997500 100644 --- a/src/pineko/cli/theory_ekos.py +++ b/src/pineko/cli/theory_ekos.py @@ -27,7 +27,7 @@ def subcommand(theory_id, datasets, logs): rich.print(f"Analyze {ds}") # iterate grids grids = parser.load_grids(theory_id, ds) - for name, grid in grids.items(): + for name, _grid in grids.items(): opcard_path = paths["operator_cards"] / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: operators_card = yaml.safe_load(f) diff --git a/src/pineko/configs.py b/src/pineko/configs.py index f5d95147..3f2ec1db 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -64,10 +64,10 @@ def add_paths(configs): ]: if key not in configs["paths"]["logs"]: configs["paths"]["logs"][key] = None - elif pathlib.Path(configs["paths"][key]).anchor == "": - configs["paths"]["logs"][key] = configs["paths"]["root"] / configs["paths"][key] + elif pathlib.Path(configs["paths"]["logs"][key]).anchor == "": + configs["paths"]["logs"][key] = configs["paths"]["root"] / configs["paths"]["logs"][key] else: - configs["paths"]["logs"][key] = pathlib.Path(configs["paths"][key]) + configs["paths"]["logs"][key] = pathlib.Path(configs["paths"]["logs"][key]) return configs From f2b3d29f8bd4acabed78d4d00f0071ee367494f3 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 3 Mar 2022 16:53:29 +0100 Subject: [PATCH 07/36] Upgrade syntax with pylint --- .pylintrc | 504 +++++++++++++++++++++++++++++++ README.md | 1 - pyproject.toml | 9 +- run.py | 2 +- run2.py | 1 + src/pineko/__main__.py | 3 +- src/pineko/cli/opcard.py | 14 +- src/pineko/cli/theory_ekos.py | 5 +- src/pineko/cli/theory_opcards.py | 2 +- src/pineko/comparator.py | 2 +- src/pineko/configs.py | 11 +- src/pineko/evolve.py | 9 +- 12 files changed, 537 insertions(+), 26 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..0405a523 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,504 @@ +[MASTER] + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist=numpy,lhapdf,numba,pineappl,eko,yadism + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Add files or directories matching the regex patterns to the blacklist. The +# regex matches against base names, not paths. +ignore-patterns= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. +jobs=1 # has to be 1 as pylint is NOT threadsafe + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +#load-plugins=pylint.extensions.mccabe +#max-complexity=20 + +# Pickle collected data for later comparisons. +persistent=yes + +# Specify a configuration file. +#rcfile= + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable= + print-statement, + parameter-unpacking, + unpacking-in-except, + old-raise-syntax, + backtick, + long-suffix, + old-ne-operator, + old-octal-literal, + import-star-module-level, + raw-checker-failed, + bad-inline-option, + locally-disabled, + locally-enabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + apply-builtin, + basestring-builtin, + buffer-builtin, + cmp-builtin, + coerce-builtin, + execfile-builtin, + file-builtin, + long-builtin, + raw_input-builtin, + reduce-builtin, + standarderror-builtin, + unicode-builtin, + xrange-builtin, + coerce-method, + delslice-method, + getslice-method, + setslice-method, + no-absolute-import, + old-division, + dict-iter-method, + dict-view-method, + next-method-called, + metaclass-assignment, + indexing-exception, + raising-string, + reload-builtin, + oct-method, + hex-method, + nonzero-method, + cmp-method, + input-builtin, + round-builtin, + intern-builtin, + unichr-builtin, + map-builtin-not-iterating, + zip-builtin-not-iterating, + range-builtin-not-iterating, + filter-builtin-not-iterating, + using-cmp-argument, + eq-without-hash, + div-method, + idiv-method, + rdiv-method, + exception-message-attribute, + invalid-str-codec, + sys-max-int, + bad-python3-import, + deprecated-string-function, + deprecated-str-translate-call, + invalid-name, + too-few-public-methods, + too-many-arguments, + bad-continuation, + redefined-outer-name, + missing-docstring, + bad-whitespace, + no-self-use, + no-else-return, + global-statement, + too-many-public-method, + too-many-ancestors + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[REPORTS] + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio).You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Activate the evaluation score. +score=yes + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[BASIC] + +# Naming hint for argument names +argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct argument names +argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for attribute names +attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct attribute names +attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming hint for function names +function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct function names +function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for method names +method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct method names +method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty + +# Naming hint for variable names +variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,future.builtins + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^.*(# )??.*$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# List of optional constructs for which whitespace checking is disabled. `dict- +# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. +# `trailing-comma` allows a space between comma and closing bracket: (a, ). +# `empty-line` allows space-only lines. +no-space-check=trailing-comma,dict-separator + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules=matplotlib.cm,numba + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in a if statement +max-bool-expr=5 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of locals for function / method body +max-locals=22 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of statements in function / method body +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[IMPORTS] + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=optparse,tkinter.tix + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/README.md b/README.md index 9ed5c1ec..eb6f8c60 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ # Pineko This repository contains specifications for the apfelcomb code replacement. - diff --git a/pyproject.toml b/pyproject.toml index 95e7fed4..2437a05f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,8 +27,6 @@ tomli = "^2.0.1" [tool.poetry.dev-dependencies] # code review -black = "^21.6b0" -isort = "^5.9.2" pylint = "^2.9.3" # debug / explore pdbpp = "^0.10.3" @@ -43,11 +41,8 @@ pineko = "pineko:command" [tool.poe.tasks] test = "pytest tests" -lint = "pylint src/*/*.py -E" -lint-warnings = "pylint src/*/*.py --exit-zero" -black = "black ." -isort = "isort ." -format = ["black", "isort"] +lint = "pylint src/**/*.py -E" +lint-warnings = "pylint src/**/*.py --exit-zero" pineko = "pineko" [tool.poetry-dynamic-versioning] diff --git a/run.py b/run.py index bf5fd20b..c183523c 100755 --- a/run.py +++ b/run.py @@ -1,6 +1,6 @@ #!/usr/bin/env python - # -*- coding: utf-8 -*- + import logging import pathlib import sys diff --git a/run2.py b/run2.py index 497a24eb..5a04444a 100644 --- a/run2.py +++ b/run2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from pineko import configs configs.configs = configs.defaults(configs.load()) diff --git a/src/pineko/__main__.py b/src/pineko/__main__.py index a6d31a17..c5dfec07 100644 --- a/src/pineko/__main__.py +++ b/src/pineko/__main__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- from . import command -command() +command() # pylint: disable=no-value-for-parameter +# see https://stackoverflow.com/questions/49680191/click-and-pylint diff --git a/src/pineko/cli/opcard.py b/src/pineko/cli/opcard.py index 66f8d79c..7ffc4dba 100644 --- a/src/pineko/cli/opcard.py +++ b/src/pineko/cli/opcard.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- import click -import pineappl import rich -import yaml from .. import evolve from ._base import command @@ -10,7 +8,9 @@ @command.command("opcard") @click.argument("pineappl_path", metavar="PINEAPPL", type=click.Path(exists=True)) -@click.argument("default_card_path", metavar="DEFAULT_CARD", type=click.Path(exists=True)) +@click.argument( + "default_card_path", metavar="DEFAULT_CARD", type=click.Path(exists=True) +) @click.argument("opcard_path", metavar="OPCARD", type=click.Path()) def subcommand(pineappl_path, default_card_path, opcard_path): """Write EKO card for PineAPPL grid. @@ -18,5 +18,9 @@ def subcommand(pineappl_path, default_card_path, opcard_path): Writes a copy of the default card from DEFAULT_CARD to OPCARD with the adjusted x grid and Q2 grid read from PINEAPPL. """ - _x_grid,q2_grid = evolve.write_operator_card_from_file(pineappl_path, default_card_path, opcard_path) - rich.print(f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}") + _x_grid, q2_grid = evolve.write_operator_card_from_file( + pineappl_path, default_card_path, opcard_path + ) + rich.print( + f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}" + ) diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py index ce997500..7a2b3d68 100644 --- a/src/pineko/cli/theory_ekos.py +++ b/src/pineko/cli/theory_ekos.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- +import logging + import click import eko import rich import yaml -import logging from .. import configs, parser from ._base import command @@ -12,7 +13,7 @@ @command.command("theory_ekos") @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -@click.option('--logs', is_flag=True, help="dump logs") +@click.option("--logs", is_flag=True, help="dump logs") def subcommand(theory_id, datasets, logs): """Compute EKOs for all FK tables in all datasets.""" # setup data diff --git a/src/pineko/cli/theory_opcards.py b/src/pineko/cli/theory_opcards.py index 315ad6c2..30dd045c 100644 --- a/src/pineko/cli/theory_opcards.py +++ b/src/pineko/cli/theory_opcards.py @@ -17,7 +17,7 @@ def subcommand(theory_id, datasets): grids = parser.load_grids(theory_id, ds) for name, grid in grids.items(): opcard_path = paths["operator_cards"] / f"{name}.yaml" - x_grid, q2_grid = evolve.write_operator_card_from_file( + _x_grid, q2_grid = evolve.write_operator_card_from_file( grid, paths["opcard_template"], opcard_path ) rich.print( diff --git a/src/pineko/comparator.py b/src/pineko/comparator.py index c963d612..2cbb28ec 100644 --- a/src/pineko/comparator.py +++ b/src/pineko/comparator.py @@ -1,6 +1,6 @@ +# -*- coding: utf-8 -*- import numpy as np import pandas as pd - import pineappl diff --git a/src/pineko/configs.py b/src/pineko/configs.py index 3f2ec1db..ba7dbad1 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -import pathlib import copy +import pathlib +import sys import appdirs import tomli @@ -51,7 +52,7 @@ def add_paths(configs): ]: if key not in configs["paths"]: raise ValueError(f"Configuration is missing a 'paths.{key}' key") - elif pathlib.Path(configs["paths"][key]).anchor == "": + if pathlib.Path(configs["paths"][key]).anchor == "": configs["paths"][key] = configs["paths"]["root"] / configs["paths"][key] else: configs["paths"][key] = pathlib.Path(configs["paths"][key]) @@ -65,7 +66,9 @@ def add_paths(configs): if key not in configs["paths"]["logs"]: configs["paths"]["logs"][key] = None elif pathlib.Path(configs["paths"]["logs"][key]).anchor == "": - configs["paths"]["logs"][key] = configs["paths"]["root"] / configs["paths"]["logs"][key] + configs["paths"]["logs"][key] = ( + configs["paths"]["root"] / configs["paths"]["logs"][key] + ) else: configs["paths"]["logs"][key] = pathlib.Path(configs["paths"]["logs"][key]) @@ -101,7 +104,7 @@ def load(path=None): return {"paths": {"root": pathlib.Path.cwd()}} else: print("Configuration path specified is not valid.") - quit() + sys.exit() with open(path, "rb") as fd: loaded = tomli.load(fd) diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index 89ca8912..8a412bbb 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -13,9 +13,10 @@ from . import check, comparator + def write_operator_card_from_file(pineappl_path, default_card_path, card_path): """Generate operator card for a grid. - + Parameters ---------- pineappl_path : str or os.PathLike @@ -40,9 +41,10 @@ def write_operator_card_from_file(pineappl_path, default_card_path, card_path): default_card = yaml.safe_load(f) return write_operator_card(pineappl_grid, default_card, card_path) + def write_operator_card(pineappl_grid, default_card, card_path): """Generate operator card for this grid. - + Parameters ---------- pineappl_grid : pineappl.grid.Grid @@ -67,6 +69,7 @@ def write_operator_card(pineappl_grid, default_card, card_path): yaml.safe_dump(operators_card, f) return x_grid, q2_grid + def evolve_grid( pineappl_path, eko_path, fktable_path, max_as, max_al, comparison_pdf=None ): @@ -89,7 +92,7 @@ def evolve_grid( if given, a comparison table (with / without evolution) will be printed """ rich.print( - rich.panel.Panel.fit(f"Computing ...", style="magenta", box=rich.box.SQUARE), + rich.panel.Panel.fit("Computing ...", style="magenta", box=rich.box.SQUARE), f" {pineappl_path}\n", f"+ {eko_path}\n", f"= {fktable_path}", From 1d1d10e9c3c53716e6ac8043b3acc83e9fe4b83f Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 3 Mar 2022 17:31:17 +0100 Subject: [PATCH 08/36] Add theory_fks --- src/pineko/cli/__init__.py | 2 +- src/pineko/cli/theory_ekos.py | 14 ++++++-------- src/pineko/cli/theory_fks.py | 36 +++++++++++++++++++++++++++++++++++ src/pineko/theory_card.py | 24 +++++++++++++++++++++++ 4 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 src/pineko/cli/theory_fks.py create mode 100644 src/pineko/theory_card.py diff --git a/src/pineko/cli/__init__.py b/src/pineko/cli/__init__.py index b90a367d..51b0ea4a 100644 --- a/src/pineko/cli/__init__.py +++ b/src/pineko/cli/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -from . import check, compare, convolute, opcard, theory_ekos, theory_opcards +from . import check, compare, convolute, opcard, theory_ekos, theory_fks, theory_opcards from ._base import command diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py index 7a2b3d68..e40ffa25 100644 --- a/src/pineko/cli/theory_ekos.py +++ b/src/pineko/cli/theory_ekos.py @@ -6,7 +6,7 @@ import rich import yaml -from .. import configs, parser +from .. import configs, parser, theory_card from ._base import command @@ -18,9 +18,7 @@ def subcommand(theory_id, datasets, logs): """Compute EKOs for all FK tables in all datasets.""" # setup data paths = configs.configs["paths"] - tcard_path = paths["theory_cards"] / f"{theory_id}.yaml" - with open(tcard_path, encoding="utf-8") as f: - theory_card = yaml.safe_load(f) + tcard = theory_card.load(theory_id) eko_path = paths["ekos"] / str(theory_id) eko_path.mkdir(exist_ok=True) # iterate datasets @@ -28,10 +26,10 @@ def subcommand(theory_id, datasets, logs): rich.print(f"Analyze {ds}") # iterate grids grids = parser.load_grids(theory_id, ds) - for name, _grid in grids.items(): + for name in grids.keys(): opcard_path = paths["operator_cards"] / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: - operators_card = yaml.safe_load(f) + ocard = yaml.safe_load(f) eko_filename = eko_path / f"{name}.tar" # activate logging if logs and paths["logs"]["eko"]: @@ -43,7 +41,7 @@ def subcommand(theory_id, datasets, logs): logging.getLogger("eko").addHandler(logStdout) logging.getLogger("eko").setLevel(logging.INFO) # do it! - ops = eko.run_dglap(theory_card=theory_card, operators_card=operators_card) + ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) ops.dump_tar(eko_filename) - rich.print(f"[green]Success:[/] Write EKO to {eko_filename}") + rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}") print() diff --git a/src/pineko/cli/theory_fks.py b/src/pineko/cli/theory_fks.py new file mode 100644 index 00000000..bf9294dd --- /dev/null +++ b/src/pineko/cli/theory_fks.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +import click +import rich + +from .. import configs, evolve, parser, theory_card +from ._base import command + + +@command.command("theory_fks") +@click.argument("theory_id", type=click.INT) +@click.argument("datasets", type=click.STRING, nargs=-1) +@click.option("--logs", is_flag=True, help="dump logs") +def subcommand(theory_id, datasets, logs): + """Compute FK tables in all datasets.""" + # setup data + paths = configs.configs["paths"] + tcard = theory_card.load(theory_id) + eko_path = paths["ekos"] / str(theory_id) + fk_path = paths["fktables"] / str(theory_id) + fk_path.mkdir(exist_ok=True) + # iterate datasets + for ds in datasets: + rich.print(f"Analyze {ds}") + # iterate grids + grids = parser.load_grids(theory_id, ds) + for name, grid in grids.items(): + eko_filename = eko_path / f"{name}.tar" + fk_filename = fk_path / f"{ds}-{name}.{parser.ext}" + # activate logging + # if logs and paths["logs"]["eko"]: + evolve.evolve_grid( + grid, eko_filename, fk_filename, 1 + int(tcard["PTO"]), 0 + ) + # do it! + rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}") + print() diff --git a/src/pineko/theory_card.py b/src/pineko/theory_card.py new file mode 100644 index 00000000..d23fe35d --- /dev/null +++ b/src/pineko/theory_card.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +import yaml + +from . import configs + + +def load(theory_id): + """ + Load a theory card. + + Parameters + ---------- + theory_id : int + theory id + + Returns + ------- + theory_card : dict + theory card + """ + tcard_path = configs.configs["paths"]["theory_cards"] / f"{theory_id}.yaml" + with open(tcard_path, encoding="utf-8") as f: + theory_card = yaml.safe_load(f) + return theory_card From 1d3c2b76c3af271a8f96734bc95d23d8265fc3ac Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 3 Mar 2022 18:11:19 +0100 Subject: [PATCH 09/36] Add FK comparison log --- pineko.toml | 1 + src/pineko/cli/theory_fks.py | 24 +++++++++++++++--------- src/pineko/configs.py | 4 +--- src/pineko/evolve.py | 1 + 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/pineko.toml b/pineko.toml index 8df9fc0a..a48137ac 100644 --- a/pineko.toml +++ b/pineko.toml @@ -10,3 +10,4 @@ fktables = "data/fktables" [paths.logs] eko = "logs/eko" +fk = "logs/fk" diff --git a/src/pineko/cli/theory_fks.py b/src/pineko/cli/theory_fks.py index bf9294dd..40726854 100644 --- a/src/pineko/cli/theory_fks.py +++ b/src/pineko/cli/theory_fks.py @@ -2,15 +2,16 @@ import click import rich -from .. import configs, evolve, parser, theory_card +from .. import comparator, configs, evolve, parser, theory_card from ._base import command @command.command("theory_fks") @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -@click.option("--logs", is_flag=True, help="dump logs") -def subcommand(theory_id, datasets, logs): +@click.option("--logs", is_flag=True, help="dump comparison") +@click.option("--pdf", "-p", default=None, help="comparison PDF") +def subcommand(theory_id, datasets, logs, pdf): """Compute FK tables in all datasets.""" # setup data paths = configs.configs["paths"] @@ -23,14 +24,19 @@ def subcommand(theory_id, datasets, logs): rich.print(f"Analyze {ds}") # iterate grids grids = parser.load_grids(theory_id, ds) - for name, grid in grids.items(): + for name, grid_path in grids.items(): eko_filename = eko_path / f"{name}.tar" fk_filename = fk_path / f"{ds}-{name}.{parser.ext}" - # activate logging - # if logs and paths["logs"]["eko"]: - evolve.evolve_grid( - grid, eko_filename, fk_filename, 1 + int(tcard["PTO"]), 0 - ) + max_as = 1 + int(tcard["PTO"]) + max_al = 0 # do it! + grid, fk = evolve.evolve_grid( + grid_path, eko_filename, fk_filename, max_as, max_al + ) + # activate logging + if logs and pdf is not None and paths["logs"]["fk"]: + df = comparator.compare(grid, fk, max_as, max_al, pdf) + logfile = paths["logs"]["fk"] / f"{theory_id}-{ds}-{name}.log" + logfile.write_text(df.to_string()) rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}") print() diff --git a/src/pineko/configs.py b/src/pineko/configs.py index ba7dbad1..fcdbb9ea 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -60,9 +60,7 @@ def add_paths(configs): if "logs" not in configs["paths"]: configs["paths"]["logs"] = {} - for key in [ - "eko", - ]: + for key in ["eko", "fk"]: if key not in configs["paths"]["logs"]: configs["paths"]["logs"][key] = None elif pathlib.Path(configs["paths"]["logs"][key]).anchor == "": diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index 8a412bbb..1a1445f2 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -118,3 +118,4 @@ def evolve_grid( pineappl_grid, fktable, max_as, max_al, comparison_pdf ).to_string() ) + return pineappl_grid, fktable From e7ef4a6fc3f2227369d97ad2d96d0663b6cc00f9 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 9 Mar 2022 14:06:10 +0100 Subject: [PATCH 10/36] Add comparison after evolution --- src/pineko/cli/convolute.py | 4 +++- src/pineko/cli/theory_ekos.py | 2 +- src/pineko/cli/theory_fks.py | 13 ++++++------- src/pineko/comparator.py | 10 +++++++--- src/pineko/evolve.py | 15 ++++++++------- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/pineko/cli/convolute.py b/src/pineko/cli/convolute.py index 226ea5a1..fd26434c 100644 --- a/src/pineko/cli/convolute.py +++ b/src/pineko/cli/convolute.py @@ -22,4 +22,6 @@ def subcommand(pineappl, eko, fktable, max_as, max_al, pdf): Moreover, MAX_AS and MAX_AL are used to specify the order in QCD and QED couplings (i.e. the maximum power allowed for each correction). """ - evolve.evolve_grid(pineappl, eko, fktable, max_as, max_al, pdf) + _grid, _fk, comp = evolve.evolve_grid(pineappl, eko, fktable, max_as, max_al, pdf) + if (comp): + print(comp.to_string()) diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py index e40ffa25..c76b6530 100644 --- a/src/pineko/cli/theory_ekos.py +++ b/src/pineko/cli/theory_ekos.py @@ -33,7 +33,7 @@ def subcommand(theory_id, datasets, logs): eko_filename = eko_path / f"{name}.tar" # activate logging if logs and paths["logs"]["eko"]: - log_path = paths["logs"]["eko"] / f"{theory_id}-{ds}-{name}.log" + log_path = paths["logs"]["eko"] / f"{theory_id}-{name}.log" logStdout = logging.FileHandler(log_path) logStdout.setLevel(logging.INFO) logStdout.setFormatter(logging.Formatter("%(message)s")) diff --git a/src/pineko/cli/theory_fks.py b/src/pineko/cli/theory_fks.py index 40726854..87a4539a 100644 --- a/src/pineko/cli/theory_fks.py +++ b/src/pineko/cli/theory_fks.py @@ -26,17 +26,16 @@ def subcommand(theory_id, datasets, logs, pdf): grids = parser.load_grids(theory_id, ds) for name, grid_path in grids.items(): eko_filename = eko_path / f"{name}.tar" - fk_filename = fk_path / f"{ds}-{name}.{parser.ext}" + fk_filename = fk_path / f"{name}.{parser.ext}" max_as = 1 + int(tcard["PTO"]) max_al = 0 # do it! - grid, fk = evolve.evolve_grid( - grid_path, eko_filename, fk_filename, max_as, max_al + _grid, _fk, comparison = evolve.evolve_grid( + grid_path, eko_filename, fk_filename, max_as, max_al, pdf ) # activate logging - if logs and pdf is not None and paths["logs"]["fk"]: - df = comparator.compare(grid, fk, max_as, max_al, pdf) - logfile = paths["logs"]["fk"] / f"{theory_id}-{ds}-{name}.log" - logfile.write_text(df.to_string()) + if logs and paths["logs"]["fk"] and comparison: + logfile = paths["logs"]["fk"] / f"{theory_id}-{name}-{pdf}.log" + logfile.write_text(comparison.to_string()) rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}") print() diff --git a/src/pineko/comparator.py b/src/pineko/comparator.py index 2cbb28ec..a2ab6320 100644 --- a/src/pineko/comparator.py +++ b/src/pineko/comparator.py @@ -40,10 +40,14 @@ def compare(pine, fktable, max_as, max_al, pdf): df = pd.DataFrame() # add bin info for d in range(pine.bin_dimensions()): - df[f"O{d} left"] = pine.bin_left(d) - df[f"O{d} right"] = pine.bin_right(d) + try: + label = pine.key_values()[f"x{d+1}_label"] + except KeyError: + label = f"O{d+1}" + df[f"{label} left"] = pine.bin_left(d) + df[f"{label} right"] = pine.bin_right(d) # add data df["PineAPPL"] = before df["FkTable"] = after - df["percent_error"] = (after / before - 1.0) * 100.0 + df["permille_error"] = (after / before - 1.0) * 1000.0 return df diff --git a/src/pineko/evolve.py b/src/pineko/evolve.py index eea01ed8..155c7aa8 100644 --- a/src/pineko/evolve.py +++ b/src/pineko/evolve.py @@ -110,13 +110,14 @@ def evolve_grid( order_mask = pineappl.grid.Order.create_mask(pineappl_grid.orders(), max_as, max_al) fktable = pineappl_grid.convolute_eko(operators, "evol", order_mask=order_mask) fktable.optimize() - # write - fktable.write_lz4(str(fktable_path)) # compare before after + comparison = None if comparison_pdf is not None: - print( - comparator.compare( + comparison = comparator.compare( pineappl_grid, fktable, max_as, max_al, comparison_pdf - ).to_string() - ) - return pineappl_grid, fktable + ) + fktable.set_key_value("results_fk", comparison.to_string()) + fktable.set_key_value("results_fk_pdfset", comparison_pdf) + # write + fktable.write_lz4(str(fktable_path)) + return pineappl_grid, fktable, comparison From 512332a47c28ece7646b41c94ede821becf2d789 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 9 Mar 2022 18:11:40 +0100 Subject: [PATCH 11/36] Init TheoryBuilder --- src/pineko/cli/theory_ekos.py | 37 ++------------------ src/pineko/theory.py | 65 +++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 src/pineko/theory.py diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py index c76b6530..c246ae42 100644 --- a/src/pineko/cli/theory_ekos.py +++ b/src/pineko/cli/theory_ekos.py @@ -1,12 +1,7 @@ # -*- coding: utf-8 -*- -import logging - import click -import eko -import rich -import yaml -from .. import configs, parser, theory_card +from .. import theory from ._base import command @@ -16,32 +11,4 @@ @click.option("--logs", is_flag=True, help="dump logs") def subcommand(theory_id, datasets, logs): """Compute EKOs for all FK tables in all datasets.""" - # setup data - paths = configs.configs["paths"] - tcard = theory_card.load(theory_id) - eko_path = paths["ekos"] / str(theory_id) - eko_path.mkdir(exist_ok=True) - # iterate datasets - for ds in datasets: - rich.print(f"Analyze {ds}") - # iterate grids - grids = parser.load_grids(theory_id, ds) - for name in grids.keys(): - opcard_path = paths["operator_cards"] / f"{name}.yaml" - with open(opcard_path, encoding="utf-8") as f: - ocard = yaml.safe_load(f) - eko_filename = eko_path / f"{name}.tar" - # activate logging - if logs and paths["logs"]["eko"]: - log_path = paths["logs"]["eko"] / f"{theory_id}-{name}.log" - logStdout = logging.FileHandler(log_path) - logStdout.setLevel(logging.INFO) - logStdout.setFormatter(logging.Formatter("%(message)s")) - logging.getLogger("eko").handlers = [] - logging.getLogger("eko").addHandler(logStdout) - logging.getLogger("eko").setLevel(logging.INFO) - # do it! - ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) - ops.dump_tar(eko_filename) - rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}") - print() + theory.TheoryBuilder(theory_id, datasets).ekos(logs) diff --git a/src/pineko/theory.py b/src/pineko/theory.py new file mode 100644 index 00000000..5e0e36f6 --- /dev/null +++ b/src/pineko/theory.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +import logging + +import eko +import rich +import yaml + +from . import configs, parser, theory_card + +class TheoryBuilder: + """Common builder application to create the ingredients for a theory. + + Parameters + ---------- + theory_id : int + theory identifier + datsets : list(str) + list of datasets + """ + def __init__(self, theory_id, datasets): + self.theory_id = theory_id + self.datasets = datasets + + @property + def eko_path(self): + """Suffix path with theory id""" + return configs.configs["paths"]["ekos"] / str(self.theory_id) + + def ekos(self, logs): + """Compute all ekos. + + Parameters + ---------- + logs : bool + save eko logs? + """ + # setup data + paths = configs.configs["paths"] + tcard = theory_card.load(self.theory_id) + self.eko_path.mkdir(exist_ok=True) + # iterate datasets + for ds in self.datasets: + rich.print(f"Analyze {ds}") + # iterate grids + grids = parser.load_grids(self.theory_id, ds) + for name in grids.keys(): + opcard_path = paths["operator_cards"] / f"{name}.yaml" + with open(opcard_path, encoding="utf-8") as f: + ocard = yaml.safe_load(f) + eko_filename = self.eko_path / f"{name}.tar" + # activate logging + if logs and paths["logs"]["eko"]: + log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" + logStdout = logging.FileHandler(log_path) + logStdout.setLevel(logging.INFO) + logStdout.setFormatter(logging.Formatter("%(message)s")) + logging.getLogger("eko").handlers = [] + logging.getLogger("eko").addHandler(logStdout) + logging.getLogger("eko").setLevel(logging.INFO) + # do it! + ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) + ops.dump_tar(eko_filename) + if eko_filename.exists(): + rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}") + rich.print() From 096f7605c21933075be65828e6e7c9258101c903 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 10 Mar 2022 13:01:46 +0100 Subject: [PATCH 12/36] Move theory opcards --- src/pineko/cli/theory_opcards.py | 17 ++--------------- src/pineko/theory.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/pineko/cli/theory_opcards.py b/src/pineko/cli/theory_opcards.py index 30dd045c..ee5305e0 100644 --- a/src/pineko/cli/theory_opcards.py +++ b/src/pineko/cli/theory_opcards.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- import click -import rich -from .. import configs, evolve, parser +from .. import theory from ._base import command @@ -11,16 +10,4 @@ @click.argument("datasets", type=click.STRING, nargs=-1) def subcommand(theory_id, datasets): """Write EKO card for all FK tables in all datasets.""" - paths = configs.configs["paths"] - for ds in datasets: - rich.print(f"Analyze {ds}") - grids = parser.load_grids(theory_id, ds) - for name, grid in grids.items(): - opcard_path = paths["operator_cards"] / f"{name}.yaml" - _x_grid, q2_grid = evolve.write_operator_card_from_file( - grid, paths["opcard_template"], opcard_path - ) - rich.print( - f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}" - ) - print() + theory.TheoryBuilder(theory_id, datasets).opcards() diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 5e0e36f6..7183ae66 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -5,7 +5,7 @@ import rich import yaml -from . import configs, parser, theory_card +from . import configs, parser, theory_card, evolve class TheoryBuilder: """Common builder application to create the ingredients for a theory. @@ -26,6 +26,24 @@ def eko_path(self): """Suffix path with theory id""" return configs.configs["paths"]["ekos"] / str(self.theory_id) + def opcards(self): + """Write operator cards.""" + paths = configs.configs["paths"] + for ds in self.datasets: + rich.print(f"Analyze {ds}") + grids = parser.load_grids(self.theory_id, ds) + for name, grid in grids.items(): + opcard_path = paths["operator_cards"] / f"{name}.yaml" + _x_grid, q2_grid = evolve.write_operator_card_from_file( + grid, paths["opcard_template"], opcard_path + ) + if opcard_path.exists(): + rich.print( + f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}" + ) + rich.print() + + def ekos(self, logs): """Compute all ekos. From a0d58fdd12d7440ae7c73cee625bffc2229cbf10 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Mon, 14 Mar 2022 11:54:58 +0100 Subject: [PATCH 13/36] Reshuffle fks into theory --- src/pineko/cli/theory_fks.py | 34 +------- src/pineko/theory.py | 162 +++++++++++++++++++++++++++-------- 2 files changed, 128 insertions(+), 68 deletions(-) diff --git a/src/pineko/cli/theory_fks.py b/src/pineko/cli/theory_fks.py index 87a4539a..150de21c 100644 --- a/src/pineko/cli/theory_fks.py +++ b/src/pineko/cli/theory_fks.py @@ -1,41 +1,15 @@ # -*- coding: utf-8 -*- import click -import rich -from .. import comparator, configs, evolve, parser, theory_card +from .. import theory from ._base import command @command.command("theory_fks") @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -@click.option("--logs", is_flag=True, help="dump comparison") @click.option("--pdf", "-p", default=None, help="comparison PDF") -def subcommand(theory_id, datasets, logs, pdf): +@click.option("--logs", is_flag=True, help="dump comparison") +def subcommand(theory_id, datasets, pdf, logs): """Compute FK tables in all datasets.""" - # setup data - paths = configs.configs["paths"] - tcard = theory_card.load(theory_id) - eko_path = paths["ekos"] / str(theory_id) - fk_path = paths["fktables"] / str(theory_id) - fk_path.mkdir(exist_ok=True) - # iterate datasets - for ds in datasets: - rich.print(f"Analyze {ds}") - # iterate grids - grids = parser.load_grids(theory_id, ds) - for name, grid_path in grids.items(): - eko_filename = eko_path / f"{name}.tar" - fk_filename = fk_path / f"{name}.{parser.ext}" - max_as = 1 + int(tcard["PTO"]) - max_al = 0 - # do it! - _grid, _fk, comparison = evolve.evolve_grid( - grid_path, eko_filename, fk_filename, max_as, max_al, pdf - ) - # activate logging - if logs and paths["logs"]["fk"] and comparison: - logfile = paths["logs"]["fk"] / f"{theory_id}-{name}-{pdf}.log" - logfile.write_text(comparison.to_string()) - rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}") - print() + theory.TheoryBuilder(theory_id, datasets).fks(pdf, logs) diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 7183ae66..d6b45567 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -5,7 +5,8 @@ import rich import yaml -from . import configs, parser, theory_card, evolve +from . import configs, evolve, parser, theory_card + class TheoryBuilder: """Common builder application to create the ingredients for a theory. @@ -17,6 +18,7 @@ class TheoryBuilder: datsets : list(str) list of datasets """ + def __init__(self, theory_id, datasets): self.theory_id = theory_id self.datasets = datasets @@ -26,23 +28,86 @@ def eko_path(self): """Suffix path with theory id""" return configs.configs["paths"]["ekos"] / str(self.theory_id) - def opcards(self): - """Write operator cards.""" - paths = configs.configs["paths"] + @property + def fk_path(self): + """Suffix path with theory id""" + return configs.configs["paths"]["fktables"] / str(self.theory_id) + + def iterate(self, f, **kwargs): + """Iterated grids in datasets. + + Additional keyword arguments are simply passed down. + + Parameters + ---------- + f : callable + iterated callable recieving name and grid as argument + """ for ds in self.datasets: rich.print(f"Analyze {ds}") grids = parser.load_grids(self.theory_id, ds) for name, grid in grids.items(): - opcard_path = paths["operator_cards"] / f"{name}.yaml" - _x_grid, q2_grid = evolve.write_operator_card_from_file( - grid, paths["opcard_template"], opcard_path - ) - if opcard_path.exists(): - rich.print( - f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}" - ) + f(name, grid, **kwargs) rich.print() + def opcard(self, name, grid): + """Write a single operator card. + + Parameters + ---------- + name : str + grid name, i.e. it's true stem + grid : pathlib.Path + path to grid + """ + paths = configs.configs["paths"] + opcard_path = paths["operator_cards"] / f"{name}.yaml" + _x_grid, q2_grid = evolve.write_operator_card_from_file( + grid, paths["opcard_template"], opcard_path + ) + if opcard_path.exists(): + rich.print( + f"[green]Success:[/] Wrote card with {len(q2_grid)} Q2 points to {opcard_path}" + ) + + def opcards(self): + """Write operator cards.""" + self.iterate(self.opcard) + + def eko(self, name, _grid, tcard, logs): + """Compute a single eko. + + Parameters + ---------- + name : str + grid name, i.e. it's true stem + grid : pathlib.Path + path to grid + tcard : dict + theory card + logs : bool + save eko logs? + """ + # setup data + paths = configs.configs["paths"] + opcard_path = paths["operator_cards"] / f"{name}.yaml" + with open(opcard_path, encoding="utf-8") as f: + ocard = yaml.safe_load(f) + eko_filename = self.eko_path / f"{name}.tar" + # activate logging + if logs and paths["logs"]["eko"]: + log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" + logFile = logging.FileHandler(log_path) + logFile.setLevel(logging.INFO) + logFile.setFormatter(logging.Formatter("%(message)s")) + logging.getLogger("eko").handlers = [] + logging.getLogger("eko").addHandler(logFile) + logging.getLogger("eko").setLevel(logging.INFO) + # do it! + ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) + ops.dump_tar(eko_filename) + if eko_filename.exists(): + rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}") def ekos(self, logs): """Compute all ekos. @@ -52,32 +117,53 @@ def ekos(self, logs): logs : bool save eko logs? """ + tcard = theory_card.load(self.theory_id) + self.eko_path.mkdir(exist_ok=True) + self.iterate(self.eko, tcard=tcard, logs=logs) + + def fk(self, name, grid_path, tcard, pdf, logs): + """Compute a single FK table. + + Parameters + ---------- + name : str + grid name, i.e. it's true stem + grid_path : pathlib.Path + path to grid + tcard : dict + theory card + pdf : str + comparison PDF + logs : bool + save eko logs? + """ # setup data paths = configs.configs["paths"] + eko_filename = self.eko_path / f"{name}.tar" + fk_filename = self.fk_path / f"{name}.{parser.ext}" + max_as = 1 + int(tcard["PTO"]) + max_al = 0 + # do it! + _grid, _fk, comparison = evolve.evolve_grid( + grid_path, eko_filename, fk_filename, max_as, max_al, pdf + ) + # activate logging + if logs and paths["logs"]["fk"] and comparison: + logfile = paths["logs"]["fk"] / f"{self.theory_id}-{name}-{pdf}.log" + logfile.write_text(comparison.to_string()) + if fk_filename.exists(): + rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}") + + def fks(self, pdf, logs): + """Compute all FK tables. + + Parameters + ---------- + pdf : str + comparison PDF + logs : bool + save eko logs? + """ tcard = theory_card.load(self.theory_id) - self.eko_path.mkdir(exist_ok=True) - # iterate datasets - for ds in self.datasets: - rich.print(f"Analyze {ds}") - # iterate grids - grids = parser.load_grids(self.theory_id, ds) - for name in grids.keys(): - opcard_path = paths["operator_cards"] / f"{name}.yaml" - with open(opcard_path, encoding="utf-8") as f: - ocard = yaml.safe_load(f) - eko_filename = self.eko_path / f"{name}.tar" - # activate logging - if logs and paths["logs"]["eko"]: - log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" - logStdout = logging.FileHandler(log_path) - logStdout.setLevel(logging.INFO) - logStdout.setFormatter(logging.Formatter("%(message)s")) - logging.getLogger("eko").handlers = [] - logging.getLogger("eko").addHandler(logStdout) - logging.getLogger("eko").setLevel(logging.INFO) - # do it! - ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) - ops.dump_tar(eko_filename) - if eko_filename.exists(): - rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}") - rich.print() + self.fk_path.mkdir(exist_ok=True) + self.iterate(self.fk, tcard=tcard, pdf=pdf, logs=logs) From 784dee8e4a016fc033461fe9ac4e21d1ead67294 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Mon, 14 Mar 2022 12:17:40 +0100 Subject: [PATCH 14/36] Regroup theory cli --- pyproject.toml | 2 +- src/pineko/cli/__init__.py | 2 +- src/pineko/cli/theory_.py | 37 ++++++++++++++++++++++++++++++++ src/pineko/cli/theory_ekos.py | 14 ------------ src/pineko/cli/theory_fks.py | 15 ------------- src/pineko/cli/theory_opcards.py | 13 ----------- 6 files changed, 39 insertions(+), 44 deletions(-) create mode 100644 src/pineko/cli/theory_.py delete mode 100644 src/pineko/cli/theory_ekos.py delete mode 100644 src/pineko/cli/theory_fks.py delete mode 100644 src/pineko/cli/theory_opcards.py diff --git a/pyproject.toml b/pyproject.toml index 51607e44..c77e67ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pineko" version = "0.1.0" -description = "Combine PineAPPL grids and EKOs" +description = "Combine PineAPPL grids and EKOs into FK tables" authors = [ "Alessandro Candido ", "Juan Cruz Martinez ", diff --git a/src/pineko/cli/__init__.py b/src/pineko/cli/__init__.py index 51b0ea4a..943ede4a 100644 --- a/src/pineko/cli/__init__.py +++ b/src/pineko/cli/__init__.py @@ -1,3 +1,3 @@ # -*- coding: utf-8 -*- -from . import check, compare, convolute, opcard, theory_ekos, theory_fks, theory_opcards +from . import check, compare, convolute, opcard, theory_ from ._base import command diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py new file mode 100644 index 00000000..22abf176 --- /dev/null +++ b/src/pineko/cli/theory_.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +import click + +from .. import theory +from ._base import command + + +@command.group("theory") +def theory_(): + """Iterate a subcommand on a given theory and list of datasets""" + + +@theory_.command() +@click.argument("theory_id", type=click.INT) +@click.argument("datasets", type=click.STRING, nargs=-1) +def opcards(theory_id, datasets): + """Write EKO card for all FK tables in all datasets.""" + theory.TheoryBuilder(theory_id, datasets).opcards() + + +@theory_.command() +@click.argument("theory_id", type=click.INT) +@click.argument("datasets", type=click.STRING, nargs=-1) +@click.option("--logs", is_flag=True, help="dump logs") +def ekos(theory_id, datasets, logs): + """Compute EKOs for all FK tables in all datasets.""" + theory.TheoryBuilder(theory_id, datasets).ekos(logs) + + +@theory_.command() +@click.argument("theory_id", type=click.INT) +@click.argument("datasets", type=click.STRING, nargs=-1) +@click.option("--pdf", "-p", default=None, help="comparison PDF") +@click.option("--logs", is_flag=True, help="dump comparison") +def fks(theory_id, datasets, pdf, logs): + """Compute FK tables in all datasets.""" + theory.TheoryBuilder(theory_id, datasets).fks(pdf, logs) diff --git a/src/pineko/cli/theory_ekos.py b/src/pineko/cli/theory_ekos.py deleted file mode 100644 index c246ae42..00000000 --- a/src/pineko/cli/theory_ekos.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -import click - -from .. import theory -from ._base import command - - -@command.command("theory_ekos") -@click.argument("theory_id", type=click.INT) -@click.argument("datasets", type=click.STRING, nargs=-1) -@click.option("--logs", is_flag=True, help="dump logs") -def subcommand(theory_id, datasets, logs): - """Compute EKOs for all FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets).ekos(logs) diff --git a/src/pineko/cli/theory_fks.py b/src/pineko/cli/theory_fks.py deleted file mode 100644 index 150de21c..00000000 --- a/src/pineko/cli/theory_fks.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -import click - -from .. import theory -from ._base import command - - -@command.command("theory_fks") -@click.argument("theory_id", type=click.INT) -@click.argument("datasets", type=click.STRING, nargs=-1) -@click.option("--pdf", "-p", default=None, help="comparison PDF") -@click.option("--logs", is_flag=True, help="dump comparison") -def subcommand(theory_id, datasets, pdf, logs): - """Compute FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets).fks(pdf, logs) diff --git a/src/pineko/cli/theory_opcards.py b/src/pineko/cli/theory_opcards.py deleted file mode 100644 index ee5305e0..00000000 --- a/src/pineko/cli/theory_opcards.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- -import click - -from .. import theory -from ._base import command - - -@command.command("theory_opcards") -@click.argument("theory_id", type=click.INT) -@click.argument("datasets", type=click.STRING, nargs=-1) -def subcommand(theory_id, datasets): - """Write EKO card for all FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets).opcards() From 3a53e564de5b65adc4db50e6aa2ad3a15bbb00ca Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 15 Mar 2022 11:05:36 +0100 Subject: [PATCH 15/36] Update config --- src/pineko/cli/_base.py | 5 ++-- src/pineko/configs.py | 61 ++++++++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/pineko/cli/_base.py b/src/pineko/cli/_base.py index b65a2187..c00b4741 100644 --- a/src/pineko/cli/_base.py +++ b/src/pineko/cli/_base.py @@ -18,7 +18,8 @@ help="Explicitly specify config file (it has to be a valid TOML file).", ) def command(cfg): - base_configs = configs.load(cfg) + path = configs.detect(cfg) + base_configs = configs.load(path) configs.configs = configs.defaults(base_configs) if cfg is not None: - print(f"Configurations loaded from '{cfg}'") + print(f"Configurations loaded from '{path}'") diff --git a/src/pineko/configs.py b/src/pineko/configs.py index fcdbb9ea..650ebbb3 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import copy import pathlib -import sys import appdirs import tomli @@ -28,18 +27,38 @@ def add_scope(base, scope_id, scope): def defaults(base_configs): """Provide additional defaults. + Parameters + ---------- + base_config : dict + user provided configuration + + Returns + ------- + configs : dict + enhanced configuration + Note ---- The general rule is to never replace user provided input. """ configs = copy.deepcopy(base_configs) - configs = add_paths(configs) + enhance_paths(configs) return configs -def add_paths(configs): +def enhance_paths(configs): + """Check required path and enhance them with root path. + + The changes are done inplace. + + Parameters + ---------- + configs : dict + configuration + """ + # required keys without default for key in [ "ymldb", "operator_cards", @@ -57,6 +76,7 @@ def add_paths(configs): else: configs["paths"][key] = pathlib.Path(configs["paths"][key]) + # optional keys which are by default None if "logs" not in configs["paths"]: configs["paths"]["logs"] = {} @@ -70,10 +90,20 @@ def add_paths(configs): else: configs["paths"]["logs"][key] = pathlib.Path(configs["paths"]["logs"][key]) - return configs - def detect(path=None): + """Autodetect configuration file path. + + Parameters + ---------- + path : str or os.PathLike + user provided guess + + Returns + ------- + pathlib.Path : + file path + """ paths = [] if path is not None: @@ -95,14 +125,19 @@ def detect(path=None): def load(path=None): - try: - path = detect(path) - except FileNotFoundError: - if path is None: - return {"paths": {"root": pathlib.Path.cwd()}} - else: - print("Configuration path specified is not valid.") - sys.exit() + """Load config file. + + Parameters + ---------- + path : str or os.PathLike + file path + + Returns + ------- + loaded : dict + configuration dictionary + """ + path = detect(path) with open(path, "rb") as fd: loaded = tomli.load(fd) From cb516d5d7274b69032bba886d5cb38d0393c2eed Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 15 Mar 2022 13:44:52 +0100 Subject: [PATCH 16/36] Add inherit-grids command --- src/pineko/cli/theory_.py | 7 +++++ src/pineko/parser.py | 33 --------------------- src/pineko/theory.py | 62 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 65 insertions(+), 37 deletions(-) diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index 22abf176..002d98f3 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -10,6 +10,13 @@ def theory_(): """Iterate a subcommand on a given theory and list of datasets""" +@theory_.command() +@click.argument("source_theory_id", type=click.INT) +@click.argument("target_theory_id", type=click.INT) +def inherit_grids(source_theory_id, target_theory_id): + """Inherit grids from one theory to another.""" + theory.TheoryBuilder(source_theory_id, ()).inherit_scoped_grids(target_theory_id) + @theory_.command() @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) diff --git a/src/pineko/parser.py b/src/pineko/parser.py index 810974d7..0f227400 100644 --- a/src/pineko/parser.py +++ b/src/pineko/parser.py @@ -4,8 +4,6 @@ import yaml -from . import configs - ext = "pineappl.lz4" @@ -90,34 +88,3 @@ def get_yaml_information(yaml_file, grids_folder, check_pineappl=False): yaml_content["operation_function"] = yaml_content["operation"] return yaml_content, ret - - -def load_grids(theory_id, ds): - """Load all grids (i.e. process scale) of a dataset. - - Parameters - ---------- - theory_id : int - theory id - ds : str - dataset name - - Returns - ------- - grids : dict - mapping basename to path - """ - paths = configs.configs["paths"] - try: - _info, grids = get_yaml_information( - paths["ymldb"] / f"{ds}.yaml", paths["grids"] / str(theory_id) - ) - except GridFileNotFound: - _info, grids = get_yaml_information( - paths["ymldb"] / f"{ds}.yaml", paths["grids_common"] - ) - # the list is still nested, so flatten - grids = [grid for opgrids in grids for grid in opgrids] - # then turn into a map name -> path - grids = {grid.stem.rsplit(".", 1)[0]: grid for grid in grids} - return grids diff --git a/src/pineko/theory.py b/src/pineko/theory.py index d6b45567..2ad5456f 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -5,7 +5,7 @@ import rich import yaml -from . import configs, evolve, parser, theory_card +from . import configs, evolve, parser, theory_card, parser class TheoryBuilder: @@ -25,14 +25,68 @@ def __init__(self, theory_id, datasets): @property def eko_path(self): - """Suffix path with theory id""" + """Suffix paths.ekos with theory id.""" return configs.configs["paths"]["ekos"] / str(self.theory_id) @property def fk_path(self): - """Suffix path with theory id""" + """Suffix paths.fktables with theory id.""" return configs.configs["paths"]["fktables"] / str(self.theory_id) + def grids_scoped_path(self, tid = None): + """Suffix paths.grids with theory id.""" + if tid is None: + tid = self.theory_id + return configs.configs["paths"]["grids"] / str(tid) + + def load_grids(self, ds): + """Load all grids (i.e. process scale) of a dataset. + + Parameters + ---------- + ds : str + dataset name + + Returns + ------- + grids : dict + mapping basename to path + """ + paths = configs.configs["paths"] + try: + _info, grids = parser.get_yaml_information( + paths["ymldb"] / f"{ds}.yaml", self.grids_scoped_path() + ) + except parser.GridFileNotFound: + _info, grids = parser.get_yaml_information( + paths["ymldb"] / f"{ds}.yaml", paths["grids_common"] + ) + # the list is still nested, so flatten + grids = [grid for opgrids in grids for grid in opgrids] + # then turn into a map name -> path + grids = {grid.stem.rsplit(".", 1)[0]: grid for grid in grids} + return grids + + def inherit_scoped_grids(self, target_theory_id): + """Inherit all scoped grids to a new theory. + + Parameters + ---------- + target_theory_id : int + target theory id + """ + other = self.grids_scoped_path(target_theory_id) + other.mkdir(exist_ok=True) + for p in self.grids_scoped_path().glob(f"*.{parser.ext}"): + name = p.stem.rsplit(".", 1)[0] + new = other / f"{name}.{parser.ext}" + # skip existing grids + if new.exists(): + continue + # link + rich.print(f"Linking {name}") + new.symlink_to(p) + def iterate(self, f, **kwargs): """Iterated grids in datasets. @@ -45,7 +99,7 @@ def iterate(self, f, **kwargs): """ for ds in self.datasets: rich.print(f"Analyze {ds}") - grids = parser.load_grids(self.theory_id, ds) + grids = self.load_grids(ds) for name, grid in grids.items(): f(name, grid, **kwargs) rich.print() From 4900d4a269803cafab6d8a23f9d5bbf76706bf24 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 15 Mar 2022 15:01:58 +0100 Subject: [PATCH 17/36] Drop paths.grids.common, add Readme --- .gitignore | 2 + README.md | 68 +++++++++++++++++++++++++++++++- pineko.toml => pineko.debug.toml | 7 ++-- src/pineko/configs.py | 3 +- src/pineko/theory.py | 13 ++---- 5 files changed, 77 insertions(+), 16 deletions(-) rename pineko.toml => pineko.debug.toml (70%) diff --git a/.gitignore b/.gitignore index b6e47617..4cc18b2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +pineko.toml + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index eb6f8c60..c440b4a3 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,67 @@ -# Pineko +# pineko -This repository contains specifications for the apfelcomb code replacement. +`pineko` combines coefficient function grids ("grids" for short) produced by [`runcards`](https://github.com/NNPDF/runcards) +in the form of [`PineAPPL` grids](https://github.com/N3PDF/pineappl) +and Evolution Kernel Operators by [`eko`](https://github.com/N3PDF/eko) +into FK tables. It is one part of the [`APFELcomb`](https://github.com/NNPDF/apfelcomb) replacement. + +## Prerequisites + +Since combines several ingredients into a new object one needs to provide a bunch of files first. + +### pineko.toml + +You need to provide a `pineko.toml`, that provides all necessary paths to the respective tools and output folders. +Look at the example in this repo, that is provided for debug reasons[1]. + +### ymldb + +You need all files of the `ymldb` [2] - look at the respective `load.sh` script. +This defines the mapping from datasets to FK tables. + +### Theory Runcards + +You need to provide the necessary theory runcards named with their respective theory ID inside the `theory_cards` paths [3]. + +### Default Operator Card + +You need to provide a default operator card for `eko` [4]. + +### Coefficient Functions Grids + +`pineko` is **NOT** computing grids, but is taking them as an input. +There are typically two ways to obtain grids: computing them from scratch with [`runcards`](https://github.com/NNPDF/runcards) +or reusing existing ones. + +#### Generate new Grids with `rr` + +You need to run [`rr`](https://github.com/NNPDF/runcards) with a given theory runcard and put the generated grid file with the same name +inside the `/` folder. + +#### Inherit Grids from Existing Theory + +You can reuse the grids from a different theory by using `pineko theory inherit-grids SOURCE_THEORY_ID TARGET_THEORY_ID`. +The relation between the source theory and the target theory is non-trivial [5]. + +## Running `pineko` + +Running `pineko` consists of two steps - each of them being potentially computationally expensive: +computing the EKO and convoluting the EKO with the grid. + +### Computing the EKO + +Again this is a two step process: +1. Generate the necessary operator cards with `pineko theory opcards THEORY_ID DATASET1 DATASET2 ...` +2. Generate the actual EKOs with `pineko theory ekos THEORY_ID DATASET1 DATASET2 ...` + +### Generating the FK Table + +You need to have the EKO computed in the previous step. +Then you can convolute the EKO with the grid by `pineko theory fks THEORY_ID DATASET1 DATASET2 ...` + +--- +[1] Actually, instead we should provide a concise description here - but let's wait to be stable first +[2] this is to be replaced by the new CommonData format +[3] this is to be replaced by a binding to the true theory DB +[4] I'm thinking how to improve this, because how could we provide a study on the interpolation accuracy? at the moment there just equal +[5] examples being SV, different evolution settings, etc. - I'm thinking whether it might be worth to add an `inherit-eko` option diff --git a/pineko.toml b/pineko.debug.toml similarity index 70% rename from pineko.toml rename to pineko.debug.toml index a48137ac..e3bfb614 100644 --- a/pineko.toml +++ b/pineko.debug.toml @@ -1,10 +1,11 @@ [paths] +# inputs ymldb = "data/ymldb" grids = "data/grids" -grids_common = "data/grids/common" -operator_cards = "data/operator_cards" theory_cards = "data/theory_cards" -opcard_template = "data/operator_cards/_template.yaml" +operator_card_template = "data/operator_cards/_template.yaml" +# outputs +operator_cards = "data/operator_cards" ekos = "data/ekos" fktables = "data/fktables" diff --git a/src/pineko/configs.py b/src/pineko/configs.py index 650ebbb3..04d75249 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -63,8 +63,7 @@ def enhance_paths(configs): "ymldb", "operator_cards", "grids", - "grids_common", - "opcard_template", + "operator_card_template", "theory_cards", "fktables", "ekos", diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 2ad5456f..6354af78 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -53,14 +53,9 @@ def load_grids(self, ds): mapping basename to path """ paths = configs.configs["paths"] - try: - _info, grids = parser.get_yaml_information( - paths["ymldb"] / f"{ds}.yaml", self.grids_scoped_path() - ) - except parser.GridFileNotFound: - _info, grids = parser.get_yaml_information( - paths["ymldb"] / f"{ds}.yaml", paths["grids_common"] - ) + _info, grids = parser.get_yaml_information( + paths["ymldb"] / f"{ds}.yaml", self.grids_scoped_path() + ) # the list is still nested, so flatten grids = [grid for opgrids in grids for grid in opgrids] # then turn into a map name -> path @@ -117,7 +112,7 @@ def opcard(self, name, grid): paths = configs.configs["paths"] opcard_path = paths["operator_cards"] / f"{name}.yaml" _x_grid, q2_grid = evolve.write_operator_card_from_file( - grid, paths["opcard_template"], opcard_path + grid, paths["operator_card_template"], opcard_path ) if opcard_path.exists(): rich.print( From e0bffc4c2728d82312092f716107edfb4ab77a1b Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 15 Mar 2022 15:05:08 +0100 Subject: [PATCH 18/36] Fix Readme --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c440b4a3..3ab20856 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ or reusing existing ones. #### Generate new Grids with `rr` -You need to run [`rr`](https://github.com/NNPDF/runcards) with a given theory runcard and put the generated grid file with the same name +You need to run `rr` with a given theory runcard and put the generated grid file with the same name inside the `/` folder. #### Inherit Grids from Existing Theory @@ -60,8 +60,13 @@ You need to have the EKO computed in the previous step. Then you can convolute the EKO with the grid by `pineko theory fks THEORY_ID DATASET1 DATASET2 ...` --- + [1] Actually, instead we should provide a concise description here - but let's wait to be stable first + [2] this is to be replaced by the new CommonData format + [3] this is to be replaced by a binding to the true theory DB + [4] I'm thinking how to improve this, because how could we provide a study on the interpolation accuracy? at the moment there just equal + [5] examples being SV, different evolution settings, etc. - I'm thinking whether it might be worth to add an `inherit-eko` option From 36205d742afa1976a7833e913b7b78429afb1f05 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 15 Mar 2022 15:08:20 +0100 Subject: [PATCH 19/36] Move old stuff --- {data1 => _data}/.gitignore | 0 {data1 => _data}/observable-f2b.yaml | 0 {data1 => _data}/observable-simple.yaml | 0 {data1 => _data}/observable.yaml | 0 {data1 => _data}/operator.yaml | 0 {data1 => _data}/theory_200.yaml | 0 {data1 => _data}/theory_208.yaml | 0 {data1 => _data}/theory_213.yaml | 0 run.py => _run.py | 2 +- data2/.gitignore | 1 - run2.py | 4 ---- 11 files changed, 1 insertion(+), 6 deletions(-) rename {data1 => _data}/.gitignore (100%) rename {data1 => _data}/observable-f2b.yaml (100%) rename {data1 => _data}/observable-simple.yaml (100%) rename {data1 => _data}/observable.yaml (100%) rename {data1 => _data}/operator.yaml (100%) rename {data1 => _data}/theory_200.yaml (100%) rename {data1 => _data}/theory_208.yaml (100%) rename {data1 => _data}/theory_213.yaml (100%) rename run.py => _run.py (97%) delete mode 100644 data2/.gitignore delete mode 100644 run2.py diff --git a/data1/.gitignore b/_data/.gitignore similarity index 100% rename from data1/.gitignore rename to _data/.gitignore diff --git a/data1/observable-f2b.yaml b/_data/observable-f2b.yaml similarity index 100% rename from data1/observable-f2b.yaml rename to _data/observable-f2b.yaml diff --git a/data1/observable-simple.yaml b/_data/observable-simple.yaml similarity index 100% rename from data1/observable-simple.yaml rename to _data/observable-simple.yaml diff --git a/data1/observable.yaml b/_data/observable.yaml similarity index 100% rename from data1/observable.yaml rename to _data/observable.yaml diff --git a/data1/operator.yaml b/_data/operator.yaml similarity index 100% rename from data1/operator.yaml rename to _data/operator.yaml diff --git a/data1/theory_200.yaml b/_data/theory_200.yaml similarity index 100% rename from data1/theory_200.yaml rename to _data/theory_200.yaml diff --git a/data1/theory_208.yaml b/_data/theory_208.yaml similarity index 100% rename from data1/theory_208.yaml rename to _data/theory_208.yaml diff --git a/data1/theory_213.yaml b/_data/theory_213.yaml similarity index 100% rename from data1/theory_213.yaml rename to _data/theory_213.yaml diff --git a/run.py b/_run.py similarity index 97% rename from run.py rename to _run.py index c183523c..5f7ca2e9 100755 --- a/run.py +++ b/_run.py @@ -11,7 +11,7 @@ from pineko import convolute -data = pathlib.Path(__file__).absolute().parent / "data1" +data = pathlib.Path(__file__).absolute().parent / "_data" myoperator_base_path = data / "myoperator.tar" mydis_path = data / "mydis.pineappl.lz4" mydis_yaml_path = data / "mydis.yaml" diff --git a/data2/.gitignore b/data2/.gitignore deleted file mode 100644 index 1e82fc7d..00000000 --- a/data2/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.yaml diff --git a/run2.py b/run2.py deleted file mode 100644 index 5a04444a..00000000 --- a/run2.py +++ /dev/null @@ -1,4 +0,0 @@ -# -*- coding: utf-8 -*- -from pineko import configs - -configs.configs = configs.defaults(configs.load()) From 2e4829e53dd0c19d9349e8c6da2215d6b11be91a Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 15 Mar 2022 15:27:40 +0100 Subject: [PATCH 20/36] Introduce overwrite option and skipping --- README.md | 2 +- src/pineko/cli/theory_.py | 15 +++++++++------ src/pineko/theory.py | 17 ++++++++++++++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3ab20856..a4ceb1dc 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Since combines several ingredients into a new object one needs to provide a bunc ### pineko.toml You need to provide a `pineko.toml`, that provides all necessary paths to the respective tools and output folders. -Look at the example in this repo, that is provided for debug reasons[1]. +Look at the example in this repo, that is provided for debug reasons [1]. ### ymldb diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index 002d98f3..ccde947d 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -20,18 +20,20 @@ def inherit_grids(source_theory_id, target_theory_id): @theory_.command() @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -def opcards(theory_id, datasets): +@click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") +def opcards(theory_id, datasets, overwrite): """Write EKO card for all FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets).opcards() + theory.TheoryBuilder(theory_id, datasets, overwrite).opcards() @theory_.command() @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) @click.option("--logs", is_flag=True, help="dump logs") -def ekos(theory_id, datasets, logs): +@click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") +def ekos(theory_id, datasets, logs, overwrite): """Compute EKOs for all FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets).ekos(logs) + theory.TheoryBuilder(theory_id, datasets, overwrite).ekos(logs) @theory_.command() @@ -39,6 +41,7 @@ def ekos(theory_id, datasets, logs): @click.argument("datasets", type=click.STRING, nargs=-1) @click.option("--pdf", "-p", default=None, help="comparison PDF") @click.option("--logs", is_flag=True, help="dump comparison") -def fks(theory_id, datasets, pdf, logs): +@click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") +def fks(theory_id, datasets, pdf, logs, overwrite): """Compute FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets).fks(pdf, logs) + theory.TheoryBuilder(theory_id, datasets, overwrite).fks(pdf, logs) diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 6354af78..837b0ccc 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -17,11 +17,14 @@ class TheoryBuilder: theory identifier datsets : list(str) list of datasets + overwrite : bool + allow files to be overwritten instead of skipping """ - def __init__(self, theory_id, datasets): + def __init__(self, theory_id, datasets, overwrite=False): self.theory_id = theory_id self.datasets = datasets + self.overwrite = overwrite @property def eko_path(self): @@ -111,6 +114,10 @@ def opcard(self, name, grid): """ paths = configs.configs["paths"] opcard_path = paths["operator_cards"] / f"{name}.yaml" + if opcard_path.exists(): + if not self.overwrite: + rich.print(f"Skipping existing operator card {opcard_path}") + return _x_grid, q2_grid = evolve.write_operator_card_from_file( grid, paths["operator_card_template"], opcard_path ) @@ -143,6 +150,10 @@ def eko(self, name, _grid, tcard, logs): with open(opcard_path, encoding="utf-8") as f: ocard = yaml.safe_load(f) eko_filename = self.eko_path / f"{name}.tar" + if eko_filename.exists(): + if not self.overwrite: + rich.print(f"Skipping existing operator {eko_filename}") + return # activate logging if logs and paths["logs"]["eko"]: log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" @@ -190,6 +201,10 @@ def fk(self, name, grid_path, tcard, pdf, logs): paths = configs.configs["paths"] eko_filename = self.eko_path / f"{name}.tar" fk_filename = self.fk_path / f"{name}.{parser.ext}" + if fk_filename.exists(): + if not self.overwrite: + rich.print(f"Skipping existing FK Table {fk_filename}") + return max_as = 1 + int(tcard["PTO"]) max_al = 0 # do it! From c2e34590d7f51e62b3bfd738cb66ded806d13ab0 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 16 Mar 2022 11:23:48 +0100 Subject: [PATCH 21/36] Make inherit-grids more consistent --- src/pineko/cli/theory_.py | 5 ++-- src/pineko/theory.py | 48 ++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index ccde947d..a6b615a2 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -13,9 +13,10 @@ def theory_(): @theory_.command() @click.argument("source_theory_id", type=click.INT) @click.argument("target_theory_id", type=click.INT) -def inherit_grids(source_theory_id, target_theory_id): +@click.argument("datasets", type=click.STRING, nargs=-1) +def inherit_grids(source_theory_id, target_theory_id, datasets): """Inherit grids from one theory to another.""" - theory.TheoryBuilder(source_theory_id, ()).inherit_scoped_grids(target_theory_id) + theory.TheoryBuilder(source_theory_id, datasets).inherit_grids(target_theory_id) @theory_.command() @click.argument("theory_id", type=click.INT) diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 837b0ccc..222c49e5 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -25,6 +25,11 @@ def __init__(self, theory_id, datasets, overwrite=False): self.theory_id = theory_id self.datasets = datasets self.overwrite = overwrite + + @property + def operator_cards_path(self): + """Suffix paths.operator_cards with theory id.""" + return configs.configs["paths"]["operator_cards"] / str(self.theory_id) @property def eko_path(self): @@ -36,7 +41,7 @@ def fk_path(self): """Suffix paths.fktables with theory id.""" return configs.configs["paths"]["fktables"] / str(self.theory_id) - def grids_scoped_path(self, tid = None): + def grids_path(self, tid = None): """Suffix paths.grids with theory id.""" if tid is None: tid = self.theory_id @@ -57,7 +62,7 @@ def load_grids(self, ds): """ paths = configs.configs["paths"] _info, grids = parser.get_yaml_information( - paths["ymldb"] / f"{ds}.yaml", self.grids_scoped_path() + paths["ymldb"] / f"{ds}.yaml", self.grids_path() ) # the list is still nested, so flatten grids = [grid for opgrids in grids for grid in opgrids] @@ -65,25 +70,33 @@ def load_grids(self, ds): grids = {grid.stem.rsplit(".", 1)[0]: grid for grid in grids} return grids - def inherit_scoped_grids(self, target_theory_id): - """Inherit all scoped grids to a new theory. + def inherit_grid(self, name, grid, other): + """Inherit grids to a new theory. Parameters ---------- target_theory_id : int target theory id """ - other = self.grids_scoped_path(target_theory_id) + new = other / f"{name}.{parser.ext}" + # skip existing grids + if new.exists(): + return + # link + rich.print(f"Linking {name}") + new.symlink_to(grid) + + def inherit_grids(self, target_theory_id): + """Inherit grids to a new theory. + + Parameters + ---------- + target_theory_id : int + target theory id + """ + other = self.grids_path(target_theory_id) other.mkdir(exist_ok=True) - for p in self.grids_scoped_path().glob(f"*.{parser.ext}"): - name = p.stem.rsplit(".", 1)[0] - new = other / f"{name}.{parser.ext}" - # skip existing grids - if new.exists(): - continue - # link - rich.print(f"Linking {name}") - new.symlink_to(p) + self.iterate(self.inherit_grid, other=other) def iterate(self, f, **kwargs): """Iterated grids in datasets. @@ -112,14 +125,13 @@ def opcard(self, name, grid): grid : pathlib.Path path to grid """ - paths = configs.configs["paths"] - opcard_path = paths["operator_cards"] / f"{name}.yaml" + opcard_path = self.operator_cards_path / f"{name}.yaml" if opcard_path.exists(): if not self.overwrite: rich.print(f"Skipping existing operator card {opcard_path}") return _x_grid, q2_grid = evolve.write_operator_card_from_file( - grid, paths["operator_card_template"], opcard_path + grid, configs.configs["paths"]["operator_card_template"], opcard_path ) if opcard_path.exists(): rich.print( @@ -146,7 +158,7 @@ def eko(self, name, _grid, tcard, logs): """ # setup data paths = configs.configs["paths"] - opcard_path = paths["operator_cards"] / f"{name}.yaml" + opcard_path = self.operator_cards_path / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: ocard = yaml.safe_load(f) eko_filename = self.eko_path / f"{name}.tar" From 1e35e50e00ca1c2f8fdc3d379f761faaa121a84c Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 16 Mar 2022 11:36:38 +0100 Subject: [PATCH 22/36] Make inherit-grids yet more consistent --- src/pineko/cli/theory_.py | 6 ++++-- src/pineko/theory.py | 18 +++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index a6b615a2..6c3e6556 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -14,9 +14,11 @@ def theory_(): @click.argument("source_theory_id", type=click.INT) @click.argument("target_theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -def inherit_grids(source_theory_id, target_theory_id, datasets): +@click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") +def inherit_grids(source_theory_id, target_theory_id, datasets, overwrite): """Inherit grids from one theory to another.""" - theory.TheoryBuilder(source_theory_id, datasets).inherit_grids(target_theory_id) + theory.TheoryBuilder(source_theory_id, datasets, overwrite).inherit_grids(target_theory_id) + @theory_.command() @click.argument("theory_id", type=click.INT) diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 222c49e5..0b3a43ba 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -79,12 +79,16 @@ def inherit_grid(self, name, grid, other): target theory id """ new = other / f"{name}.{parser.ext}" - # skip existing grids if new.exists(): - return + if not self.overwrite: + rich.print(f"Skipping existing grid {new}") + return + else: + new.unlink() # link - rich.print(f"Linking {name}") new.symlink_to(grid) + if new.exists(): + rich.print(f"[green]Success:[/] Created link at {new}") def inherit_grids(self, target_theory_id): """Inherit grids to a new theory. @@ -169,11 +173,11 @@ def eko(self, name, _grid, tcard, logs): # activate logging if logs and paths["logs"]["eko"]: log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" - logFile = logging.FileHandler(log_path) - logFile.setLevel(logging.INFO) - logFile.setFormatter(logging.Formatter("%(message)s")) + log_file = logging.FileHandler(log_path) + log_file.setLevel(logging.INFO) + log_file.setFormatter(logging.Formatter("%(message)s")) logging.getLogger("eko").handlers = [] - logging.getLogger("eko").addHandler(logFile) + logging.getLogger("eko").addHandler(log_file) logging.getLogger("eko").setLevel(logging.INFO) # do it! ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) From 11e52ae36e91e921b2b0a6684b5d824cdc4111f3 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 16 Mar 2022 11:46:35 +0100 Subject: [PATCH 23/36] Update Readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a4ceb1dc..e8262d88 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,11 @@ Since combines several ingredients into a new object one needs to provide a bunc ### pineko.toml You need to provide a `pineko.toml`, that provides all necessary paths to the respective tools and output folders. -Look at the example in this repo, that is provided for debug reasons [1]. +[DEBUG: Look at the debug example in this repo [1]]. ### ymldb -You need all files of the `ymldb` [2] - look at the respective `load.sh` script. +You need all files of the `ymldb` [2]. [DEBUG: Look at the respective `load.sh` script to load from dom.] This defines the mapping from datasets to FK tables. ### Theory Runcards @@ -26,6 +26,7 @@ You need to provide the necessary theory runcards named with their respective th ### Default Operator Card You need to provide a default operator card for `eko` [4]. +[DEBUG: Look at the respective `load.sh` script to load from dom.] ### Coefficient Functions Grids @@ -36,11 +37,11 @@ or reusing existing ones. #### Generate new Grids with `rr` You need to run `rr` with a given theory runcard and put the generated grid file with the same name -inside the `/` folder. +inside the `/` folder. The name has to match the `ymldb` which is the case by default. #### Inherit Grids from Existing Theory -You can reuse the grids from a different theory by using `pineko theory inherit-grids SOURCE_THEORY_ID TARGET_THEORY_ID`. +You can reuse the grids from a different theory by using `pineko theory inherit-grids SOURCE_THEORY_ID TARGET_THEORY_ID DATASET1 DATASET2 ...`. The relation between the source theory and the target theory is non-trivial [5]. ## Running `pineko` From 9c3dd18e44c567c8de09ee3c38f354019d577ad8 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 16 Mar 2022 12:10:32 +0100 Subject: [PATCH 24/36] Add inherit-ekos command --- README.md | 25 ++++++++---- src/pineko/cli/theory_.py | 9 ++++ src/pineko/theory.py | 86 ++++++++++++++++++++++++++++++++++----- 3 files changed, 101 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index e8262d88..5a3e417c 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ Since combines several ingredients into a new object one needs to provide a bunc ### pineko.toml -You need to provide a `pineko.toml`, that provides all necessary paths to the respective tools and output folders. -[DEBUG: Look at the debug example in this repo [1]]. +You need to provide a `pineko.toml`, that provides all necessary paths to the input and output folders. +[DEBUG: Look at the debug example in this repo [1].] ### ymldb @@ -21,7 +21,7 @@ This defines the mapping from datasets to FK tables. ### Theory Runcards -You need to provide the necessary theory runcards named with their respective theory ID inside the `theory_cards` paths [3]. +You need to provide the necessary theory runcards named with their respective theory ID inside the `` folder [3]. ### Default Operator Card @@ -41,7 +41,7 @@ inside the `/` folder. The name has to match the `ymldb` #### Inherit Grids from Existing Theory -You can reuse the grids from a different theory by using `pineko theory inherit-grids SOURCE_THEORY_ID TARGET_THEORY_ID DATASET1 DATASET2 ...`. +You can reuse the grids from a different theory by running `pineko theory inherit-grids SOURCE_THEORY_ID TARGET_THEORY_ID DATASET1 DATASET2 ...`. The relation between the source theory and the target theory is non-trivial [5]. ## Running `pineko` @@ -51,14 +51,21 @@ computing the EKO and convoluting the EKO with the grid. ### Computing the EKO -Again this is a two step process: +#### Generating new EKOs + +This is a two step process: 1. Generate the necessary operator cards with `pineko theory opcards THEORY_ID DATASET1 DATASET2 ...` 2. Generate the actual EKOs with `pineko theory ekos THEORY_ID DATASET1 DATASET2 ...` +#### Inherit EKOs from Existing Theory + +You can reuse the EKOs from a different theory by running `pineko theory inherit-ekos SOURCE_THEORY_ID TARGET_THEORY_ID DATASET1 DATASET2 ...`. +The relation between the source theory and the target theory is non-trivial [6]. + ### Generating the FK Table -You need to have the EKO computed in the previous step. -Then you can convolute the EKO with the grid by `pineko theory fks THEORY_ID DATASET1 DATASET2 ...` +You need to have the EKOs computed in the previous step. +Then you can convolute the EKOs with the grids by running `pineko theory fks THEORY_ID DATASET1 DATASET2 ...` --- @@ -70,4 +77,6 @@ Then you can convolute the EKO with the grid by `pineko theory fks THEORY_ID DAT [4] I'm thinking how to improve this, because how could we provide a study on the interpolation accuracy? at the moment there just equal -[5] examples being SV, different evolution settings, etc. - I'm thinking whether it might be worth to add an `inherit-eko` option +[5] examples being SV, different evolution settings, etc. + +[6] examples being SV, different DIS settings, etc. diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index 6c3e6556..128b6b77 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -39,6 +39,15 @@ def ekos(theory_id, datasets, logs, overwrite): theory.TheoryBuilder(theory_id, datasets, overwrite).ekos(logs) +@theory_.command() +@click.argument("source_theory_id", type=click.INT) +@click.argument("target_theory_id", type=click.INT) +@click.argument("datasets", type=click.STRING, nargs=-1) +@click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") +def inherit_ekos(source_theory_id, target_theory_id, datasets, overwrite): + """Inherit eko from one theory to another.""" + theory.TheoryBuilder(source_theory_id, datasets, overwrite).inherit_ekos(target_theory_id) + @theory_.command() @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 0b3a43ba..8e6f3cda 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -31,10 +31,22 @@ def operator_cards_path(self): """Suffix paths.operator_cards with theory id.""" return configs.configs["paths"]["operator_cards"] / str(self.theory_id) - @property - def eko_path(self): - """Suffix paths.ekos with theory id.""" - return configs.configs["paths"]["ekos"] / str(self.theory_id) + def eko_path(self, tid=None): + """Suffix paths.ekos with theory id. + + Parameters + ---------- + tid : int + theory id, defaults to my theory id + + Returns + ------- + pathlib.Path : + true path + """ + if tid is None: + tid = self.theory_id + return configs.configs["paths"]["ekos"] / str(tid) @property def fk_path(self): @@ -42,7 +54,18 @@ def fk_path(self): return configs.configs["paths"]["fktables"] / str(self.theory_id) def grids_path(self, tid = None): - """Suffix paths.grids with theory id.""" + """Suffix paths.grids with theory id. + + Parameters + ---------- + tid : int + theory id, defaults to my theory id + + Returns + ------- + pathlib.Path : + true path + """ if tid is None: tid = self.theory_id return configs.configs["paths"]["grids"] / str(tid) @@ -71,12 +94,16 @@ def load_grids(self, ds): return grids def inherit_grid(self, name, grid, other): - """Inherit grids to a new theory. + """Inherit a grid to a new theory. Parameters ---------- - target_theory_id : int - target theory id + name : str + grid name, i.e. it's true stem + grid : pathlib.Path + path to grid + other : pathlib.Path + new folder """ new = other / f"{name}.{parser.ext}" if new.exists(): @@ -102,6 +129,43 @@ def inherit_grids(self, target_theory_id): other.mkdir(exist_ok=True) self.iterate(self.inherit_grid, other=other) + def inherit_eko(self, name, _grid, other): + """Inherit a EKO to a new theory. + + Parameters + ---------- + name : str + grid name, i.e. it's true stem + grid : pathlib.Path + path to grid + other : pathlib.Path + new folder + """ + eko = self.eko_path() / f"{name}.tar" + new = other / f"{name}.tar" + if new.exists(): + if not self.overwrite: + rich.print(f"Skipping existing eko {new}") + return + else: + new.unlink() + # link + new.symlink_to(eko) + if new.exists(): + rich.print(f"[green]Success:[/] Created link at {new}") + + def inherit_ekos(self, target_theory_id): + """Inherit ekos to a new theory. + + Parameters + ---------- + target_theory_id : int + target theory id + """ + other = self.eko_path(target_theory_id) + other.mkdir(exist_ok=True) + self.iterate(self.inherit_eko, other=other) + def iterate(self, f, **kwargs): """Iterated grids in datasets. @@ -165,7 +229,7 @@ def eko(self, name, _grid, tcard, logs): opcard_path = self.operator_cards_path / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: ocard = yaml.safe_load(f) - eko_filename = self.eko_path / f"{name}.tar" + eko_filename = self.eko_path() / f"{name}.tar" if eko_filename.exists(): if not self.overwrite: rich.print(f"Skipping existing operator {eko_filename}") @@ -194,7 +258,7 @@ def ekos(self, logs): save eko logs? """ tcard = theory_card.load(self.theory_id) - self.eko_path.mkdir(exist_ok=True) + self.eko_path().mkdir(exist_ok=True) self.iterate(self.eko, tcard=tcard, logs=logs) def fk(self, name, grid_path, tcard, pdf, logs): @@ -215,7 +279,7 @@ def fk(self, name, grid_path, tcard, pdf, logs): """ # setup data paths = configs.configs["paths"] - eko_filename = self.eko_path / f"{name}.tar" + eko_filename = self.eko_path() / f"{name}.tar" fk_filename = self.fk_path / f"{name}.{parser.ext}" if fk_filename.exists(): if not self.overwrite: From a10887b7d085b57c5057f4e3dcfdcce2a1a5d42b Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 16 Mar 2022 12:13:30 +0100 Subject: [PATCH 25/36] Drop config.add_scope --- src/pineko/configs.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/pineko/configs.py b/src/pineko/configs.py index 04d75249..b741ac24 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -14,16 +14,6 @@ "Holds loaded configurations" -def add_scope(base, scope_id, scope): - "Do not override." - if scope_id not in base: - base[scope_id] = scope - else: - for key, value in scope.items(): - if key not in base[scope_id]: - base[scope_id] = value - - def defaults(base_configs): """Provide additional defaults. From 23449b60e0c09f35a77976c5870847eab9330421 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 16 Mar 2022 14:40:28 +0100 Subject: [PATCH 26/36] Improve logging --- src/pineko/cli/theory_.py | 12 ++--- src/pineko/theory.py | 95 ++++++++++++++++++++++++--------------- 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index 128b6b77..05f78f9a 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -32,11 +32,11 @@ def opcards(theory_id, datasets, overwrite): @theory_.command() @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -@click.option("--logs", is_flag=True, help="dump logs") +@click.option("--no-logs", is_flag=True, help="suppress logs") @click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") -def ekos(theory_id, datasets, logs, overwrite): +def ekos(theory_id, datasets, no_logs, overwrite): """Compute EKOs for all FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets, overwrite).ekos(logs) + theory.TheoryBuilder(theory_id, datasets, overwrite).ekos(no_logs) @theory_.command() @@ -52,8 +52,8 @@ def inherit_ekos(source_theory_id, target_theory_id, datasets, overwrite): @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) @click.option("--pdf", "-p", default=None, help="comparison PDF") -@click.option("--logs", is_flag=True, help="dump comparison") +@click.option("--no-logs", is_flag=True, help="suppress logs with comparison") @click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") -def fks(theory_id, datasets, pdf, logs, overwrite): +def fks(theory_id, datasets, pdf, no_logs, overwrite): """Compute FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets, overwrite).fks(pdf, logs) + theory.TheoryBuilder(theory_id, datasets, overwrite).fks(pdf, no_logs) diff --git a/src/pineko/theory.py b/src/pineko/theory.py index 8e6f3cda..a322b99a 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import logging +import time import eko import rich @@ -8,6 +9,9 @@ from . import configs, evolve, parser, theory_card, parser +logger = logging.getLogger(__name__) + + class TheoryBuilder: """Common builder application to create the ingredients for a theory. @@ -31,7 +35,7 @@ def operator_cards_path(self): """Suffix paths.operator_cards with theory id.""" return configs.configs["paths"]["operator_cards"] / str(self.theory_id) - def eko_path(self, tid=None): + def ekos_path(self, tid=None): """Suffix paths.ekos with theory id. Parameters @@ -49,7 +53,7 @@ def eko_path(self, tid=None): return configs.configs["paths"]["ekos"] / str(tid) @property - def fk_path(self): + def fks_path(self): """Suffix paths.fktables with theory id.""" return configs.configs["paths"]["fktables"] / str(self.theory_id) @@ -141,7 +145,7 @@ def inherit_eko(self, name, _grid, other): other : pathlib.Path new folder """ - eko = self.eko_path() / f"{name}.tar" + eko = self.ekos_path() / f"{name}.tar" new = other / f"{name}.tar" if new.exists(): if not self.overwrite: @@ -162,7 +166,7 @@ def inherit_ekos(self, target_theory_id): target_theory_id : int target theory id """ - other = self.eko_path(target_theory_id) + other = self.ekos_path(target_theory_id) other.mkdir(exist_ok=True) self.iterate(self.inherit_eko, other=other) @@ -208,9 +212,10 @@ def opcard(self, name, grid): def opcards(self): """Write operator cards.""" + self.operator_cards_path.mkdir(exist_ok=True) self.iterate(self.opcard) - def eko(self, name, _grid, tcard, logs): + def eko(self, name, _grid, tcard, no_logs): """Compute a single eko. Parameters @@ -222,46 +227,50 @@ def eko(self, name, _grid, tcard, logs): tcard : dict theory card logs : bool - save eko logs? + save logs? """ - # setup data paths = configs.configs["paths"] + # activate logging + if not no_logs and paths["logs"]["eko"]: + log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" + log_file = logging.FileHandler(log_path) + log_file.setLevel(logging.INFO) + log_file.setFormatter(logging.Formatter("%(asctime)s %(name)s/%(levelname)s: %(message)s")) + for logger_ in (logger, logging.getLogger("eko")): + logger_.handlers = [] + logger_.addHandler(log_file) + logger_.setLevel(logging.INFO) + # setup data opcard_path = self.operator_cards_path / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: ocard = yaml.safe_load(f) - eko_filename = self.eko_path() / f"{name}.tar" + eko_filename = self.ekos_path() / f"{name}.tar" if eko_filename.exists(): if not self.overwrite: rich.print(f"Skipping existing operator {eko_filename}") return - # activate logging - if logs and paths["logs"]["eko"]: - log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" - log_file = logging.FileHandler(log_path) - log_file.setLevel(logging.INFO) - log_file.setFormatter(logging.Formatter("%(message)s")) - logging.getLogger("eko").handlers = [] - logging.getLogger("eko").addHandler(log_file) - logging.getLogger("eko").setLevel(logging.INFO) # do it! + logger.info("Start computation of %s", name) + start_time = time.perf_counter() ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) ops.dump_tar(eko_filename) + logger.info("Finished computation of %s - took %f s", name, time.perf_counter() - start_time) if eko_filename.exists(): rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}") - def ekos(self, logs): + def ekos(self, no_logs): """Compute all ekos. Parameters ---------- - logs : bool - save eko logs? + no_logs : bool + suppress logs? """ tcard = theory_card.load(self.theory_id) - self.eko_path().mkdir(exist_ok=True) - self.iterate(self.eko, tcard=tcard, logs=logs) + self.ekos_path().mkdir(exist_ok=True) + self.iterate(self.eko, tcard=tcard, no_logs=no_logs) - def fk(self, name, grid_path, tcard, pdf, logs): + def fk(self, name, grid_path, tcard, pdf, no_logs): """Compute a single FK table. Parameters @@ -274,13 +283,24 @@ def fk(self, name, grid_path, tcard, pdf, logs): theory card pdf : str comparison PDF - logs : bool - save eko logs? + no_logs : bool + suppress logs? """ - # setup data + # activate logging paths = configs.configs["paths"] - eko_filename = self.eko_path() / f"{name}.tar" - fk_filename = self.fk_path / f"{name}.{parser.ext}" + do_log = not no_logs and paths["logs"]["fk"] + if do_log: + log_path = paths["logs"]["fk"] / f"{self.theory_id}-{name}-{pdf}.log" + log_file = logging.FileHandler(log_path) + log_file.setLevel(logging.INFO) + log_file.setFormatter(logging.Formatter("%(asctime)s %(name)s/%(levelname)s: %(message)s")) + logger.handlers = [] + logger.addHandler(log_file) + logger.setLevel(logging.INFO) + + # setup data + eko_filename = self.ekos_path() / f"{name}.tar" + fk_filename = self.fks_path / f"{name}.{parser.ext}" if fk_filename.exists(): if not self.overwrite: rich.print(f"Skipping existing FK Table {fk_filename}") @@ -288,26 +308,27 @@ def fk(self, name, grid_path, tcard, pdf, logs): max_as = 1 + int(tcard["PTO"]) max_al = 0 # do it! + logger.info("Start computation of %s", name) + start_time = time.perf_counter() _grid, _fk, comparison = evolve.evolve_grid( grid_path, eko_filename, fk_filename, max_as, max_al, pdf ) - # activate logging - if logs and paths["logs"]["fk"] and comparison: - logfile = paths["logs"]["fk"] / f"{self.theory_id}-{name}-{pdf}.log" - logfile.write_text(comparison.to_string()) + logger.info("Finished computation of %s - took %f s", name, time.perf_counter() - start_time) + if do_log and comparison is not None: + logger.info("Comparison with %s:\n %s", pdf, comparison.to_string()) if fk_filename.exists(): rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}") - def fks(self, pdf, logs): + def fks(self, pdf, no_logs): """Compute all FK tables. Parameters ---------- pdf : str comparison PDF - logs : bool - save eko logs? + no_logs : bool + suppress logs? """ tcard = theory_card.load(self.theory_id) - self.fk_path.mkdir(exist_ok=True) - self.iterate(self.fk, tcard=tcard, pdf=pdf, logs=logs) + self.fks_path.mkdir(exist_ok=True) + self.iterate(self.fk, tcard=tcard, pdf=pdf, no_logs=no_logs) From b464b52d60f24fd709d8309efd5a25df684831d5 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 16 Mar 2022 15:12:24 +0100 Subject: [PATCH 27/36] Improve logging --- src/pineko/cli/theory_.py | 43 ++++++++++---- src/pineko/theory.py | 115 ++++++++++++++++++++++---------------- 2 files changed, 100 insertions(+), 58 deletions(-) diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index 05f78f9a..f7f4f6c2 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -16,8 +16,10 @@ def theory_(): @click.argument("datasets", type=click.STRING, nargs=-1) @click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") def inherit_grids(source_theory_id, target_theory_id, datasets, overwrite): - """Inherit grids from one theory to another.""" - theory.TheoryBuilder(source_theory_id, datasets, overwrite).inherit_grids(target_theory_id) + """Inherit grids for datasets from one theory to another.""" + theory.TheoryBuilder(source_theory_id, datasets, overwrite).inherit_grids( + target_theory_id + ) @theory_.command() @@ -32,11 +34,19 @@ def opcards(theory_id, datasets, overwrite): @theory_.command() @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -@click.option("--no-logs", is_flag=True, help="suppress logs") +@click.option("--silent", is_flag=True, help="Suppress logs") +@click.option( + "-cl", + "--clear-logs", + is_flag=True, + help="Erease previos logs (instead of appending)", +) @click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") -def ekos(theory_id, datasets, no_logs, overwrite): +def ekos(theory_id, datasets, silent, clear_logs, overwrite): """Compute EKOs for all FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets, overwrite).ekos(no_logs) + theory.TheoryBuilder( + theory_id, datasets, silent=silent, clear_logs=clear_logs, overwrite=overwrite + ).ekos() @theory_.command() @@ -45,15 +55,26 @@ def ekos(theory_id, datasets, no_logs, overwrite): @click.argument("datasets", type=click.STRING, nargs=-1) @click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") def inherit_ekos(source_theory_id, target_theory_id, datasets, overwrite): - """Inherit eko from one theory to another.""" - theory.TheoryBuilder(source_theory_id, datasets, overwrite).inherit_ekos(target_theory_id) + """Inherit ekos from one theory to another.""" + theory.TheoryBuilder(source_theory_id, datasets, overwrite=overwrite).inherit_ekos( + target_theory_id + ) + @theory_.command() @click.argument("theory_id", type=click.INT) @click.argument("datasets", type=click.STRING, nargs=-1) -@click.option("--pdf", "-p", default=None, help="comparison PDF") -@click.option("--no-logs", is_flag=True, help="suppress logs with comparison") +@click.option("--pdf", "-p", default=None, help="PDF set used for comparison") +@click.option("--silent", is_flag=True, help="Suppress logs with comparison") +@click.option( + "-cl", + "--clear-logs", + is_flag=True, + help="Erease previos logs (instead of appending)", +) @click.option("--overwrite", is_flag=True, help="Allow files to be overwritten") -def fks(theory_id, datasets, pdf, no_logs, overwrite): +def fks(theory_id, datasets, pdf, silent, clear_logs, overwrite): """Compute FK tables in all datasets.""" - theory.TheoryBuilder(theory_id, datasets, overwrite).fks(pdf, no_logs) + theory.TheoryBuilder( + theory_id, datasets, silent=silent, clear_logs=clear_logs, overwrite=overwrite + ).fks(pdf) diff --git a/src/pineko/theory.py b/src/pineko/theory.py index a322b99a..e165d9e9 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -6,8 +6,7 @@ import rich import yaml -from . import configs, evolve, parser, theory_card, parser - +from . import configs, evolve, parser, theory_card logger = logging.getLogger(__name__) @@ -21,15 +20,23 @@ class TheoryBuilder: theory identifier datsets : list(str) list of datasets + silent : bool + suppress logs + clear_logs : bool + erease previos logs (instead of appending) overwrite : bool allow files to be overwritten instead of skipping """ - def __init__(self, theory_id, datasets, overwrite=False): + def __init__( + self, theory_id, datasets, silent=False, clear_logs=False, overwrite=False + ): self.theory_id = theory_id self.datasets = datasets + self.silent = silent + self.clear_logs = clear_logs self.overwrite = overwrite - + @property def operator_cards_path(self): """Suffix paths.operator_cards with theory id.""" @@ -37,7 +44,7 @@ def operator_cards_path(self): def ekos_path(self, tid=None): """Suffix paths.ekos with theory id. - + Parameters ---------- tid : int @@ -57,9 +64,9 @@ def fks_path(self): """Suffix paths.fktables with theory id.""" return configs.configs["paths"]["fktables"] / str(self.theory_id) - def grids_path(self, tid = None): + def grids_path(self, tid=None): """Suffix paths.grids with theory id. - + Parameters ---------- tid : int @@ -215,7 +222,38 @@ def opcards(self): self.operator_cards_path.mkdir(exist_ok=True) self.iterate(self.opcard) - def eko(self, name, _grid, tcard, no_logs): + def activate_logging(self, path, filename, activated_loggers=()): + """Activate the logging facilities. + + Parameters: + ----------- + path : pathlib.Path + source directory + filename : str + log file name + activated_loggers : list(str) + list of loggers that get registered + """ + # nothing to do? + if self.silent or not path: + return False + # evtually remove old stuff? + log_path = path / filename + if self.clear_logs: + log_path.write_text("") + # register everything + log_file = logging.FileHandler(log_path) + log_file.setLevel(logging.INFO) + log_file.setFormatter( + logging.Formatter("%(asctime)s %(name)s/%(levelname)s: %(message)s") + ) + for logger_ in (logger, *[logging.getLogger(n) for n in activated_loggers]): + logger_.handlers = [] + logger_.addHandler(log_file) + logger_.setLevel(logging.INFO) + return True + + def eko(self, name, _grid, tcard): """Compute a single eko. Parameters @@ -226,20 +264,12 @@ def eko(self, name, _grid, tcard, no_logs): path to grid tcard : dict theory card - logs : bool - save logs? """ paths = configs.configs["paths"] # activate logging - if not no_logs and paths["logs"]["eko"]: - log_path = paths["logs"]["eko"] / f"{self.theory_id}-{name}.log" - log_file = logging.FileHandler(log_path) - log_file.setLevel(logging.INFO) - log_file.setFormatter(logging.Formatter("%(asctime)s %(name)s/%(levelname)s: %(message)s")) - for logger_ in (logger, logging.getLogger("eko")): - logger_.handlers = [] - logger_.addHandler(log_file) - logger_.setLevel(logging.INFO) + self.activate_logging( + paths["logs"]["eko"], f"{self.theory_id}-{name}.log", ("eko",) + ) # setup data opcard_path = self.operator_cards_path / f"{name}.yaml" with open(opcard_path, encoding="utf-8") as f: @@ -254,23 +284,21 @@ def eko(self, name, _grid, tcard, no_logs): start_time = time.perf_counter() ops = eko.run_dglap(theory_card=tcard, operators_card=ocard) ops.dump_tar(eko_filename) - logger.info("Finished computation of %s - took %f s", name, time.perf_counter() - start_time) + logger.info( + "Finished computation of %s - took %f s", + name, + time.perf_counter() - start_time, + ) if eko_filename.exists(): rich.print(f"[green]Success:[/] Wrote EKO to {eko_filename}") - def ekos(self, no_logs): - """Compute all ekos. - - Parameters - ---------- - no_logs : bool - suppress logs? - """ + def ekos(self): + """Compute all ekos.""" tcard = theory_card.load(self.theory_id) self.ekos_path().mkdir(exist_ok=True) - self.iterate(self.eko, tcard=tcard, no_logs=no_logs) + self.iterate(self.eko, tcard=tcard) - def fk(self, name, grid_path, tcard, pdf, no_logs): + def fk(self, name, grid_path, tcard, pdf): """Compute a single FK table. Parameters @@ -283,21 +311,12 @@ def fk(self, name, grid_path, tcard, pdf, no_logs): theory card pdf : str comparison PDF - no_logs : bool - suppress logs? """ # activate logging paths = configs.configs["paths"] - do_log = not no_logs and paths["logs"]["fk"] - if do_log: - log_path = paths["logs"]["fk"] / f"{self.theory_id}-{name}-{pdf}.log" - log_file = logging.FileHandler(log_path) - log_file.setLevel(logging.INFO) - log_file.setFormatter(logging.Formatter("%(asctime)s %(name)s/%(levelname)s: %(message)s")) - logger.handlers = [] - logger.addHandler(log_file) - logger.setLevel(logging.INFO) - + do_log = self.activate_logging( + paths["logs"]["fk"], f"{self.theory_id}-{name}-{pdf}.log" + ) # setup data eko_filename = self.ekos_path() / f"{name}.tar" fk_filename = self.fks_path / f"{name}.{parser.ext}" @@ -313,22 +332,24 @@ def fk(self, name, grid_path, tcard, pdf, no_logs): _grid, _fk, comparison = evolve.evolve_grid( grid_path, eko_filename, fk_filename, max_as, max_al, pdf ) - logger.info("Finished computation of %s - took %f s", name, time.perf_counter() - start_time) + logger.info( + "Finished computation of %s - took %f s", + name, + time.perf_counter() - start_time, + ) if do_log and comparison is not None: logger.info("Comparison with %s:\n %s", pdf, comparison.to_string()) if fk_filename.exists(): rich.print(f"[green]Success:[/] Wrote FK table to {fk_filename}") - def fks(self, pdf, no_logs): + def fks(self, pdf): """Compute all FK tables. Parameters ---------- pdf : str comparison PDF - no_logs : bool - suppress logs? """ tcard = theory_card.load(self.theory_id) self.fks_path.mkdir(exist_ok=True) - self.iterate(self.fk, tcard=tcard, pdf=pdf, no_logs=no_logs) + self.iterate(self.fk, tcard=tcard, pdf=pdf) From 19ccfee1697256497ed49ded6a76385eb76071e9 Mon Sep 17 00:00:00 2001 From: Christopher Schwan Date: Wed, 16 Mar 2022 16:50:48 +0100 Subject: [PATCH 28/36] Adjust README a bit --- README.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 5a3e417c..38850209 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,24 @@ -# pineko +# `pineko` = `PineAPPL` + `eko` -`pineko` combines coefficient function grids ("grids" for short) produced by [`runcards`](https://github.com/NNPDF/runcards) -in the form of [`PineAPPL` grids](https://github.com/N3PDF/pineappl) -and Evolution Kernel Operators by [`eko`](https://github.com/N3PDF/eko) -into FK tables. It is one part of the [`APFELcomb`](https://github.com/NNPDF/apfelcomb) replacement. +`pineko` converts + +- interpolation grids for theory predictions ('grids' for short) in the form of + [`PineAPPL`](https://github.com/N3PDF/pineappl) grids, together with +- Evolution Kernel Operators (EKO) generated by + [`eko`](https://github.com/N3PDF/eko) + +into fast-kernel (FK) tables. The collection of all FK tables constitute the +theory predictions for a PDF fit and therefore is often simply called 'theory'. + +`pineko` replaces [`APFELcomb`](https://github.com/NNPDF/apfelcomb), which was +used up to NNPDF4.0. ## Prerequisites -Since combines several ingredients into a new object one needs to provide a bunch of files first. +Generating a 'theory', as defined above, requires several files which are +described next. -### pineko.toml +### `pineko.toml` You need to provide a `pineko.toml`, that provides all necessary paths to the input and output folders. [DEBUG: Look at the debug example in this repo [1].] @@ -28,9 +37,9 @@ You need to provide the necessary theory runcards named with their respective th You need to provide a default operator card for `eko` [4]. [DEBUG: Look at the respective `load.sh` script to load from dom.] -### Coefficient Functions Grids +### Grids -`pineko` is **NOT** computing grids, but is taking them as an input. +`pineko` does **NOT** compute grids, which are instead expected input to `pineko`. There are typically two ways to obtain grids: computing them from scratch with [`runcards`](https://github.com/NNPDF/runcards) or reusing existing ones. From f5702e50eb451adc652cdd551b00a09df5dc753d Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Mon, 21 Mar 2022 16:47:39 +0100 Subject: [PATCH 29/36] Remove common grids dir --- data/grids/common/.gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 data/grids/common/.gitignore diff --git a/data/grids/common/.gitignore b/data/grids/common/.gitignore deleted file mode 100644 index 9727d870..00000000 --- a/data/grids/common/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pineappl.lz4 From d2018a467c3860212a2636ea71abb7c037290eac Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 22 Mar 2022 11:50:52 +0100 Subject: [PATCH 30/36] Move pylint and apply some suggestions --- .pylintrc | 504 ------------------------------------ pyproject.toml | 19 ++ src/pineko/cli/convolute.py | 2 +- src/pineko/comparator.py | 2 + src/pineko/configs.py | 42 +-- src/pineko/theory.py | 10 +- 6 files changed, 48 insertions(+), 531 deletions(-) delete mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 0405a523..00000000 --- a/.pylintrc +++ /dev/null @@ -1,504 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist=numpy,lhapdf,numba,pineappl,eko,yadism - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 # has to be 1 as pylint is NOT threadsafe - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -#load-plugins=pylint.extensions.mccabe -#max-complexity=20 - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable= - print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - raw-checker-failed, - bad-inline-option, - locally-disabled, - locally-enabled, - file-ignored, - suppressed-message, - useless-suppression, - deprecated-pragma, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - invalid-name, - too-few-public-methods, - too-many-arguments, - bad-continuation, - redefined-outer-name, - missing-docstring, - bad-whitespace, - no-self-use, - no-else-return, - global-statement, - too-many-public-method, - too-many-ancestors - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable= - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[BASIC] - -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^.*(# )??.*$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules=matplotlib.cm,numba - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=22 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/pyproject.toml b/pyproject.toml index c77e67ac..1e43f961 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,3 +53,22 @@ dirty = true [tool.poetry-dynamic-versioning.substitution] files = ["src/pineko/__init__.py"] + +[tool.pylint.master] +# extensions not to check +extension-pkg-whitelist = ["numpy", "numba", "lhapdf", "pineappl"] +ignore-paths = ["benchmarks/", "doc/", "tests/", "data/"] +# jobs has to be 1 as pylint is NOT threadsafe +jobs = 1 +[tool.pylint.messages_control] +disable = ["invalid-name","fixme"] +[tool.pylint.reports] +# Available formats are: +# text, parseable, colorized, json and msvs (visual studio). +output-format = "colorized" +[tool.pylint.format] +# Maximum number of characters on a single line. +max-line-length = 100 +[tool.pylint.design] +# Maximum number of arguments for function / method +max-args = 10 diff --git a/src/pineko/cli/convolute.py b/src/pineko/cli/convolute.py index fd26434c..594679e2 100644 --- a/src/pineko/cli/convolute.py +++ b/src/pineko/cli/convolute.py @@ -23,5 +23,5 @@ def subcommand(pineappl, eko, fktable, max_as, max_al, pdf): couplings (i.e. the maximum power allowed for each correction). """ _grid, _fk, comp = evolve.evolve_grid(pineappl, eko, fktable, max_as, max_al, pdf) - if (comp): + if comp: print(comp.to_string()) diff --git a/src/pineko/comparator.py b/src/pineko/comparator.py index a2ab6320..cf7357c9 100644 --- a/src/pineko/comparator.py +++ b/src/pineko/comparator.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +"""Provide tools to compare grids and FK tables""" + import numpy as np import pandas as pd import pineappl diff --git a/src/pineko/configs.py b/src/pineko/configs.py index b741ac24..e3b270c2 100644 --- a/src/pineko/configs.py +++ b/src/pineko/configs.py @@ -21,31 +21,31 @@ def defaults(base_configs): ---------- base_config : dict user provided configuration - + Returns ------- - configs : dict + configs_ : dict enhanced configuration Note ---- The general rule is to never replace user provided input. """ - configs = copy.deepcopy(base_configs) + configs_ = copy.deepcopy(base_configs) - enhance_paths(configs) + enhance_paths(configs_) - return configs + return configs_ -def enhance_paths(configs): +def enhance_paths(configs_): """Check required path and enhance them with root path. - + The changes are done inplace. Parameters ---------- - configs : dict + configs_ : dict configuration """ # required keys without default @@ -58,26 +58,28 @@ def enhance_paths(configs): "fktables", "ekos", ]: - if key not in configs["paths"]: + if key not in configs_["paths"]: raise ValueError(f"Configuration is missing a 'paths.{key}' key") - if pathlib.Path(configs["paths"][key]).anchor == "": - configs["paths"][key] = configs["paths"]["root"] / configs["paths"][key] + if pathlib.Path(configs_["paths"][key]).anchor == "": + configs_["paths"][key] = configs_["paths"]["root"] / configs_["paths"][key] else: - configs["paths"][key] = pathlib.Path(configs["paths"][key]) + configs_["paths"][key] = pathlib.Path(configs_["paths"][key]) # optional keys which are by default None - if "logs" not in configs["paths"]: - configs["paths"]["logs"] = {} + if "logs" not in configs_["paths"]: + configs_["paths"]["logs"] = {} for key in ["eko", "fk"]: - if key not in configs["paths"]["logs"]: - configs["paths"]["logs"][key] = None - elif pathlib.Path(configs["paths"]["logs"][key]).anchor == "": - configs["paths"]["logs"][key] = ( - configs["paths"]["root"] / configs["paths"]["logs"][key] + if key not in configs_["paths"]["logs"]: + configs_["paths"]["logs"][key] = None + elif pathlib.Path(configs_["paths"]["logs"][key]).anchor == "": + configs_["paths"]["logs"][key] = ( + configs_["paths"]["root"] / configs_["paths"]["logs"][key] ) else: - configs["paths"]["logs"][key] = pathlib.Path(configs["paths"]["logs"][key]) + configs_["paths"]["logs"][key] = pathlib.Path( + configs_["paths"]["logs"][key] + ) def detect(path=None): diff --git a/src/pineko/theory.py b/src/pineko/theory.py index e165d9e9..e1e142fc 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -121,8 +121,7 @@ def inherit_grid(self, name, grid, other): if not self.overwrite: rich.print(f"Skipping existing grid {new}") return - else: - new.unlink() + new.unlink() # link new.symlink_to(grid) if new.exists(): @@ -152,16 +151,15 @@ def inherit_eko(self, name, _grid, other): other : pathlib.Path new folder """ - eko = self.ekos_path() / f"{name}.tar" + eko_path = self.ekos_path() / f"{name}.tar" new = other / f"{name}.tar" if new.exists(): if not self.overwrite: rich.print(f"Skipping existing eko {new}") return - else: - new.unlink() + new.unlink() # link - new.symlink_to(eko) + new.symlink_to(eko_path) if new.exists(): rich.print(f"[green]Success:[/] Created link at {new}") From 24674cf9564ecde93532c5c3f18b128446f8de01 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 22 Mar 2022 11:53:08 +0100 Subject: [PATCH 31/36] Remove old code and folder --- _data/.gitignore | 2 - _data/observable-f2b.yaml | 114 ----------------------------------- _data/observable-simple.yaml | 62 ------------------- _data/observable.yaml | 114 ----------------------------------- _data/operator.yaml | 61 ------------------- _data/theory_200.yaml | 46 -------------- _data/theory_208.yaml | 47 --------------- _data/theory_213.yaml | 46 -------------- _run.py | 83 ------------------------- 9 files changed, 575 deletions(-) delete mode 100644 _data/.gitignore delete mode 100644 _data/observable-f2b.yaml delete mode 100644 _data/observable-simple.yaml delete mode 100644 _data/observable.yaml delete mode 100644 _data/operator.yaml delete mode 100644 _data/theory_200.yaml delete mode 100644 _data/theory_208.yaml delete mode 100644 _data/theory_213.yaml delete mode 100755 _run.py diff --git a/_data/.gitignore b/_data/.gitignore deleted file mode 100644 index 8b449210..00000000 --- a/_data/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -my* -*.pineappl* diff --git a/_data/observable-f2b.yaml b/_data/observable-f2b.yaml deleted file mode 100644 index e2e1751b..00000000 --- a/_data/observable-f2b.yaml +++ /dev/null @@ -1,114 +0,0 @@ -interpolation_polynomial_degree: 4 -interpolation_is_log: true -interpolation_xgrid: - - 1.9999999999999954e-07 - - 3.034304765867952e-07 - - 4.6035014748963906e-07 - - 6.984208530700364e-07 - - 1.0596094959101024e-06 - - 1.607585498470808e-06 - - 2.438943292891682e-06 - - 3.7002272069854957e-06 - - 5.613757716930151e-06 - - 8.516806677573355e-06 - - 1.292101569074731e-05 - - 1.9602505002391748e-05 - - 2.97384953722449e-05 - - 4.511438394964044e-05 - - 6.843744918967896e-05 - - 0.00010381172986576898 - - 0.00015745605600841445 - - 0.00023878782918561914 - - 0.00036205449638139736 - - 0.0005487795323670796 - - 0.0008314068836488144 - - 0.0012586797144272762 - - 0.0019034634022867384 - - 0.0028738675812817515 - - 0.004328500638820811 - - 0.006496206194633799 - - 0.009699159574043398 - - 0.014375068581090129 - - 0.02108918668378717 - - 0.030521584007828916 - - 0.04341491741702269 - - 0.060480028754447364 - - 0.08228122126204893 - - 0.10914375746330703 - - 0.14112080644440345 - - 0.17802566042569432 - - 0.2195041265003886 - - 0.2651137041582823 - - 0.31438740076927585 - - 0.3668753186482242 - - 0.4221667753589648 - - 0.4798989029610255 - - 0.5397572337880445 - - 0.601472197967335 - - 0.6648139482473823 - - 0.7295868442414312 - - 0.7956242522922756 - - 0.8627839323906108 - - 0.9309440808717544 - - 1 -observables: - F2_bottom: - - Q2: 50.0 - x: 1.9999999999999954e-07 - - Q2: 50.0 - x: 6.984208530700364e-07 - - Q2: 50.0 - x: 2.438943292891682e-06 - - Q2: 50.0 - x: 8.516806677573355e-06 - - Q2: 50.0 - x: 2.97384953722449e-05 - - Q2: 50.0 - x: 0.00010381172986576898 - - Q2: 50.0 - x: 0.00036205449638139736 - - Q2: 50.0 - x: 0.0012586797144272762 - - Q2: 50.0 - x: 0.004328500638820811 - - Q2: 50.0 - x: 0.014375068581090129 - - Q2: 50.0 - x: 0.04341491741702269 - - Q2: 50.0 - x: 0.10914375746330703 - - Q2: 50.0 - x: 0.2195041265003886 - - Q2: 50.0 - x: 0.3668753186482242 - - Q2: 50.0 - x: 0.5397572337880445 - - Q2: 50.0 - x: 0.7295868442414312 - - Q2: 50.0 - x: 0.9309440808717544 - # - Q2: 4.0 - # x: 0.10914375746330703 - # - Q2: 7.387504697919031 - # x: 0.10914375746330703 - # - Q2: 13.643806415443933 - # x: 0.10914375746330703 - # - Q2: 25.198420997897458 - # x: 0.10914375746330703 - - Q2: 46.53836337552725 - x: 0.10914375746330703 - - Q2: 85.9505945175426 - x: 0.10914375746330703 - - Q2: 158.74010519682002 - x: 0.10914375746330703 - # - Q2: 293.17331822241715 - # x: 0.10914375746330703 - # - Q2: 541.4548164181543 - # x: 0.10914375746330703 - # - Q2: 1000.0 - # x: 0.10914375746330703 -prDIS: EM -PolarizationDIS: 0. -ProjectileDIS: electron -TargetDIS: proton -PropagatorCorrection: 0. diff --git a/_data/observable-simple.yaml b/_data/observable-simple.yaml deleted file mode 100644 index 1585ab81..00000000 --- a/_data/observable-simple.yaml +++ /dev/null @@ -1,62 +0,0 @@ -interpolation_polynomial_degree: 4 -interpolation_xgrid: - - 1.9999999999999954e-07 - - 3.034304765867952e-07 - - 4.6035014748963906e-07 - - 6.984208530700364e-07 - - 1.0596094959101024e-06 - - 1.607585498470808e-06 - - 2.438943292891682e-06 - - 3.7002272069854957e-06 - - 5.613757716930151e-06 - - 8.516806677573355e-06 - - 1.292101569074731e-05 - - 1.9602505002391748e-05 - - 2.97384953722449e-05 - - 4.511438394964044e-05 - - 6.843744918967896e-05 - - 0.00010381172986576898 - - 0.00015745605600841445 - - 0.00023878782918561914 - - 0.00036205449638139736 - - 0.0005487795323670796 - - 0.0008314068836488144 - - 0.0012586797144272762 - - 0.0019034634022867384 - - 0.0028738675812817515 - - 0.004328500638820811 - - 0.006496206194633799 - - 0.009699159574043398 - - 0.014375068581090129 - - 0.02108918668378717 - - 0.030521584007828916 - - 0.04341491741702269 - - 0.060480028754447364 - - 0.08228122126204893 - - 0.10914375746330703 - - 0.14112080644440345 - - 0.17802566042569432 - - 0.2195041265003886 - - 0.2651137041582823 - - 0.31438740076927585 - - 0.3668753186482242 - - 0.4221667753589648 - - 0.4798989029610255 - - 0.5397572337880445 - - 0.601472197967335 - - 0.6648139482473823 - - 0.7295868442414312 - - 0.7956242522922756 - - 0.8627839323906108 - - 0.9309440808717544 - - 1 -interpolation_is_log: true -observables: - F2_total: - - Q2: 50.0 - x: 0.10914375746330703 -prDIS: EM -PolarizationDIS: 0. -ProjectileDIS: electron -TargetDIS: proton -PropagatorCorrection: 0. diff --git a/_data/observable.yaml b/_data/observable.yaml deleted file mode 100644 index d6ffdf7e..00000000 --- a/_data/observable.yaml +++ /dev/null @@ -1,114 +0,0 @@ -interpolation_polynomial_degree: 4 -interpolation_is_log: true -interpolation_xgrid: - - 1.9999999999999954e-07 - - 3.034304765867952e-07 - - 4.6035014748963906e-07 - - 6.984208530700364e-07 - - 1.0596094959101024e-06 - - 1.607585498470808e-06 - - 2.438943292891682e-06 - - 3.7002272069854957e-06 - - 5.613757716930151e-06 - - 8.516806677573355e-06 - - 1.292101569074731e-05 - - 1.9602505002391748e-05 - - 2.97384953722449e-05 - - 4.511438394964044e-05 - - 6.843744918967896e-05 - - 0.00010381172986576898 - - 0.00015745605600841445 - - 0.00023878782918561914 - - 0.00036205449638139736 - - 0.0005487795323670796 - - 0.0008314068836488144 - - 0.0012586797144272762 - - 0.0019034634022867384 - - 0.0028738675812817515 - - 0.004328500638820811 - - 0.006496206194633799 - - 0.009699159574043398 - - 0.014375068581090129 - - 0.02108918668378717 - - 0.030521584007828916 - - 0.04341491741702269 - - 0.060480028754447364 - - 0.08228122126204893 - - 0.10914375746330703 - - 0.14112080644440345 - - 0.17802566042569432 - - 0.2195041265003886 - - 0.2651137041582823 - - 0.31438740076927585 - - 0.3668753186482242 - - 0.4221667753589648 - - 0.4798989029610255 - - 0.5397572337880445 - - 0.601472197967335 - - 0.6648139482473823 - - 0.7295868442414312 - - 0.7956242522922756 - - 0.8627839323906108 - - 0.9309440808717544 - - 1 -observables: - F2_total: - - Q2: 50.0 - x: 1.9999999999999954e-07 - - Q2: 50.0 - x: 6.984208530700364e-07 - - Q2: 50.0 - x: 2.438943292891682e-06 - - Q2: 50.0 - x: 8.516806677573355e-06 - - Q2: 50.0 - x: 2.97384953722449e-05 - - Q2: 50.0 - x: 0.00010381172986576898 - - Q2: 50.0 - x: 0.00036205449638139736 - - Q2: 50.0 - x: 0.0012586797144272762 - - Q2: 50.0 - x: 0.004328500638820811 - - Q2: 50.0 - x: 0.014375068581090129 - - Q2: 50.0 - x: 0.04341491741702269 - - Q2: 50.0 - x: 0.10914375746330703 - - Q2: 50.0 - x: 0.2195041265003886 - - Q2: 50.0 - x: 0.3668753186482242 - - Q2: 50.0 - x: 0.5397572337880445 - - Q2: 50.0 - x: 0.7295868442414312 - - Q2: 50.0 - x: 0.9309440808717544 - # - Q2: 4.0 - # x: 0.10914375746330703 - # - Q2: 7.387504697919031 - # x: 0.10914375746330703 - # - Q2: 13.643806415443933 - # x: 0.10914375746330703 - # - Q2: 25.198420997897458 - # x: 0.10914375746330703 - - Q2: 46.53836337552725 - x: 0.10914375746330703 - - Q2: 85.9505945175426 - x: 0.10914375746330703 - - Q2: 158.74010519682002 - x: 0.10914375746330703 - # - Q2: 293.17331822241715 - # x: 0.10914375746330703 - # - Q2: 541.4548164181543 - # x: 0.10914375746330703 - # - Q2: 1000.0 - # x: 0.10914375746330703 -prDIS: EM -PolarizationDIS: 0. -ProjectileDIS: electron -TargetDIS: proton -PropagatorCorrection: 0. diff --git a/_data/operator.yaml b/_data/operator.yaml deleted file mode 100644 index 463c30d1..00000000 --- a/_data/operator.yaml +++ /dev/null @@ -1,61 +0,0 @@ -ev_op_max_order: 10 -ev_op_iterations: 1 -backward_inversion: "exact" -Q2grid: [50.] -# interpolation -interpolation_is_log: True -interpolation_polynomial_degree: 4 -interpolation_xgrid: - - 1.9999999999999954e-07 - - 3.034304765867952e-07 - - 4.6035014748963906e-07 - - 6.984208530700364e-07 - - 1.0596094959101024e-06 - - 1.607585498470808e-06 - - 2.438943292891682e-06 - - 3.7002272069854957e-06 - - 5.613757716930151e-06 - - 8.516806677573355e-06 - - 1.292101569074731e-05 - - 1.9602505002391748e-05 - - 2.97384953722449e-05 - - 4.511438394964044e-05 - - 6.843744918967896e-05 - - 0.00010381172986576898 - - 0.00015745605600841445 - - 0.00023878782918561914 - - 0.00036205449638139736 - - 0.0005487795323670796 - - 0.0008314068836488144 - - 0.0012586797144272762 - - 0.0019034634022867384 - - 0.0028738675812817515 - - 0.004328500638820811 - - 0.006496206194633799 - - 0.009699159574043398 - - 0.014375068581090129 - - 0.02108918668378717 - - 0.030521584007828916 - - 0.04341491741702269 - - 0.060480028754447364 - - 0.08228122126204893 - - 0.10914375746330703 - - 0.14112080644440345 - - 0.17802566042569432 - - 0.2195041265003886 - - 0.2651137041582823 - - 0.31438740076927585 - - 0.3668753186482242 - - 0.4221667753589648 - - 0.4798989029610255 - - 0.5397572337880445 - - 0.601472197967335 - - 0.6648139482473823 - - 0.7295868442414312 - - 0.7956242522922756 - - 0.8627839323906108 - - 0.9309440808717544 - - 1 -# debug options -debug_skip_non_singlet: False -debug_skip_singlet: False diff --git a/_data/theory_200.yaml b/_data/theory_200.yaml deleted file mode 100644 index 241f0263..00000000 --- a/_data/theory_200.yaml +++ /dev/null @@ -1,46 +0,0 @@ -CKM: 0.97428 0.22530 0.003470 0.22520 0.97345 0.041000 0.00862 0.04030 0.999152 -Comments: NNPDF4.0 NNLO alphas=0.118 -DAMP: 0 -EScaleVar: 1 -FNS: FONLL-C -GF: 1.1663787e-05 -HQ: POLE -IC: 1 -ID: 200 -MP: 0.938 -MW: 80.398 -MZ: 91.1876 -MaxNfAs: 5 -MaxNfPdf: 5 -ModEv: TRN -NfFF: 5 -PTO: 2 -Q0: 1.65 -QED: 0 -Qedref: 1.777 -Qmb: 4.92 -Qmc: 1.51 -Qmt: 172.5 -Qref: 91.2 -SIN2TW: 0.23126 -SxOrd: LL -SxRes: 0 -TMC: 1 -XIF: 1.0 -XIR: 1.0 -alphaqed: 0.007496252 -alphas: 0.118 -global_nx: 0 -kbThr: 1.0 -kcThr: 1.0 -ktThr: 1.0 -mb: 4.92 -mc: 1.51 -mt: 172.5 -nfref: null -nf0: null -fact_to_ren_scale_ratio: 1.0 -kDIScThr: 1.0 -kDISbThr: 1.0 -kDIStThr: 1.0 -IB: 0 diff --git a/_data/theory_208.yaml b/_data/theory_208.yaml deleted file mode 100644 index 3e71049a..00000000 --- a/_data/theory_208.yaml +++ /dev/null @@ -1,47 +0,0 @@ -ID: 208 -PTO: 1 -FNS: FONLL-B -DAMP: 0 -IC: 1 -IB: 0 -ModEv: TRN -XIR: 1.0 -XIF: 1.0 -fact_to_ren_scale_ratio: 1.0 -NfFF: 4 -MaxNfAs: 5 -MaxNfPdf: 5 -Q0: 1.65 -alphas: 0.118 -Qref: 91.2 -nf0: null -nfref: null -QED: 0 -alphaqed: 0.007496252 -Qedref: 1.777 -SxRes: 0 -SxOrd: LL -HQ: POLE -mc: 1.51 -Qmc: 1.51 -kcThr: 1.0 -mb: 4.92 -Qmb: 4.92 -kbThr: 1.0 -mt: 172.5 -Qmt: 172.5 -ktThr: 1.0 -CKM: 0.97428 0.22530 0.003470 0.22520 0.97345 0.041000 0.00862 0.04030 0.999152 -MZ: 91.1876 -MW: 80.398 -GF: 1.1663787e-05 -SIN2TW: 0.23126 -TMC: 1 -MP: 0.938 -Comments: NNPDF4.0 NLO alphas=0.118 -global_nx: 0 -EScaleVar: 1 -kDIScThr: 1.0 -kDISbThr: 1.0 -kDIStThr: 1.0 -IB: 0 diff --git a/_data/theory_213.yaml b/_data/theory_213.yaml deleted file mode 100644 index 9dea01f8..00000000 --- a/_data/theory_213.yaml +++ /dev/null @@ -1,46 +0,0 @@ -CKM: 0.97428 0.22530 0.003470 0.22520 0.97345 0.041000 0.00862 0.04030 0.999152 -Comments: NNPDF4.0 LO fitted charm, alphas=0.118 -DAMP: 0 -EScaleVar: 1 -FNS: ZM-VFNS -GF: 1.1663787e-05 -HQ: POLE -IC: 1 -ID: 213 -MP: 0.938 -MW: 80.398 -MZ: 91.1876 -MaxNfAs: 5 -MaxNfPdf: 5 -ModEv: TRN -NfFF: 5 -PTO: 0 -Q0: 1.65 -QED: 0 -Qedref: 1.777 -Qmb: 4.92 -Qmc: 1.51 -Qmt: 172.5 -Qref: 91.2 -SIN2TW: 0.23126 -SxOrd: LL -SxRes: 0 -TMC: 1 -XIF: 1.0 -XIR: 1.0 -alphaqed: 0.007496252 -alphas: 0.118 -global_nx: 0 -kbThr: 1.0 -kcThr: 1.0 -ktThr: 1.0 -mb: 4.92 -mc: 1.51 -mt: 172.5 -nfref: null -nf0: null -fact_to_ren_scale_ratio: 1.0 -kDIScThr: 1.0 -kDISbThr: 1.0 -kDIStThr: 1.0 -IB: 0 diff --git a/_run.py b/_run.py deleted file mode 100755 index 5f7ca2e9..00000000 --- a/_run.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import logging -import pathlib -import sys - -import eko -import pineappl -import yaml - -from pineko import convolute - -data = pathlib.Path(__file__).absolute().parent / "_data" -myoperator_base_path = data / "myoperator.tar" -mydis_path = data / "mydis.pineappl.lz4" -mydis_yaml_path = data / "mydis.yaml" -mydy_path = data / "ATLASZHIGHMASS49FB.pineappl.lz4" -mydylo_path = data / "ATLASZHIGHMASS49FB-LO.pineappl.lz4" -myttbar_path = data / "ATLAS_TTB_8TEV_TOT.pineappl.lz4" -myttbarlo_path = data / "CMSTTBARTOT5TEV-LO.pineappl.lz4" -myfktable_base_path = data / "myfktable.pineappl.lz4" - -with open(data / "theory_213.yaml") as f: - theory_card = yaml.safe_load(f) - -# activate logging -logStdout = logging.StreamHandler(sys.stdout) -logStdout.setLevel(logging.INFO) -logStdout.setFormatter(logging.Formatter("%(message)s")) -logging.getLogger("eko").handlers = [] -logging.getLogger("eko").addHandler(logStdout) -logging.getLogger("eko").setLevel(logging.INFO) - - -def ensure_eko(pineappl_path, target_filename): - """Generate EKO on the fly""" - if target_filename.exists(): - return - with open(data / "operator.yaml") as f: - operators_card = yaml.safe_load(f) - - pineappl_grid = pineappl.grid.Grid.read(str(pineappl_path)) - x_grid, _pids, muf2_grid = pineappl_grid.axes() - operators_card["Q2grid"] = muf2_grid - operators_card["targetgrid"] = x_grid - ops = eko.run_dglap(theory_card=theory_card, operators_card=operators_card) - ops.dump_tar(target_filename) - - -def generate_yadism(target_filename): - """Generate yadism on the fly""" - import yadism - - t = theory_card.copy() - t["PTO"] = 0 - t["TMC"] = 0 - with open(data / "observable.yaml") as f: - observable_card = yaml.safe_load(f) - dis_cf = yadism.run_yadism(theory=t, observables=observable_card) - dis_cf.dump_pineappl_to_file(str(target_filename), "F2_total") - dis_cf.dump_yaml_to_file(str(mydis_yaml_path)) - - -# collect all path and fake the objects -pineappl_path = myttbar_path -# pineappl_path = mydis_path -if "dis" in str(pineappl_path): - generate_yadism(pineappl_path) -pine_stem = str(pineappl_path.stem).rsplit(".", 1)[0] -myoperator_path = data / (myoperator_base_path.stem + "-" + pine_stem + ".tar") -ensure_eko(pineappl_path, myoperator_path) -fk_stem = str(myfktable_base_path.stem).rsplit(".", 1)[0] -myfktable_path = data / (fk_stem + "-" + pine_stem + ".pineappl.lz4") -# doit -convolute( - pineappl_path, - myoperator_path, - myfktable_path, - 1 + theory_card["PTO"], - 0, - "NNPDF40_lo_as_01180", -) From b69353e7296efa6d7673dadbf519fb7b670848f9 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 22 Mar 2022 11:59:10 +0100 Subject: [PATCH 32/36] Drop module entry point --- src/pineko/__main__.py | 5 ----- src/pineko/cli/_base.py | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 src/pineko/__main__.py diff --git a/src/pineko/__main__.py b/src/pineko/__main__.py deleted file mode 100644 index c5dfec07..00000000 --- a/src/pineko/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -# -*- coding: utf-8 -*- -from . import command - -command() # pylint: disable=no-value-for-parameter -# see https://stackoverflow.com/questions/49680191/click-and-pylint diff --git a/src/pineko/cli/_base.py b/src/pineko/cli/_base.py index c00b4741..54217008 100644 --- a/src/pineko/cli/_base.py +++ b/src/pineko/cli/_base.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +"""Adds global CLI options.""" import pathlib import click @@ -18,6 +19,7 @@ help="Explicitly specify config file (it has to be a valid TOML file).", ) def command(cfg): + """Explicitly specify config file (it has to be a valid TOML file).""" path = configs.detect(cfg) base_configs = configs.load(path) configs.configs = configs.defaults(base_configs) From ada17a499ed7af1b9b2e45df89a81c25d1740d59 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Tue, 22 Mar 2022 12:15:49 +0100 Subject: [PATCH 33/36] Update parser --- src/pineko/parser.py | 41 +++++++++++------------------------------ src/pineko/theory.py | 4 ++-- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/src/pineko/parser.py b/src/pineko/parser.py index 0f227400..74f005c9 100644 --- a/src/pineko/parser.py +++ b/src/pineko/parser.py @@ -1,22 +1,17 @@ # -*- coding: utf-8 -*- # ATTENTION: this is a partial copy from -# https://github.com/NNPDF/nnpdf/blob/ec73c9c5d3765c8b600e3015d3f5d6238dd89400/validphys2/src/validphys/fkparser.py +# https://github.com/NNPDF/nnpdf/blob/7cb96fc05ca2a2914bc1ccc864865e0ca4e66983/validphys2/src/validphys/pineparser.py import yaml -ext = "pineappl.lz4" - - -class PineAPPLEquivalentNotKnown(Exception): - pass - +EXT = "pineappl.lz4" class YamlFileNotFound(FileNotFoundError): - pass + """ymldb file for dataset not found.""" class GridFileNotFound(FileNotFoundError): - pass + """PineAPPL file for FK table not found.""" def _load_yaml(yaml_file): @@ -40,21 +35,18 @@ def _load_yaml(yaml_file): return ret -def get_yaml_information(yaml_file, grids_folder, check_pineappl=False): - """Given a yaml_file, returns the corresponding dictionary. +def get_yaml_information(yaml_file, grids_folder): + """Given a yaml_file, returns the corresponding dictionary and grids. - The dictionary contains all information and an extra field "paths" + The dictionary contains all information and we return an extra field with all the grids to be loaded for the given dataset. - Checks whether the grid is apfelcomb or pineappl: - if check_pineappl is True this function will raise PineAPPLEquivalentNotKnown - if a pineappl grid is not found. Parameters ---------- - yaml_file : Path + yaml_file : pathlib.Path path of the yaml file for the given dataset - grids_folder : Path - path of the theory folder where to find the grids + grids_folder : pathlib.Path + path of the grids folder Returns ------- @@ -65,26 +57,15 @@ def get_yaml_information(yaml_file, grids_folder, check_pineappl=False): """ yaml_content = _load_yaml(yaml_file) - if yaml_content.get("appl") and check_pineappl: - # This might be useful to use the "legacy loader" when there is no actual pineappl available - raise PineAPPLEquivalentNotKnown(yaml_content["target_dataset"]) - # Turn the operands and the members into paths (and check all of them exist) ret = [] for operand in yaml_content["operands"]: tmp = [] for member in operand: - p = grids_folder / f"{member}.{ext}" + p = grids_folder / f"{member}.{EXT}" if not p.exists(): raise GridFileNotFound(f"Failed to find {p}") tmp.append(p) ret.append(tmp) - # We have added a new operation, "NORM" so we need to play this game here: - if yaml_content["operation"] == "NORM": - # Case not yet considered in VP - yaml_content["operation_function"] = "NULL" - else: - yaml_content["operation_function"] = yaml_content["operation"] - return yaml_content, ret diff --git a/src/pineko/theory.py b/src/pineko/theory.py index e1e142fc..5fc2c168 100644 --- a/src/pineko/theory.py +++ b/src/pineko/theory.py @@ -116,7 +116,7 @@ def inherit_grid(self, name, grid, other): other : pathlib.Path new folder """ - new = other / f"{name}.{parser.ext}" + new = other / f"{name}.{parser.EXT}" if new.exists(): if not self.overwrite: rich.print(f"Skipping existing grid {new}") @@ -317,7 +317,7 @@ def fk(self, name, grid_path, tcard, pdf): ) # setup data eko_filename = self.ekos_path() / f"{name}.tar" - fk_filename = self.fks_path / f"{name}.{parser.ext}" + fk_filename = self.fks_path / f"{name}.{parser.EXT}" if fk_filename.exists(): if not self.overwrite: rich.print(f"Skipping existing FK Table {fk_filename}") From 3bf4bbcd2aeb0853273f4b678c22bc4a5d96b2d7 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Wed, 23 Mar 2022 18:17:59 +0100 Subject: [PATCH 34/36] Fix compare arg names --- src/pineko/cli/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pineko/cli/compare.py b/src/pineko/cli/compare.py index 604e4661..9793e3f4 100644 --- a/src/pineko/cli/compare.py +++ b/src/pineko/cli/compare.py @@ -7,10 +7,10 @@ @command.command("compare") @click.argument("pineappl", type=click.Path(exists=True)) -@click.argument("eko", type=click.Path(exists=True)) @click.argument("fktable", type=click.Path()) @click.argument("max_as", type=int) @click.argument("max_al", type=int) +@click.argument("pdf", type=str) def subcommand(pineappl, fktable, max_as, max_al, pdf): """Compare process level PineAPPL grid and derived FkTable. From 0e0b14125c704d710cdc6dd1dcd740141b56065f Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 24 Mar 2022 14:43:43 +0100 Subject: [PATCH 35/36] Recover file base operations --- src/pineko/cli/_base.py | 17 ++--------------- src/pineko/cli/check.py | 2 +- src/pineko/cli/compare.py | 16 ++++++++++------ src/pineko/cli/theory_.py | 19 +++++++++++++++++-- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/pineko/cli/_base.py b/src/pineko/cli/_base.py index 54217008..13c66d64 100644 --- a/src/pineko/cli/_base.py +++ b/src/pineko/cli/_base.py @@ -10,18 +10,5 @@ @click.group(context_settings=CONTEXT_SETTINGS) -@click.option( - "-c", - "--configs", - "cfg", - default=None, - type=click.Path(resolve_path=True, path_type=pathlib.Path), - help="Explicitly specify config file (it has to be a valid TOML file).", -) -def command(cfg): - """Explicitly specify config file (it has to be a valid TOML file).""" - path = configs.detect(cfg) - base_configs = configs.load(path) - configs.configs = configs.defaults(base_configs) - if cfg is not None: - print(f"Configurations loaded from '{path}'") +def command(): + """pineko: Combine PineAPPL grids and EKOs into FK tables""" diff --git a/src/pineko/cli/check.py b/src/pineko/cli/check.py index d135e86e..afe53e55 100644 --- a/src/pineko/cli/check.py +++ b/src/pineko/cli/check.py @@ -18,7 +18,7 @@ def subcommand(pineappl_path, eko_path): provided in EKO, have to expose the same x grid and Q2 grid. """ pineappl_grid = pineappl.grid.Grid.read(pineappl_path) - operators = eko.output.Output.load_yaml_from_file(eko_path) + operators = eko.output.Output.load_tar(eko_path) try: check.check_grid_and_eko_compatible(pineappl_grid, operators) rich.print("[green]Success:[/] grids are compatible") diff --git a/src/pineko/cli/compare.py b/src/pineko/cli/compare.py index 9793e3f4..34e77be9 100644 --- a/src/pineko/cli/compare.py +++ b/src/pineko/cli/compare.py @@ -1,25 +1,29 @@ # -*- coding: utf-8 -*- import click +import pineappl +import rich from .. import comparator from ._base import command @command.command("compare") -@click.argument("pineappl", type=click.Path(exists=True)) -@click.argument("fktable", type=click.Path()) +@click.argument("pineappl_path", type=click.Path(exists=True)) +@click.argument("fktable_path", type=click.Path()) @click.argument("max_as", type=int) @click.argument("max_al", type=int) @click.argument("pdf", type=str) -def subcommand(pineappl, fktable, max_as, max_al, pdf): +def subcommand(pineappl_path, fktable_path, max_as, max_al, pdf): """Compare process level PineAPPL grid and derived FkTable. - The comparison between the grid stored at PINEAPPL path, and the table - stored in FKTABLE, is performed by convoluting both the grids with the PDF + The comparison between the grid stored at PINEAPPL_PATH, and the FK table + stored at FKTABLE_PATH, is performed by convoluting both the grids with the PDF set, evaluating its interpolation grid at the two different scales (thus comparing the EKO evolution, with the one stored inside LHAPDF grid). The comparison involves the orders in QCD and QED up to the maximum power of the coupling corresponding respectively to MAX_AS and MAX_AL. """ - comparator.compare(pineappl, fktable, max_as, max_al, pdf) + pine = pineappl.grid.Grid.read(pineappl_path) + fk = pineappl.fk_table.FkTable.read(fktable_path) + rich.print(comparator.compare(pine, fk, max_as, max_al, pdf)) diff --git a/src/pineko/cli/theory_.py b/src/pineko/cli/theory_.py index f7f4f6c2..df067dd0 100644 --- a/src/pineko/cli/theory_.py +++ b/src/pineko/cli/theory_.py @@ -1,13 +1,28 @@ # -*- coding: utf-8 -*- +import pathlib + import click -from .. import theory +from .. import configs, theory from ._base import command @command.group("theory") -def theory_(): +@click.option( + "-c", + "--configs", + "cfg", + default=None, + type=click.Path(resolve_path=True, path_type=pathlib.Path), + help="Explicitly specify config file (it has to be a valid TOML file).", +) +def theory_(cfg): """Iterate a subcommand on a given theory and list of datasets""" + path = configs.detect(cfg) + base_configs = configs.load(path) + configs.configs = configs.defaults(base_configs) + if cfg is not None: + print(f"Configurations loaded from '{path}'") @theory_.command() From ee6b9da3e7ed6078f2816543cf3952399beaba04 Mon Sep 17 00:00:00 2001 From: Felix Hekhorn Date: Thu, 24 Mar 2022 15:06:21 +0100 Subject: [PATCH 36/36] Run poetry update --- poetry.lock | 142 ++-------------------------------------------------- 1 file changed, 4 insertions(+), 138 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2e2e9a20..9b4aa741 100644 --- a/poetry.lock +++ b/poetry.lock @@ -57,28 +57,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "black" -version = "21.6b0" -description = "The uncompromising code formatter." -category = "dev" -optional = false -python-versions = ">=3.6.2" - -[package.dependencies] -appdirs = "*" -click = ">=7.1.2" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.8.1,<1" -regex = ">=2020.1.8" -toml = ">=0.10.1" - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.6.0)", "aiohttp-cors (>=0.4.0)"] -python2 = ["typed-ast (>=1.4.2)"] -uvloop = ["uvloop (>=0.15.2)"] - [[package]] name = "click" version = "8.0.4" @@ -276,14 +254,6 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "numba" version = "0.55.1" @@ -348,14 +318,6 @@ python-versions = ">=3.6" qa = ["flake8 (==3.8.3)", "mypy (==0.782)"] testing = ["docopt", "pytest (<6.0.0)"] -[[package]] -name = "pathspec" -version = "0.9.0" -description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - [[package]] name = "pdbpp" version = "0.10.3" @@ -567,7 +529,7 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2021.3" +version = "2022.1" description = "World timezone definitions, modern and historical" category = "main" optional = false @@ -581,14 +543,6 @@ category = "main" optional = false python-versions = ">=3.6" -[[package]] -name = "regex" -version = "2022.1.18" -description = "Alternative regular expression module, to replace re." -category = "dev" -optional = false -python-versions = "*" - [[package]] name = "rich" version = "11.2.0" @@ -686,7 +640,7 @@ python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" [metadata] lock-version = "1.1" python-versions = ">=3.8,<3.11" -content-hash = "3150755635fd1c3b27e714700723fa194ec5a7660babdbffd6e9302818fc4d9b" +content-hash = "e5086d2e86e9dac2ac07af43343a13d9f30fcf7bf69d1a671890760d2bf17ab6" [metadata.files] appdirs = [ @@ -713,10 +667,6 @@ backcall = [ {file = "backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, {file = "backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, ] -black = [ - {file = "black-21.6b0-py3-none-any.whl", hash = "sha256:dfb8c5a069012b2ab1e972e7b908f5fb42b6bbabcba0a788b86dc05067c7d9c7"}, - {file = "black-21.6b0.tar.gz", hash = "sha256:dc132348a88d103016726fe360cb9ede02cecf99b76e3660ce6c596be132ce04"}, -] click = [ {file = "click-8.0.4-py3-none-any.whl", hash = "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1"}, {file = "click-8.0.4.tar.gz", hash = "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"}, @@ -897,10 +847,6 @@ mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] numba = [ {file = "numba-0.55.1-1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:be56fb78303973e6c19c7c2759996a5863bac69ca87570543d9f18f2f287a441"}, {file = "numba-0.55.1-1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:ee71407be9cba09b4f68afa668317e97d66d5f83c37ab4caa20d8abcf5fad32b"}, @@ -991,10 +937,6 @@ parso = [ {file = "parso-0.8.3-py2.py3-none-any.whl", hash = "sha256:c001d4636cd3aecdaf33cbb40aebb59b094be2a74c556778ef5576c175e19e75"}, {file = "parso-0.8.3.tar.gz", hash = "sha256:8c07be290bb59f03588915921e29e8a50002acaf2cdc5fa0e0114f91709fafa0"}, ] -pathspec = [ - {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, - {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, -] pdbpp = [ {file = "pdbpp-0.10.3-py2.py3-none-any.whl", hash = "sha256:79580568e33eb3d6f6b462b1187f53e10cd8e4538f7d31495c9181e2cf9665d1"}, {file = "pdbpp-0.10.3.tar.gz", hash = "sha256:d9e43f4fda388eeb365f2887f4e7b66ac09dce9b6236b76f63616530e2f669f5"}, @@ -1078,8 +1020,8 @@ python-dateutil = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] pytz = [ - {file = "pytz-2021.3-py2.py3-none-any.whl", hash = "sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c"}, - {file = "pytz-2021.3.tar.gz", hash = "sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326"}, + {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, + {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, ] pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, @@ -1116,82 +1058,6 @@ pyyaml = [ {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] -regex = [ - {file = "regex-2022.1.18-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:34316bf693b1d2d29c087ee7e4bb10cdfa39da5f9c50fa15b07489b4ab93a1b5"}, - {file = "regex-2022.1.18-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7a0b9f6a1a15d494b35f25ed07abda03209fa76c33564c09c9e81d34f4b919d7"}, - {file = "regex-2022.1.18-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f99112aed4fb7cee00c7f77e8b964a9b10f69488cdff626ffd797d02e2e4484f"}, - {file = "regex-2022.1.18-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a2bf98ac92f58777c0fafc772bf0493e67fcf677302e0c0a630ee517a43b949"}, - {file = "regex-2022.1.18-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8618d9213a863c468a865e9d2ec50221015f7abf52221bc927152ef26c484b4c"}, - {file = "regex-2022.1.18-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b52cc45e71657bc4743a5606d9023459de929b2a198d545868e11898ba1c3f59"}, - {file = "regex-2022.1.18-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e12949e5071c20ec49ef00c75121ed2b076972132fc1913ddf5f76cae8d10b4"}, - {file = "regex-2022.1.18-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b02e3e72665cd02afafb933453b0c9f6c59ff6e3708bd28d0d8580450e7e88af"}, - {file = "regex-2022.1.18-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:abfcb0ef78df0ee9df4ea81f03beea41849340ce33a4c4bd4dbb99e23ec781b6"}, - {file = "regex-2022.1.18-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6213713ac743b190ecbf3f316d6e41d099e774812d470422b3a0f137ea635832"}, - {file = "regex-2022.1.18-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:61ebbcd208d78658b09e19c78920f1ad38936a0aa0f9c459c46c197d11c580a0"}, - {file = "regex-2022.1.18-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:b013f759cd69cb0a62de954d6d2096d648bc210034b79b1881406b07ed0a83f9"}, - {file = "regex-2022.1.18-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9187500d83fd0cef4669385cbb0961e227a41c0c9bc39219044e35810793edf7"}, - {file = "regex-2022.1.18-cp310-cp310-win32.whl", hash = "sha256:94c623c331a48a5ccc7d25271399aff29729fa202c737ae3b4b28b89d2b0976d"}, - {file = "regex-2022.1.18-cp310-cp310-win_amd64.whl", hash = "sha256:1a171eaac36a08964d023eeff740b18a415f79aeb212169080c170ec42dd5184"}, - {file = "regex-2022.1.18-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:49810f907dfe6de8da5da7d2b238d343e6add62f01a15d03e2195afc180059ed"}, - {file = "regex-2022.1.18-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d2f5c3f7057530afd7b739ed42eb04f1011203bc5e4663e1e1d01bb50f813e3"}, - {file = "regex-2022.1.18-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:85ffd6b1cb0dfb037ede50ff3bef80d9bf7fa60515d192403af6745524524f3b"}, - {file = "regex-2022.1.18-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba37f11e1d020969e8a779c06b4af866ffb6b854d7229db63c5fdddfceaa917f"}, - {file = "regex-2022.1.18-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637e27ea1ebe4a561db75a880ac659ff439dec7f55588212e71700bb1ddd5af9"}, - {file = "regex-2022.1.18-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:37978254d9d00cda01acc1997513f786b6b971e57b778fbe7c20e30ae81a97f3"}, - {file = "regex-2022.1.18-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e54a1eb9fd38f2779e973d2f8958fd575b532fe26013405d1afb9ee2374e7ab8"}, - {file = "regex-2022.1.18-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:768632fd8172ae03852e3245f11c8a425d95f65ff444ce46b3e673ae5b057b74"}, - {file = "regex-2022.1.18-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:de2923886b5d3214be951bc2ce3f6b8ac0d6dfd4a0d0e2a4d2e5523d8046fdfb"}, - {file = "regex-2022.1.18-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:1333b3ce73269f986b1fa4d5d395643810074dc2de5b9d262eb258daf37dc98f"}, - {file = "regex-2022.1.18-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:d19a34f8a3429bd536996ad53597b805c10352a8561d8382e05830df389d2b43"}, - {file = "regex-2022.1.18-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:8d2f355a951f60f0843f2368b39970e4667517e54e86b1508e76f92b44811a8a"}, - {file = "regex-2022.1.18-cp36-cp36m-win32.whl", hash = "sha256:2245441445099411b528379dee83e56eadf449db924648e5feb9b747473f42e3"}, - {file = "regex-2022.1.18-cp36-cp36m-win_amd64.whl", hash = "sha256:25716aa70a0d153cd844fe861d4f3315a6ccafce22b39d8aadbf7fcadff2b633"}, - {file = "regex-2022.1.18-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7e070d3aef50ac3856f2ef5ec7214798453da878bb5e5a16c16a61edf1817cc3"}, - {file = "regex-2022.1.18-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22709d701e7037e64dae2a04855021b62efd64a66c3ceed99dfd684bfef09e38"}, - {file = "regex-2022.1.18-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9099bf89078675c372339011ccfc9ec310310bf6c292b413c013eb90ffdcafc"}, - {file = "regex-2022.1.18-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04611cc0f627fc4a50bc4a9a2e6178a974c6a6a4aa9c1cca921635d2c47b9c87"}, - {file = "regex-2022.1.18-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:552a39987ac6655dad4bf6f17dd2b55c7b0c6e949d933b8846d2e312ee80005a"}, - {file = "regex-2022.1.18-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e031899cb2bc92c0cf4d45389eff5b078d1936860a1be3aa8c94fa25fb46ed8"}, - {file = "regex-2022.1.18-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2dacb3dae6b8cc579637a7b72f008bff50a94cde5e36e432352f4ca57b9e54c4"}, - {file = "regex-2022.1.18-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e5c31d70a478b0ca22a9d2d76d520ae996214019d39ed7dd93af872c7f301e52"}, - {file = "regex-2022.1.18-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bb804c7d0bfbd7e3f33924ff49757de9106c44e27979e2492819c16972ec0da2"}, - {file = "regex-2022.1.18-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:36b2d700a27e168fa96272b42d28c7ac3ff72030c67b32f37c05616ebd22a202"}, - {file = "regex-2022.1.18-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:16f81025bb3556eccb0681d7946e2b35ff254f9f888cff7d2120e8826330315c"}, - {file = "regex-2022.1.18-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:da80047524eac2acf7c04c18ac7a7da05a9136241f642dd2ed94269ef0d0a45a"}, - {file = "regex-2022.1.18-cp37-cp37m-win32.whl", hash = "sha256:6ca45359d7a21644793de0e29de497ef7f1ae7268e346c4faf87b421fea364e6"}, - {file = "regex-2022.1.18-cp37-cp37m-win_amd64.whl", hash = "sha256:38289f1690a7e27aacd049e420769b996826f3728756859420eeee21cc857118"}, - {file = "regex-2022.1.18-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6014038f52b4b2ac1fa41a58d439a8a00f015b5c0735a0cd4b09afe344c94899"}, - {file = "regex-2022.1.18-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0b5d6f9aed3153487252d00a18e53f19b7f52a1651bc1d0c4b5844bc286dfa52"}, - {file = "regex-2022.1.18-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9d24b03daf7415f78abc2d25a208f234e2c585e5e6f92f0204d2ab7b9ab48e3"}, - {file = "regex-2022.1.18-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bf594cc7cc9d528338d66674c10a5b25e3cde7dd75c3e96784df8f371d77a298"}, - {file = "regex-2022.1.18-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd914db437ec25bfa410f8aa0aa2f3ba87cdfc04d9919d608d02330947afaeab"}, - {file = "regex-2022.1.18-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90b6840b6448203228a9d8464a7a0d99aa8fa9f027ef95fe230579abaf8a6ee1"}, - {file = "regex-2022.1.18-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11772be1eb1748e0e197a40ffb82fb8fd0d6914cd147d841d9703e2bef24d288"}, - {file = "regex-2022.1.18-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a602bdc8607c99eb5b391592d58c92618dcd1537fdd87df1813f03fed49957a6"}, - {file = "regex-2022.1.18-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7e26eac9e52e8ce86f915fd33380f1b6896a2b51994e40bb094841e5003429b4"}, - {file = "regex-2022.1.18-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:519c0b3a6fbb68afaa0febf0d28f6c4b0a1074aefc484802ecb9709faf181607"}, - {file = "regex-2022.1.18-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:3c7ea86b9ca83e30fa4d4cd0eaf01db3ebcc7b2726a25990966627e39577d729"}, - {file = "regex-2022.1.18-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:51f02ca184518702975b56affde6c573ebad4e411599005ce4468b1014b4786c"}, - {file = "regex-2022.1.18-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:385ccf6d011b97768a640e9d4de25412204fbe8d6b9ae39ff115d4ff03f6fe5d"}, - {file = "regex-2022.1.18-cp38-cp38-win32.whl", hash = "sha256:1f8c0ae0a0de4e19fddaaff036f508db175f6f03db318c80bbc239a1def62d02"}, - {file = "regex-2022.1.18-cp38-cp38-win_amd64.whl", hash = "sha256:760c54ad1b8a9b81951030a7e8e7c3ec0964c1cb9fee585a03ff53d9e531bb8e"}, - {file = "regex-2022.1.18-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:93c20777a72cae8620203ac11c4010365706062aa13aaedd1a21bb07adbb9d5d"}, - {file = "regex-2022.1.18-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6aa427c55a0abec450bca10b64446331b5ca8f79b648531138f357569705bc4a"}, - {file = "regex-2022.1.18-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c38baee6bdb7fe1b110b6b3aaa555e6e872d322206b7245aa39572d3fc991ee4"}, - {file = "regex-2022.1.18-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:752e7ddfb743344d447367baa85bccd3629c2c3940f70506eb5f01abce98ee68"}, - {file = "regex-2022.1.18-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8acef4d8a4353f6678fd1035422a937c2170de58a2b29f7da045d5249e934101"}, - {file = "regex-2022.1.18-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c73d2166e4b210b73d1429c4f1ca97cea9cc090e5302df2a7a0a96ce55373f1c"}, - {file = "regex-2022.1.18-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24c89346734a4e4d60ecf9b27cac4c1fee3431a413f7aa00be7c4d7bbacc2c4d"}, - {file = "regex-2022.1.18-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:596f5ae2eeddb79b595583c2e0285312b2783b0ec759930c272dbf02f851ff75"}, - {file = "regex-2022.1.18-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ecfe51abf7f045e0b9cdde71ca9e153d11238679ef7b5da6c82093874adf3338"}, - {file = "regex-2022.1.18-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1d6301f5288e9bdca65fab3de6b7de17362c5016d6bf8ee4ba4cbe833b2eda0f"}, - {file = "regex-2022.1.18-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:93cce7d422a0093cfb3606beae38a8e47a25232eea0f292c878af580a9dc7605"}, - {file = "regex-2022.1.18-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cf0db26a1f76aa6b3aa314a74b8facd586b7a5457d05b64f8082a62c9c49582a"}, - {file = "regex-2022.1.18-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:defa0652696ff0ba48c8aff5a1fac1eef1ca6ac9c660b047fc8e7623c4eb5093"}, - {file = "regex-2022.1.18-cp39-cp39-win32.whl", hash = "sha256:6db1b52c6f2c04fafc8da17ea506608e6be7086715dab498570c3e55e4f8fbd1"}, - {file = "regex-2022.1.18-cp39-cp39-win_amd64.whl", hash = "sha256:ebaeb93f90c0903233b11ce913a7cb8f6ee069158406e056f884854c737d2442"}, - {file = "regex-2022.1.18.tar.gz", hash = "sha256:97f32dc03a8054a4c4a5ab5d761ed4861e828b2c200febd4e46857069a483916"}, -] rich = [ {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"},