From d9992ae4ff8f34b3dd40b51b3ad78bda60467448 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 24 Oct 2023 16:38:46 +0200 Subject: [PATCH 01/39] Added pkl_file attribute and extra FileTrials methods that save/load pickle files --- .../n3fit/hyper_optimization/filetrials.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index c3d2d8e68f..81179896c1 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -1,7 +1,8 @@ """ Custom hyperopt trial object for persistent file storage - in the form of a json file within the nnfit folder + in the form of json and pickle files within the nnfit folder """ +import pickle import json import logging from validphys.hyperoptplot import HyperoptTrial @@ -61,6 +62,7 @@ class FileTrials(Trials): def __init__(self, replica_path, parameters=None, **kwargs): self._store_trial = False self._json_file = "{0}/tries.json".format(replica_path) + self.pkl_file = "{0}/tries.pkl".format(replica_path) self._parameters = parameters super().__init__(**kwargs) @@ -78,9 +80,7 @@ def refresh(self): local_trials = [] for idx, t in enumerate(self._dynamic_trials): local_trials.append(t) - local_trials[idx]["misc"]["space_vals"] = space_eval_trial( - self._parameters, t - ) + local_trials[idx]["misc"]["space_vals"] = space_eval_trial(self._parameters, t) all_to_str = json.dumps(local_trials, default=str) with open(self._json_file, "w") as f: @@ -95,3 +95,21 @@ def new_trial_ids(self, n): def new_trial_docs(self, tids, specs, results, miscs): self._store_trial = True return super().new_trial_docs(tids, specs, results, miscs) + + def to_pkl(self): + """Dump `FileTrials` object into a pickle file.""" + with open(self.pkl_file, "wb") as file: + pickle.dump(self, file) + + @classmethod + def from_pkl(cls, pickle_filepath): + """Load and return an instance of `FileTrials` from a pickle file. + + If a pickle file from previous run is present this method can be used + to instantiate an initial `FileTrials` object to restart. + """ + try: + with open(pickle_filepath, "rb") as file: + return pickle.load(file) + except FileNotFoundError as err: + log.error("Failed to open pickle file: %s", err) From 062b0308eda42935c4b163aa271eb1e6b8298e14 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 24 Oct 2023 16:45:03 +0200 Subject: [PATCH 02/39] Added --continue option to N3FitApp --- n3fit/src/n3fit/scripts/n3fit_exec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index cf6b276037..2bac7a2941 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -232,6 +232,8 @@ def produce_hyperscanner(self, parameters, hyperscan_config=None, hyperopt=None) if hyperscan_config is None or hyperopt is None: return None + if hyperopt and self.environment.restart: + hyperscan_config.update({'restart': 'true'}) return HyperScanner(parameters, hyperscan_config) @@ -258,6 +260,7 @@ def check_positive(value): return ivalue parser.add_argument("--hyperopt", help="Enable hyperopt scan", default=None, type=int) + parser.add_argument("--continue", help="Enable hyperopt restarts", action="store_true") parser.add_argument("replica", help="MC replica number", type=check_positive) parser.add_argument( "-r", @@ -283,6 +286,7 @@ def run(self): replicas = [replica] self.environment.replicas = NSList(replicas, nskey="replica") self.environment.hyperopt = self.args["hyperopt"] + self.environment.restart = self.args["continue"] super().run() except N3FitError as e: log.error(f"Error in n3fit:\n{e}") From 7dd614b8dc6f8db86f516d4742b1530112ed9163 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 24 Oct 2023 16:49:39 +0200 Subject: [PATCH 03/39] Added restart_hyperopt attribute to HyperScanner --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index 08814dc859..914a03ff9b 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -21,6 +21,7 @@ log = logging.getLogger(__name__) + # These are just wrapper around some hyperopt's sampling expresions defined in here # https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions # with a bit of extra documentation for the ones that are not obvious @@ -174,6 +175,13 @@ def __init__(self, parameters, sampling_dict, steps=5): self.parameters = copy.deepcopy(parameters) self.steps = steps + # adding extra options for restarting + restart_config = sampling_dict.get("restart") + if restart_config: + self.restart_hyperopt = True + else: + self.restart_hyperopt = False + self.hyper_keys = set([]) if "parameters" in sampling_dict: @@ -256,8 +264,7 @@ def stopping(self, min_epochs=None, max_epochs=None, min_patience=None, max_pati stopping_key = "stopping_patience" if min_epochs is not None and max_epochs is not None: - epochs = hp_quniform(epochs_key, min_epochs, max_epochs, - step_size=1000) + epochs = hp_quniform(epochs_key, min_epochs, max_epochs, step_size=1000) self._update_param(epochs_key, epochs) if min_patience is not None or max_patience is not None: @@ -414,8 +421,7 @@ def architecture( units = [] for i in range(n): units_label = "nl{0}:-{1}/{0}".format(n, i) - units_sampler = hp_quniform(units_label, min_units, max_units, - step_size=1) + units_sampler = hp_quniform(units_label, min_units, max_units, step_size=1) units.append(units_sampler) # The number of nodes in the last layer are read from the runcard units.append(output_size) From 3050923b36894a86fcaf7e93714be4f06a1cc4ef Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 24 Oct 2023 16:57:22 +0200 Subject: [PATCH 04/39] Added rstate attribute to FileTrials needed to store the last random.Generator --- .../n3fit/hyper_optimization/filetrials.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index 81179896c1..4b9ebb3b37 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -5,6 +5,7 @@ import pickle import json import logging +from numpy.random._generator import Generator from validphys.hyperoptplot import HyperoptTrial from hyperopt import Trials, space_eval @@ -64,8 +65,32 @@ def __init__(self, replica_path, parameters=None, **kwargs): self._json_file = "{0}/tries.json".format(replica_path) self.pkl_file = "{0}/tries.pkl".format(replica_path) self._parameters = parameters + self._rstate = None super().__init__(**kwargs) + @property + def rstate(self) -> Generator: + """ + Returs the rstate attribute. + + Notes: + Rstate stores a `numpy.random.Generator` which is important to make + hyperopt restarts reproducible in the hyperparameter space. It can + be passed later as the `rstate` parameters of `hyperopt.fmin`. + """ + return self._rstate + + @rstate.setter + def rstate(self, random_generator: Generator) -> None: + """ + Sets the rstate attribute. + + Example: + >>> trials = FileTrials(replica_path_set, parameters=parameters) + >>> trials.rstate = np.random.default_rng(42) + """ + self._rstate = random_generator + def refresh(self): """ This is the "flushing" method which is called at the end of every trial to From 632ff19af43abcddf57bb043e977f7994b6ba7a0 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 24 Oct 2023 17:02:33 +0200 Subject: [PATCH 05/39] Added restart option in hyper_scan_wrapper --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index 914a03ff9b..45f8f6767d 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -16,7 +16,7 @@ import hyperopt import numpy as np from n3fit.backends import MetaModel, MetaLayer -import n3fit.hyper_optimization.filetrials as filetrials +from n3fit.hyper_optimization.filetrials import FileTrials import logging log = logging.getLogger(__name__) @@ -110,7 +110,15 @@ def hyper_scan_wrapper(replica_path_set, model_trainer, hyperscanner, max_evals= # Tell the trainer we are doing hpyeropt model_trainer.set_hyperopt(True, keys=hyperscanner.hyper_keys, status_ok=hyperopt.STATUS_OK) # Generate the trials object - trials = filetrials.FileTrials(replica_path_set, parameters=hyperscanner.as_dict()) + trials = FileTrials(replica_path_set, parameters=hyperscanner.as_dict()) + # Initialize seed for hyperopt + trials.rstate = np.random.default_rng(42) + + # For restarts, reset the state of `FileTrials` saved in the pickle file + if hyperscanner.restart_hyperopt: + pickle_file_to_load = f"{replica_path_set}/tries.pkl" + log.info("Restarting hyperopt run using the pickle file %s", pickle_file_to_load) + trials = FileTrials.from_pkl(pickle_file_to_load) # Perform the scan best = hyperopt.fmin( @@ -120,6 +128,8 @@ def hyper_scan_wrapper(replica_path_set, model_trainer, hyperscanner, max_evals= max_evals=max_evals, show_progressbar=False, trials=trials, + rstate=trials.rstate, + trials_save_file=trials.pkl_file, ) return hyperscanner.space_eval(best) From 4f1bdfa23e474814e0b82d82d5ecd65c797736e7 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 24 Oct 2023 17:05:28 +0200 Subject: [PATCH 06/39] Added docs to hyper_scan_wrapper --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index 45f8f6767d..a913627c7f 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -10,7 +10,7 @@ - a function - a dictionary of spaces of parameters you can do so by simply modifying the wrappers to point somewhere else -(and, of course the function in the fitting action that calls the miniimization). +(and, of course the function in the fitting action that calls the minimization). """ import copy import hyperopt @@ -89,12 +89,13 @@ def hyper_scan_wrapper(replica_path_set, model_trainer, hyperscanner, max_evals= and performs ``max_evals`` evaluations of the hyperparametrizable function of ``model_trainer``. A ``tries.json`` file will be saved in the ``replica_path_set`` folder with the information - of all trials. + of all trials. An additional ``tries.pkl`` file will also be generated in the same folder + that stores the previous states of `FileTrials`. This file can be used for restarting purposes. Parameters ----------- replica_path_set: path - folder where to create the json ``tries.json`` file + folder where to create the json ``tries.json`` and pickle ``tries.pkl`` files model_trainer: :py:class:`n3fit.ModelTrainer.ModelTrainer` a ``ModelTrainer`` object with the ``hyperparametrizable`` method hyperscanner: :py:class:`n3fit.hyper_optimization.hyper_scan.HyperScanner` From 6db6c3089bfdb50c21e509b81e3fb3463c4a35e6 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 24 Oct 2023 17:51:54 +0200 Subject: [PATCH 07/39] Added provisory test for hyperopt restart --- .../n3fit/tests/hyperopt/hyper-quickcard.yml | 136 ++++++++++++++++++ n3fit/src/n3fit/tests/test_hyperopt.py | 74 +++++++++- 2 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml diff --git a/n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml b/n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml new file mode 100644 index 0000000000..2ff9e97640 --- /dev/null +++ b/n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml @@ -0,0 +1,136 @@ +# +# Configuration file for n3fit hyperopt tests +# + +############################################################ +description: n3fit hyperopt test + +############################################################ +# frac: training fraction +# ewk: apply ewk k-factors +# sys: systematics treatment (see systypes) +dataset_inputs: +- { dataset: NMC, frac: 0.5 } +- { dataset: SLACP, frac: 0.5} +- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 } + + +############################################################ +datacuts: + t0pdfset: NNPDF31_nnlo_as_0118 # PDF set to generate t0 covmat + q2min: 3.49 # Q2 minimum + w2min: 12.5 # W2 minimum + combocuts: NNPDF31 # NNPDF3.0 final kin. cuts + jetptcut_tev: 0 # jet pt cut for tevatron + jetptcut_lhc: 0 # jet pt cut for lhc + wptcut_lhc: 30.0 # Minimum pT for W pT diff distributions + jetycut_tev: 1e30 # jet rap. cut for tevatron + jetycut_lhc: 1e30 # jet rap. cut for lhc + dymasscut_min: 0 # dy inv.mass. min cut + dymasscut_max: 1e30 # dy inv.mass. max cut + jetcfactcut: 1e30 # jet cfact. cut + +############################################################ +theory: + theoryid: 399 # database id + +hyperscan_config: + architecture: + n_layers: [2] + min_units: 10 + max_units: 45 + optimizer: + - optimizer_name: 'Nadam' + learning_rate: + sampling: log + min: 1e-4 + max: 1e-2 + clipnorm: + sampling: log + min: 1e-7 + max: 1e-4 + - optimizer_name: 'Adam' + learning_rate: + sampling: log + min: 1e-4 + max: 1e-2 + clipnorm: + sampling: log + min: 1e-7 + max: 1e-4 + +kfold: + target: average + penalties: + - saturation + - patience + - integrability + threshold: 500.0 + partitions: + - datasets: + - NMC + - datasets: + - SLACP + - CMSZDIFF12 + +############################################################ +trvlseed: 2182363835 +nnseed: 4044040809 +mcseed: 1977428487 +genrep: false # true = generate MC replicas, false = use real data + +# The baseline parameters, best ones as taken from table 3.3 in the 2021 paper +# or equivalently the NNPDF40_nnlo_as_01180_1000 runcard +# These are used for parameters that are not hyperoptimized over +parameters: # This defines the parameter dictionary that is passed to the Model Trainer + nodes_per_layer: [25, 20, 8] + activation_per_layer: [tanh, tanh, linear] + initializer: glorot_normal + optimizer: + clipnorm: 6.073e-6 + learning_rate: 2.621e-3 + optimizer_name: Nadam + epochs: 10 + positivity: + initial: 184.8 + multiplier: + integrability: + initial: 10 + multiplier: + stopping_patience: 0.1 + layer_type: dense + dropout: 0.0 + threshold_chi2: 3.5 + +fitting: + savepseudodata: false + # NN23(QED) = sng=0,g=1,v=2,t3=3,ds=4,sp=5,sm=6,(pht=7) + # EVOL(QED) = sng=0,g=1,v=2,v3=3,v8=4,t3=5,t8=6,(pht=7) + # EVOLS(QED)= sng=0,g=1,v=2,v8=4,t3=4,t8=5,ds=6,(pht=7) + # FLVR(QED) = g=0, u=1, ubar=2, d=3, dbar=4, s=5, sbar=6, (pht=7) + fitbasis: EVOL # EVOL (7), EVOLQED (8), etc. + basis: + - {fl: sng, trainable: false, smallx: [1.086, 1.121], largex: [1.459, 3.165]} + - {fl: g, trainable: false, smallx: [0.7832, 1.059], largex: [2.734, 8.173]} + - {fl: v, trainable: false, smallx: [0.5596, 0.7524], largex: [1.534, 3.82]} + - {fl: v3, trainable: false, smallx: [-0.0291, 0.5957], largex: [1.708, 3.706]} + - {fl: v8, trainable: false, smallx: [0.6072, 0.819], largex: [1.519, 3.691]} + - {fl: t3, trainable: false, smallx: [-0.4322, 1.087], largex: [1.718, 3.742]} + - {fl: t8, trainable: false, smallx: [0.6132, 0.958], largex: [1.531, 3.506]} + - {fl: t15, trainable: false, smallx: [1.057, 1.142], largex: [1.469, 3.23]} + +############################################################ +positivity: + posdatasets: + - { dataset: POSF2U, maxlambda: 1e6 } # Positivity Lagrange Multiplier + - { dataset: POSDYS , maxlambda: 1e5 } + +integrability: + integdatasets: + - {dataset: INTEGXT8, maxlambda: 1e2} + +############################################################ +debug: true +maxcores: 28 +parallel_models: true +same_trvl_per_replica: true diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index f06ec133f1..a783a4d83c 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -1,13 +1,85 @@ """ Test hyperoptimization features """ +import pathlib +import shutil +import subprocess as sp +import json from numpy.testing import assert_approx_equal from n3fit.hyper_optimization import rewards + def test_rewards(): - """ Ensure that rewards continue doing what they are supposed to do """ + """Ensure that rewards continue doing what they are supposed to do""" losses = [0.0, 1.0, 2.0] assert_approx_equal(rewards.average(losses), 1.0) assert_approx_equal(rewards.best_worst(losses), 2.0) assert_approx_equal(rewards.std(losses), 0.816496580927726) + + +HYPEROPT_FOLDER = pathlib.Path(__file__).with_name("hyperopt") +QUICKNAME = "quickcard" +EXE = "n3fit" +REPLICA = "1" + + +def load_data(info_file): + """Loads the information of the fit from the json files""" + with open(info_file, "r", encoding='utf-8') as file: + return json.load(file) + + +def test_restart_from_pickle(tmp_path): + """Ensure that our hyperopt restart works as expected""" + # Prepare the run + quickcard = f"hyper-{QUICKNAME}.yml" + quickpath = HYPEROPT_FOLDER / quickcard + # Set up some options + n_trials_stop = 2 + n_trials_total = 4 + output_restart = ( + HYPEROPT_FOLDER / f"run_{n_trials_stop}_trials_and_then_{n_trials_total}_trials" + ) + output_direct = HYPEROPT_FOLDER / f"run_{n_trials_total}_trials" + + # cp runcard to tmp folder + shutil.copy(quickpath, tmp_path) + # run some trials for the first time + sp.run( + f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_stop} " f"-o {output_restart}".split(), + check=True, + cwd=tmp_path, + ) + # restart and calculate more trials + sp.run( + f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_total} " + f"-o {output_restart} --continue".split(), + check=True, + cwd=tmp_path, + ) + # start again and calculate all trials at once + sp.run( + f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_total} " f"-o {output_direct}".split(), + check=True, + cwd=tmp_path, + ) + + # read up generated json files + restart_json_path = f"{output_restart}/nnfit/replica_{REPLICA}/tries.json" + restart_json = load_data(restart_json_path) + direct_json_path = f"{output_direct}/nnfit/replica_{REPLICA}/tries.json" + direct_json = load_data(direct_json_path) + + # Remove the 'output_restart' and 'output_direct' directories + shutil.rmtree(output_restart, ignore_errors=True) + shutil.rmtree(output_direct, ignore_errors=True) + + # minimum check: the generated list of nested dictionaries have same lenght + assert len(restart_json) == len(direct_json) + + for i in range(n_trials_total): + assert restart_json[i]['misc'] == direct_json[i]['misc'] + assert restart_json[i]['state'] == direct_json[i]['state'] + assert restart_json[i]['tid'] == direct_json[i]['tid'] + assert restart_json[i]['results'] == direct_json[i]['results'] From 46ccc57ceca3f9e2330c753d930cf562d6813dcb Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Thu, 26 Oct 2023 21:24:01 +0200 Subject: [PATCH 08/39] Change in the way seeds are generated for each k-fold --- n3fit/src/n3fit/model_trainer.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index b96737481c..5b2ca07227 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -850,7 +850,9 @@ def hyperparametrizable(self, params): # Initialize all photon classes for the different replicas: if self.lux_params: photons = Photon( - theoryid=self.theoryid, lux_params=self.lux_params, replicas=self.replicas, + theoryid=self.theoryid, + lux_params=self.lux_params, + replicas=self.replicas, ) else: photons = None @@ -860,7 +862,9 @@ def hyperparametrizable(self, params): # and the seed needs to be updated accordingly seeds = self._nn_seeds if k > 0: - seeds = [np.random.randint(0, pow(2, 31)) for _ in seeds] + # seeds = [np.random.randint(0, pow(2, 31)) for _ in seeds] + # generate seeds for each k-fold from the input `nnseeds` + seeds = [seed * k for seed in seeds] # Generate the pdf model pdf_models = self._generate_pdf( @@ -922,7 +926,11 @@ def hyperparametrizable(self, params): for model in models.values(): model.compile(**params["optimizer"]) - passed = self._train_and_fit(models["training"], stopping_object, epochs=epochs,) + passed = self._train_and_fit( + models["training"], + stopping_object, + epochs=epochs, + ) if self.mode_hyperopt: # If doing a hyperparameter scan we need to keep track of the loss function From 6d00ade20a5c8a73f9ac4bdeede686a5109d13d3 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Thu, 26 Oct 2023 21:30:47 +0200 Subject: [PATCH 09/39] Fix in test_hyperopt.py --- n3fit/src/n3fit/tests/test_hyperopt.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index a783a4d83c..6d7eb93606 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -5,6 +5,7 @@ import shutil import subprocess as sp import json +import pytest from numpy.testing import assert_approx_equal from n3fit.hyper_optimization import rewards @@ -18,7 +19,7 @@ def test_rewards(): assert_approx_equal(rewards.std(losses), 0.816496580927726) -HYPEROPT_FOLDER = pathlib.Path(__file__).with_name("hyperopt") +HYPEROPT_FOLDER = pathlib.Path(__file__).resolve().parent / "hyperopt" QUICKNAME = "quickcard" EXE = "n3fit" REPLICA = "1" @@ -82,4 +83,8 @@ def test_restart_from_pickle(tmp_path): assert restart_json[i]['misc'] == direct_json[i]['misc'] assert restart_json[i]['state'] == direct_json[i]['state'] assert restart_json[i]['tid'] == direct_json[i]['tid'] - assert restart_json[i]['results'] == direct_json[i]['results'] + assert restart_json[i]['result'] == direct_json[i]['result'] + + +if __name__ == '__main__': + pytest.main() From e80895dd1fc57989031678c78143d10ced78e7fb Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 07:40:52 +0100 Subject: [PATCH 10/39] Fix in test_hyperopt.py --- n3fit/src/n3fit/tests/test_hyperopt.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index 6d7eb93606..7d8998d61c 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -45,25 +45,22 @@ def test_restart_from_pickle(tmp_path): output_direct = HYPEROPT_FOLDER / f"run_{n_trials_total}_trials" # cp runcard to tmp folder - shutil.copy(quickpath, tmp_path) + # shutil.copy(quickpath, tmp_path) # run some trials for the first time sp.run( f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_stop} " f"-o {output_restart}".split(), check=True, - cwd=tmp_path, ) # restart and calculate more trials sp.run( f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_total} " f"-o {output_restart} --continue".split(), check=True, - cwd=tmp_path, ) # start again and calculate all trials at once sp.run( f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_total} " f"-o {output_direct}".split(), check=True, - cwd=tmp_path, ) # read up generated json files @@ -80,6 +77,7 @@ def test_restart_from_pickle(tmp_path): assert len(restart_json) == len(direct_json) for i in range(n_trials_total): + # check that the files share exactly the same hyperopt history assert restart_json[i]['misc'] == direct_json[i]['misc'] assert restart_json[i]['state'] == direct_json[i]['state'] assert restart_json[i]['tid'] == direct_json[i]['tid'] From 4992ba8f14d5178b4ebea043f52461418bc154cf Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 09:51:43 +0100 Subject: [PATCH 11/39] Removed tmp_path argument from test --- n3fit/src/n3fit/tests/test_hyperopt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index 7d8998d61c..42b5421482 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -31,7 +31,7 @@ def load_data(info_file): return json.load(file) -def test_restart_from_pickle(tmp_path): +def test_restart_from_pickle(): """Ensure that our hyperopt restart works as expected""" # Prepare the run quickcard = f"hyper-{QUICKNAME}.yml" From 04fd58504aea488f5b303c234b1bee2e555e1006 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 12:03:21 +0100 Subject: [PATCH 12/39] Updated test to make it work in CI/CD --- n3fit/src/n3fit/tests/test_hyperopt.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index 42b5421482..928ea146f8 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -19,7 +19,8 @@ def test_rewards(): assert_approx_equal(rewards.std(losses), 0.816496580927726) -HYPEROPT_FOLDER = pathlib.Path(__file__).resolve().parent / "hyperopt" +# HYPEROPT_FOLDER = pathlib.Path(__file__).resolve().parent / "hyperopt" +HYPEROPT_FOLDER = pathlib.Path(__file__).with_name("hyperopt") QUICKNAME = "quickcard" EXE = "n3fit" REPLICA = "1" @@ -82,7 +83,3 @@ def test_restart_from_pickle(): assert restart_json[i]['state'] == direct_json[i]['state'] assert restart_json[i]['tid'] == direct_json[i]['tid'] assert restart_json[i]['result'] == direct_json[i]['result'] - - -if __name__ == '__main__': - pytest.main() From 1490757e7226acf9f971d0e620cd12eaf6498a73 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 13:28:40 +0100 Subject: [PATCH 13/39] Updated tries.json and tries.pkl paths definition without format --- n3fit/src/n3fit/hyper_optimization/filetrials.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index 4b9ebb3b37..bbd1039770 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -2,12 +2,14 @@ Custom hyperopt trial object for persistent file storage in the form of json and pickle files within the nnfit folder """ -import pickle import json import logging +import pickle + +from hyperopt import Trials, space_eval from numpy.random._generator import Generator + from validphys.hyperoptplot import HyperoptTrial -from hyperopt import Trials, space_eval log = logging.getLogger(__name__) @@ -62,8 +64,8 @@ class FileTrials(Trials): def __init__(self, replica_path, parameters=None, **kwargs): self._store_trial = False - self._json_file = "{0}/tries.json".format(replica_path) - self.pkl_file = "{0}/tries.pkl".format(replica_path) + self._json_file = replica_path / "tries.json" + self.pkl_file = replica_path / "tries.pkl" self._parameters = parameters self._rstate = None super().__init__(**kwargs) From 01842c8167444512b80de98d3d744c72b5b05521 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 13:34:07 +0100 Subject: [PATCH 14/39] Run isort in test_hyperopt.py --- n3fit/src/n3fit/tests/test_hyperopt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index 928ea146f8..e423e88435 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -1,13 +1,13 @@ """ Test hyperoptimization features """ +import json import pathlib import shutil import subprocess as sp -import json -import pytest from numpy.testing import assert_approx_equal + from n3fit.hyper_optimization import rewards From 3ea6feac8ae4b9edefaaf079d59087d09f6b84ff Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 15:33:07 +0100 Subject: [PATCH 15/39] Changed keyword from continue to restart --- n3fit/src/n3fit/model_trainer.py | 24 ++++-------------------- n3fit/src/n3fit/scripts/n3fit_exec.py | 4 ++-- n3fit/src/n3fit/tests/test_hyperopt.py | 2 +- 3 files changed, 7 insertions(+), 23 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 5b2ca07227..5001d92c02 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -222,13 +222,7 @@ def __init__( "folds": [], "posdatasets": [], } - self.experimental = { - "output": [], - "expdata": [], - "ndata": 0, - "model": None, - "folds": [], - } + self.experimental = {"output": [], "expdata": [], "ndata": 0, "model": None, "folds": []} self._fill_the_dictionaries() @@ -483,11 +477,7 @@ def _model_generation(self, xinput, pdf_models, partition, partition_idx): except ValueError: pass - models = { - "training": training, - "validation": validation, - "experimental": experimental, - } + models = {"training": training, "validation": validation, "experimental": experimental} return models @@ -850,9 +840,7 @@ def hyperparametrizable(self, params): # Initialize all photon classes for the different replicas: if self.lux_params: photons = Photon( - theoryid=self.theoryid, - lux_params=self.lux_params, - replicas=self.replicas, + theoryid=self.theoryid, lux_params=self.lux_params, replicas=self.replicas ) else: photons = None @@ -926,11 +914,7 @@ def hyperparametrizable(self, params): for model in models.values(): model.compile(**params["optimizer"]) - passed = self._train_and_fit( - models["training"], - stopping_object, - epochs=epochs, - ) + passed = self._train_and_fit(models["training"], stopping_object, epochs=epochs) if self.mode_hyperopt: # If doing a hyperparameter scan we need to keep track of the loss function diff --git a/n3fit/src/n3fit/scripts/n3fit_exec.py b/n3fit/src/n3fit/scripts/n3fit_exec.py index 2bac7a2941..32ddf5260f 100755 --- a/n3fit/src/n3fit/scripts/n3fit_exec.py +++ b/n3fit/src/n3fit/scripts/n3fit_exec.py @@ -260,7 +260,7 @@ def check_positive(value): return ivalue parser.add_argument("--hyperopt", help="Enable hyperopt scan", default=None, type=int) - parser.add_argument("--continue", help="Enable hyperopt restarts", action="store_true") + parser.add_argument("--restart", help="Enable hyperopt restarts", action="store_true") parser.add_argument("replica", help="MC replica number", type=check_positive) parser.add_argument( "-r", @@ -286,7 +286,7 @@ def run(self): replicas = [replica] self.environment.replicas = NSList(replicas, nskey="replica") self.environment.hyperopt = self.args["hyperopt"] - self.environment.restart = self.args["continue"] + self.environment.restart = self.args["restart"] super().run() except N3FitError as e: log.error(f"Error in n3fit:\n{e}") diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index e423e88435..8bca8c1855 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -55,7 +55,7 @@ def test_restart_from_pickle(): # restart and calculate more trials sp.run( f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_total} " - f"-o {output_restart} --continue".split(), + f"-o {output_restart} --restart".split(), check=True, ) # start again and calculate all trials at once From 693fde1d6be2c13fd1c9ee08c6b3f4c3d3f60fa6 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 18:11:23 +0100 Subject: [PATCH 16/39] Changed step_size to 10 in stopping method --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index a913627c7f..d9d1792acd 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -13,11 +13,13 @@ (and, of course the function in the fitting action that calls the minimization). """ import copy +import logging + import hyperopt import numpy as np -from n3fit.backends import MetaModel, MetaLayer + +from n3fit.backends import MetaLayer, MetaModel from n3fit.hyper_optimization.filetrials import FileTrials -import logging log = logging.getLogger(__name__) @@ -275,7 +277,7 @@ def stopping(self, min_epochs=None, max_epochs=None, min_patience=None, max_pati stopping_key = "stopping_patience" if min_epochs is not None and max_epochs is not None: - epochs = hp_quniform(epochs_key, min_epochs, max_epochs, step_size=1000) + epochs = hp_quniform(epochs_key, min_epochs, max_epochs, step_size=10) self._update_param(epochs_key, epochs) if min_patience is not None or max_patience is not None: @@ -351,11 +353,7 @@ def optimizer(self, optimizers): self._update_param(opt_key, opt_val) def positivity( - self, - min_multiplier=None, - max_multiplier=None, - min_initial=None, - max_initial=None, + self, min_multiplier=None, max_multiplier=None, min_initial=None, max_initial=None ): """ Modifies the following entries of the `parameters` dictionary: From 53237df2012b85e7f62963644acf5bd9636c64eb Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 30 Oct 2023 18:12:32 +0100 Subject: [PATCH 17/39] Fix test_hyperopt to use runcard in regressions folder --- n3fit/src/n3fit/tests/test_hyperopt.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index 8bca8c1855..8c6a2faf71 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -19,8 +19,7 @@ def test_rewards(): assert_approx_equal(rewards.std(losses), 0.816496580927726) -# HYPEROPT_FOLDER = pathlib.Path(__file__).resolve().parent / "hyperopt" -HYPEROPT_FOLDER = pathlib.Path(__file__).with_name("hyperopt") +REGRESSION_FOLDER = pathlib.Path(__file__).with_name("regressions") QUICKNAME = "quickcard" EXE = "n3fit" REPLICA = "1" @@ -32,35 +31,38 @@ def load_data(info_file): return json.load(file) -def test_restart_from_pickle(): +def test_restart_from_pickle(tmp_path): """Ensure that our hyperopt restart works as expected""" # Prepare the run quickcard = f"hyper-{QUICKNAME}.yml" - quickpath = HYPEROPT_FOLDER / quickcard + quickpath = REGRESSION_FOLDER / quickcard # Set up some options n_trials_stop = 2 n_trials_total = 4 output_restart = ( - HYPEROPT_FOLDER / f"run_{n_trials_stop}_trials_and_then_{n_trials_total}_trials" + REGRESSION_FOLDER / f"run_{n_trials_stop}_trials_and_then_{n_trials_total}_trials" ) - output_direct = HYPEROPT_FOLDER / f"run_{n_trials_total}_trials" + output_direct = REGRESSION_FOLDER / f"run_{n_trials_total}_trials" # cp runcard to tmp folder - # shutil.copy(quickpath, tmp_path) + shutil.copy(quickpath, tmp_path) # run some trials for the first time sp.run( f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_stop} " f"-o {output_restart}".split(), + cwd=tmp_path, check=True, ) # restart and calculate more trials sp.run( f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_total} " f"-o {output_restart} --restart".split(), + cwd=tmp_path, check=True, ) # start again and calculate all trials at once sp.run( f"{EXE} {quickpath} {REPLICA} --hyperopt {n_trials_total} " f"-o {output_direct}".split(), + cwd=tmp_path, check=True, ) From 209a1d2c59b40774277d9879d94ba25d517bddfa Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 31 Oct 2023 07:48:47 +0100 Subject: [PATCH 18/39] Changed step_size to 1 in stopping method --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index d9d1792acd..08d16199ad 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -277,7 +277,7 @@ def stopping(self, min_epochs=None, max_epochs=None, min_patience=None, max_pati stopping_key = "stopping_patience" if min_epochs is not None and max_epochs is not None: - epochs = hp_quniform(epochs_key, min_epochs, max_epochs, step_size=10) + epochs = hp_quniform(epochs_key, min_epochs, max_epochs, step_size=1) self._update_param(epochs_key, epochs) if min_patience is not None or max_patience is not None: From 8280524de1b8620c0c5cce5fcbf81058af9a4ea1 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 31 Oct 2023 07:54:22 +0100 Subject: [PATCH 19/39] Removed hyperopt folder from tests; using regression folder --- .../n3fit/tests/hyperopt/hyper-quickcard.yml | 136 ------------------ 1 file changed, 136 deletions(-) delete mode 100644 n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml diff --git a/n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml b/n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml deleted file mode 100644 index 2ff9e97640..0000000000 --- a/n3fit/src/n3fit/tests/hyperopt/hyper-quickcard.yml +++ /dev/null @@ -1,136 +0,0 @@ -# -# Configuration file for n3fit hyperopt tests -# - -############################################################ -description: n3fit hyperopt test - -############################################################ -# frac: training fraction -# ewk: apply ewk k-factors -# sys: systematics treatment (see systypes) -dataset_inputs: -- { dataset: NMC, frac: 0.5 } -- { dataset: SLACP, frac: 0.5} -- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 } - - -############################################################ -datacuts: - t0pdfset: NNPDF31_nnlo_as_0118 # PDF set to generate t0 covmat - q2min: 3.49 # Q2 minimum - w2min: 12.5 # W2 minimum - combocuts: NNPDF31 # NNPDF3.0 final kin. cuts - jetptcut_tev: 0 # jet pt cut for tevatron - jetptcut_lhc: 0 # jet pt cut for lhc - wptcut_lhc: 30.0 # Minimum pT for W pT diff distributions - jetycut_tev: 1e30 # jet rap. cut for tevatron - jetycut_lhc: 1e30 # jet rap. cut for lhc - dymasscut_min: 0 # dy inv.mass. min cut - dymasscut_max: 1e30 # dy inv.mass. max cut - jetcfactcut: 1e30 # jet cfact. cut - -############################################################ -theory: - theoryid: 399 # database id - -hyperscan_config: - architecture: - n_layers: [2] - min_units: 10 - max_units: 45 - optimizer: - - optimizer_name: 'Nadam' - learning_rate: - sampling: log - min: 1e-4 - max: 1e-2 - clipnorm: - sampling: log - min: 1e-7 - max: 1e-4 - - optimizer_name: 'Adam' - learning_rate: - sampling: log - min: 1e-4 - max: 1e-2 - clipnorm: - sampling: log - min: 1e-7 - max: 1e-4 - -kfold: - target: average - penalties: - - saturation - - patience - - integrability - threshold: 500.0 - partitions: - - datasets: - - NMC - - datasets: - - SLACP - - CMSZDIFF12 - -############################################################ -trvlseed: 2182363835 -nnseed: 4044040809 -mcseed: 1977428487 -genrep: false # true = generate MC replicas, false = use real data - -# The baseline parameters, best ones as taken from table 3.3 in the 2021 paper -# or equivalently the NNPDF40_nnlo_as_01180_1000 runcard -# These are used for parameters that are not hyperoptimized over -parameters: # This defines the parameter dictionary that is passed to the Model Trainer - nodes_per_layer: [25, 20, 8] - activation_per_layer: [tanh, tanh, linear] - initializer: glorot_normal - optimizer: - clipnorm: 6.073e-6 - learning_rate: 2.621e-3 - optimizer_name: Nadam - epochs: 10 - positivity: - initial: 184.8 - multiplier: - integrability: - initial: 10 - multiplier: - stopping_patience: 0.1 - layer_type: dense - dropout: 0.0 - threshold_chi2: 3.5 - -fitting: - savepseudodata: false - # NN23(QED) = sng=0,g=1,v=2,t3=3,ds=4,sp=5,sm=6,(pht=7) - # EVOL(QED) = sng=0,g=1,v=2,v3=3,v8=4,t3=5,t8=6,(pht=7) - # EVOLS(QED)= sng=0,g=1,v=2,v8=4,t3=4,t8=5,ds=6,(pht=7) - # FLVR(QED) = g=0, u=1, ubar=2, d=3, dbar=4, s=5, sbar=6, (pht=7) - fitbasis: EVOL # EVOL (7), EVOLQED (8), etc. - basis: - - {fl: sng, trainable: false, smallx: [1.086, 1.121], largex: [1.459, 3.165]} - - {fl: g, trainable: false, smallx: [0.7832, 1.059], largex: [2.734, 8.173]} - - {fl: v, trainable: false, smallx: [0.5596, 0.7524], largex: [1.534, 3.82]} - - {fl: v3, trainable: false, smallx: [-0.0291, 0.5957], largex: [1.708, 3.706]} - - {fl: v8, trainable: false, smallx: [0.6072, 0.819], largex: [1.519, 3.691]} - - {fl: t3, trainable: false, smallx: [-0.4322, 1.087], largex: [1.718, 3.742]} - - {fl: t8, trainable: false, smallx: [0.6132, 0.958], largex: [1.531, 3.506]} - - {fl: t15, trainable: false, smallx: [1.057, 1.142], largex: [1.469, 3.23]} - -############################################################ -positivity: - posdatasets: - - { dataset: POSF2U, maxlambda: 1e6 } # Positivity Lagrange Multiplier - - { dataset: POSDYS , maxlambda: 1e5 } - -integrability: - integdatasets: - - {dataset: INTEGXT8, maxlambda: 1e2} - -############################################################ -debug: true -maxcores: 28 -parallel_models: true -same_trvl_per_replica: true From d45dfea7d193825c6e476617db70bac2cac25374 Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Tue, 31 Oct 2023 08:03:34 +0100 Subject: [PATCH 20/39] Update doc string in hyper_scan_wrapper Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index 08d16199ad..fa1483a2cc 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -92,7 +92,7 @@ def hyper_scan_wrapper(replica_path_set, model_trainer, hyperscanner, max_evals= A ``tries.json`` file will be saved in the ``replica_path_set`` folder with the information of all trials. An additional ``tries.pkl`` file will also be generated in the same folder - that stores the previous states of `FileTrials`. This file can be used for restarting purposes. + that stores the previous states of `FileTrials`, this file can be used for restarting purposes. Parameters ----------- From 1c702298e53c1874a0d16720e92e183449a8e037 Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Tue, 31 Oct 2023 08:04:15 +0100 Subject: [PATCH 21/39] Update doc string n3fit/src/n3fit/hyper_optimization/hyper_scan.py Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index fa1483a2cc..6cb8ab398c 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -97,7 +97,7 @@ def hyper_scan_wrapper(replica_path_set, model_trainer, hyperscanner, max_evals= Parameters ----------- replica_path_set: path - folder where to create the json ``tries.json`` and pickle ``tries.pkl`` files + folder where to create the ``tries.json`` and ``tries.pkl`` files model_trainer: :py:class:`n3fit.ModelTrainer.ModelTrainer` a ``ModelTrainer`` object with the ``hyperparametrizable`` method hyperscanner: :py:class:`n3fit.hyper_optimization.hyper_scan.HyperScanner` From fa2a6dd8a790d3749316eb8ac7417c0571a8f460 Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Tue, 31 Oct 2023 08:05:32 +0100 Subject: [PATCH 22/39] Updated np.random.default_rng to be constant Co-authored-by: Roy Stegeman --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index 6cb8ab398c..b2928d52eb 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -115,7 +115,7 @@ def hyper_scan_wrapper(replica_path_set, model_trainer, hyperscanner, max_evals= # Generate the trials object trials = FileTrials(replica_path_set, parameters=hyperscanner.as_dict()) # Initialize seed for hyperopt - trials.rstate = np.random.default_rng(42) + trials.rstate = np.random.default_rng(HYPEROPT_SEED) # For restarts, reset the state of `FileTrials` saved in the pickle file if hyperscanner.restart_hyperopt: From 5ef77876072d55dfa1d5cda033f69af589416044 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 31 Oct 2023 08:08:33 +0100 Subject: [PATCH 23/39] Added HYPEROPT_SEED as constant at the top of the hyper_scan.py module --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index b2928d52eb..5482131da2 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -23,6 +23,8 @@ log = logging.getLogger(__name__) +HYPEROPT_SEED = 42 + # These are just wrapper around some hyperopt's sampling expresions defined in here # https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions From 3ce533ed64f81cc30ee324f06275c8306fd4608c Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 31 Oct 2023 09:48:09 +0100 Subject: [PATCH 24/39] Provisory suggestion to generate reproducible seeds for each fold in ModelTrainer.hyperparametrizable --- n3fit/src/n3fit/model_trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 5001d92c02..6f5d404d26 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -852,7 +852,8 @@ def hyperparametrizable(self, params): if k > 0: # seeds = [np.random.randint(0, pow(2, 31)) for _ in seeds] # generate seeds for each k-fold from the input `nnseeds` - seeds = [seed * k for seed in seeds] + seeds_gerator = [np.random.default_rng(seed=seed) for seed in seeds] + seeds = [generator.random() for generator in seeds_gerator] # Generate the pdf model pdf_models = self._generate_pdf( From aaafc80507c9180fa641e45882d1a78d74a06b0a Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 31 Oct 2023 09:48:09 +0100 Subject: [PATCH 25/39] Provisory suggestion to generate reproducible seeds for each fold in ModelTrainer.hyperparametrizable --- n3fit/src/n3fit/model_trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 5001d92c02..1dbdc9c9ba 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -852,7 +852,8 @@ def hyperparametrizable(self, params): if k > 0: # seeds = [np.random.randint(0, pow(2, 31)) for _ in seeds] # generate seeds for each k-fold from the input `nnseeds` - seeds = [seed * k for seed in seeds] + seeds_gerator = [np.random.default_rng(seed=seed) for seed in seeds] + seeds = [generator.integers(0, pow(2, 31)) for generator in seeds_gerator] # Generate the pdf model pdf_models = self._generate_pdf( From c872bccf1ce96289125311bea1be641f9f9cd65e Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Tue, 31 Oct 2023 16:00:40 +0100 Subject: [PATCH 26/39] Added new integer generator --- n3fit/src/n3fit/model_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 1dbdc9c9ba..9b3f1ec110 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -851,7 +851,7 @@ def hyperparametrizable(self, params): seeds = self._nn_seeds if k > 0: # seeds = [np.random.randint(0, pow(2, 31)) for _ in seeds] - # generate seeds for each k-fold from the input `nnseeds` + # generate random integers for each k-fold from the input `nnseeds` seeds_gerator = [np.random.default_rng(seed=seed) for seed in seeds] seeds = [generator.integers(0, pow(2, 31)) for generator in seeds_gerator] From 5f2f6aa035eb8006fa9c9d61b94e2fb0b24aeef3 Mon Sep 17 00:00:00 2001 From: RoyStegeman Date: Wed, 1 Nov 2023 10:02:34 +0000 Subject: [PATCH 27/39] different nnseed per fold --- n3fit/src/n3fit/model_trainer.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 9b3f1ec110..ea8bdb0b7a 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -850,10 +850,11 @@ def hyperparametrizable(self, params): # and the seed needs to be updated accordingly seeds = self._nn_seeds if k > 0: - # seeds = [np.random.randint(0, pow(2, 31)) for _ in seeds] # generate random integers for each k-fold from the input `nnseeds` - seeds_gerator = [np.random.default_rng(seed=seed) for seed in seeds] - seeds = [generator.integers(0, pow(2, 31)) for generator in seeds_gerator] + # we generate new seeds to avoid the integer overflow that may + # occur when doing k*nnseeds + rngs = [np.random.default_rng(seed=seed) for seed in seeds] + seeds = [generator.integers(1, pow(2, 30)) * k for generator in rngs] # Generate the pdf model pdf_models = self._generate_pdf( From 08af2ddf44cd99ca6fd1cd175800cbec55d282aa Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Mon, 6 Nov 2023 07:40:02 +0100 Subject: [PATCH 28/39] Updated filetrials.py; fix in rstate docstring Co-authored-by: Tanjona Rabemananjara --- n3fit/src/n3fit/hyper_optimization/filetrials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index bbd1039770..83f292c30e 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -73,7 +73,7 @@ def __init__(self, replica_path, parameters=None, **kwargs): @property def rstate(self) -> Generator: """ - Returs the rstate attribute. + Returns the rstate attribute. Notes: Rstate stores a `numpy.random.Generator` which is important to make From 1f2445c509e4da4507de08f34e2d0c2308cffa94 Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Mon, 6 Nov 2023 07:43:09 +0100 Subject: [PATCH 29/39] Updated filetrials.py; fix in rstate docstring Co-authored-by: Tanjona Rabemananjara --- n3fit/src/n3fit/hyper_optimization/filetrials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index 83f292c30e..bda7fac5a3 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -76,7 +76,7 @@ def rstate(self) -> Generator: Returns the rstate attribute. Notes: - Rstate stores a `numpy.random.Generator` which is important to make + :func:`rstate` stores a `numpy.random.Generator` which is important to make hyperopt restarts reproducible in the hyperparameter space. It can be passed later as the `rstate` parameters of `hyperopt.fmin`. """ From 9dd531c7448ba69d46afed39ec9b3b8aeb1dd00a Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Mon, 6 Nov 2023 07:43:47 +0100 Subject: [PATCH 30/39] Update n3fit/src/n3fit/hyper_optimization/filetrials.py Co-authored-by: Tanjona Rabemananjara --- n3fit/src/n3fit/hyper_optimization/filetrials.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index bda7fac5a3..bca14268c8 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -87,9 +87,10 @@ def rstate(self, random_generator: Generator) -> None: """ Sets the rstate attribute. - Example: - >>> trials = FileTrials(replica_path_set, parameters=parameters) - >>> trials.rstate = np.random.default_rng(42) + Example + -------- + >>> trials = FileTrials(replica_path_set, parameters=parameters) + >>> trials.rstate = np.random.default_rng(42) """ self._rstate = random_generator From 5ce4c02663dfe16bf1af617f886376970b979337 Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Mon, 6 Nov 2023 07:45:11 +0100 Subject: [PATCH 31/39] Updated filetrials.py; fix in from_pkl docstring Co-authored-by: Tanjona Rabemananjara --- n3fit/src/n3fit/hyper_optimization/filetrials.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index bca14268c8..96ff684bda 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -131,7 +131,8 @@ def to_pkl(self): @classmethod def from_pkl(cls, pickle_filepath): - """Load and return an instance of `FileTrials` from a pickle file. + """ + Load and return an instance of `FileTrials` from a pickle file. If a pickle file from previous run is present this method can be used to instantiate an initial `FileTrials` object to restart. From 2e2b9f906e419df3c849baa71443b6368e63160d Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Mon, 6 Nov 2023 07:49:26 +0100 Subject: [PATCH 32/39] Updated hyper_scan.py; fix in HyperScanner Co-authored-by: Tanjona Rabemananjara --- n3fit/src/n3fit/hyper_optimization/hyper_scan.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py index 5482131da2..174e921677 100644 --- a/n3fit/src/n3fit/hyper_optimization/hyper_scan.py +++ b/n3fit/src/n3fit/hyper_optimization/hyper_scan.py @@ -192,10 +192,7 @@ def __init__(self, parameters, sampling_dict, steps=5): # adding extra options for restarting restart_config = sampling_dict.get("restart") - if restart_config: - self.restart_hyperopt = True - else: - self.restart_hyperopt = False + self.restart_hyperopt = True if restart_config else False self.hyper_keys = set([]) From 7d3c219ce88dacaebd66e8c40e09ac8eff7bd11d Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 6 Nov 2023 08:50:42 +0100 Subject: [PATCH 33/39] Fix in test_hyperopt; added output_restart and output_direct as sub-directories of tmp_path --- n3fit/src/n3fit/tests/test_hyperopt.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index 8c6a2faf71..cecc747452 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -39,10 +39,8 @@ def test_restart_from_pickle(tmp_path): # Set up some options n_trials_stop = 2 n_trials_total = 4 - output_restart = ( - REGRESSION_FOLDER / f"run_{n_trials_stop}_trials_and_then_{n_trials_total}_trials" - ) - output_direct = REGRESSION_FOLDER / f"run_{n_trials_total}_trials" + output_restart = tmp_path / f"run_{n_trials_stop}_trials_and_then_{n_trials_total}_trials" + output_direct = tmp_path / f"run_{n_trials_total}_trials" # cp runcard to tmp folder shutil.copy(quickpath, tmp_path) @@ -72,10 +70,6 @@ def test_restart_from_pickle(tmp_path): direct_json_path = f"{output_direct}/nnfit/replica_{REPLICA}/tries.json" direct_json = load_data(direct_json_path) - # Remove the 'output_restart' and 'output_direct' directories - shutil.rmtree(output_restart, ignore_errors=True) - shutil.rmtree(output_direct, ignore_errors=True) - # minimum check: the generated list of nested dictionaries have same lenght assert len(restart_json) == len(direct_json) From 200c7076c2db21eb20bc0d277b7fc6add42dc584 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 6 Nov 2023 10:30:08 +0100 Subject: [PATCH 34/39] Fix in from_pkl exception --- n3fit/src/n3fit/hyper_optimization/filetrials.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index 96ff684bda..e4e9e13d96 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -141,4 +141,7 @@ def from_pkl(cls, pickle_filepath): with open(pickle_filepath, "rb") as file: return pickle.load(file) except FileNotFoundError as err: - log.error("Failed to open pickle file: %s", err) + raise FileNotFoundError( + "Failed to open 'tries.pkl' pickle file for restarting. " + f"Please ensure it is located in: {pickle_filepath}" + ) from err From ba84878353e3ed950426ae6a12674abce2dfb903 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 6 Nov 2023 10:50:14 +0100 Subject: [PATCH 35/39] Removed static typing and added more descritive docstring to rstate attribute --- n3fit/src/n3fit/hyper_optimization/filetrials.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/filetrials.py b/n3fit/src/n3fit/hyper_optimization/filetrials.py index e4e9e13d96..4c9ee07647 100644 --- a/n3fit/src/n3fit/hyper_optimization/filetrials.py +++ b/n3fit/src/n3fit/hyper_optimization/filetrials.py @@ -7,7 +7,6 @@ import pickle from hyperopt import Trials, space_eval -from numpy.random._generator import Generator from validphys.hyperoptplot import HyperoptTrial @@ -71,7 +70,7 @@ def __init__(self, replica_path, parameters=None, **kwargs): super().__init__(**kwargs) @property - def rstate(self) -> Generator: + def rstate(self): """ Returns the rstate attribute. @@ -83,12 +82,18 @@ def rstate(self) -> Generator: return self._rstate @rstate.setter - def rstate(self, random_generator: Generator) -> None: + def rstate(self, random_generator): """ Sets the rstate attribute. + # Arguments: + - `random_generator`: `numpy.random.Generator` + Example -------- + >>> import numpy as np + >>> from n3fit.hyper_optimization.filetrials import FileTrials + >>> >>> trials = FileTrials(replica_path_set, parameters=parameters) >>> trials.rstate = np.random.default_rng(42) """ From d720280dabd5595d527b574e40fc6dc291cb9edb Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 6 Nov 2023 15:59:32 +0100 Subject: [PATCH 36/39] Added documentation on hyperopt restarts --- doc/sphinx/source/n3fit/hyperopt.rst | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/doc/sphinx/source/n3fit/hyperopt.rst b/doc/sphinx/source/n3fit/hyperopt.rst index f345e49604..dff30580c7 100644 --- a/doc/sphinx/source/n3fit/hyperopt.rst +++ b/doc/sphinx/source/n3fit/hyperopt.rst @@ -320,10 +320,10 @@ Changing the hyperoptimization target ----------------------------------- Beyond the usual :math:`\chi2`-based optimization figures above, it is possible to utilize other measures as the target for hyperoptimization. -One possibility is to use a :ref:`future test`-based metric for which the goal is not to get the minimum :math:`\chi2` but to get the same :math:`\chi2` (with PDF errors considered) for different datasets. The idea is that this way we select models of which the prediction is stable upon variations in the dataset. +One possibility is to use a :ref:`future test`-based metric for which the goal is not to get the minimum :math:`\chi2` but to get the same :math:`\chi2` (with PDF errors considered) for different datasets. The idea is that this way we select models of which the prediction is stable upon variations in the dataset. In order to obtain the PDF errors used in the figure of merit it is necessary to run multiple replicas, luckily ``n3fit`` provides such a possibility also during hyperoptimization. -Take the following modifications to a normal hyperopt runcard +Take the following modifications to a normal hyperopt runcard (note that for convenience we take the trials directly from a previous run, so we don't have to create a new hyperopt configuration dictionary). @@ -345,7 +345,7 @@ hyperopt configuration dictionary). kfold: target: fit_future_tests - partitions: + partitions: - datasets: - HERACOMBCCEP - HERACOMBCCEM @@ -370,3 +370,19 @@ The figure of merit will be the difference between the :math:`\chi2` of the seco .. math:: L_{\rm hyperopt} = \chi^{2}_{(1) \rm pdferr} - \chi^{2}_{(2)} + + +Restarting hyperoptimization runs +--------------------------------- + +In addition to the ``tries.json`` files, hyperparameter scans also produce ``tries.pkl`` `pickle ` files, +which are located in the same directory as the corresponding ``tries.json`` file. +The generated ``tries.pkl`` file stores the complete history of a previous hyperoptimization run, making it possible to resume the process using the ``hyperopt`` framework. +To achieve this, you can utilize the ``--restart`` option within the ``n3fit`` command, e.g.,: + +.. code-block:: bash + + n3fit runcard.yml 1 -r 10 --hyperopt 20 --restart + +The above command example is effective when the number of saved trials in the ``test_run/nnfit/replica_1/tries.pkl`` is +less than ``20``. If there are ``20`` or more saved trials, ``n3fit`` will simply terminate, displaying the best results. From 260716fe0684d987b66d7e9b87d1639b3d6ca98d Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 6 Nov 2023 16:07:03 +0100 Subject: [PATCH 37/39] Fix in reference docs --- doc/sphinx/source/n3fit/hyperopt.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/n3fit/hyperopt.rst b/doc/sphinx/source/n3fit/hyperopt.rst index dff30580c7..c9ab2b65a0 100644 --- a/doc/sphinx/source/n3fit/hyperopt.rst +++ b/doc/sphinx/source/n3fit/hyperopt.rst @@ -375,7 +375,7 @@ The figure of merit will be the difference between the :math:`\chi2` of the seco Restarting hyperoptimization runs --------------------------------- -In addition to the ``tries.json`` files, hyperparameter scans also produce ``tries.pkl`` `pickle ` files, +In addition to the ``tries.json`` files, hyperparameter scans also produce ``tries.pkl`` `pickle _` files, which are located in the same directory as the corresponding ``tries.json`` file. The generated ``tries.pkl`` file stores the complete history of a previous hyperoptimization run, making it possible to resume the process using the ``hyperopt`` framework. To achieve this, you can utilize the ``--restart`` option within the ``n3fit`` command, e.g.,: From a00cc9f71890dc13c2c6d6147970a55ff14f7980 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 6 Nov 2023 16:09:17 +0100 Subject: [PATCH 38/39] Fix in reference to pickle --- doc/sphinx/source/n3fit/hyperopt.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/n3fit/hyperopt.rst b/doc/sphinx/source/n3fit/hyperopt.rst index c9ab2b65a0..85b2a883a0 100644 --- a/doc/sphinx/source/n3fit/hyperopt.rst +++ b/doc/sphinx/source/n3fit/hyperopt.rst @@ -375,7 +375,7 @@ The figure of merit will be the difference between the :math:`\chi2` of the seco Restarting hyperoptimization runs --------------------------------- -In addition to the ``tries.json`` files, hyperparameter scans also produce ``tries.pkl`` `pickle _` files, +In addition to the ``tries.json`` files, hyperparameter scans also produce ``tries.pkl`` `pickle `_ files, which are located in the same directory as the corresponding ``tries.json`` file. The generated ``tries.pkl`` file stores the complete history of a previous hyperoptimization run, making it possible to resume the process using the ``hyperopt`` framework. To achieve this, you can utilize the ``--restart`` option within the ``n3fit`` command, e.g.,: From ec4d507683634b9320f82f56533062df7f21336e Mon Sep 17 00:00:00 2001 From: Carlos Murilo Romero Rocha <114645116+Cmurilochem@users.noreply.github.com> Date: Mon, 6 Nov 2023 19:58:40 +0100 Subject: [PATCH 39/39] Fix in hyperopt.rst docs Co-authored-by: Roy Stegeman --- doc/sphinx/source/n3fit/hyperopt.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx/source/n3fit/hyperopt.rst b/doc/sphinx/source/n3fit/hyperopt.rst index 85b2a883a0..7eb5ae2343 100644 --- a/doc/sphinx/source/n3fit/hyperopt.rst +++ b/doc/sphinx/source/n3fit/hyperopt.rst @@ -378,7 +378,7 @@ Restarting hyperoptimization runs In addition to the ``tries.json`` files, hyperparameter scans also produce ``tries.pkl`` `pickle `_ files, which are located in the same directory as the corresponding ``tries.json`` file. The generated ``tries.pkl`` file stores the complete history of a previous hyperoptimization run, making it possible to resume the process using the ``hyperopt`` framework. -To achieve this, you can utilize the ``--restart`` option within the ``n3fit`` command, e.g.,: +To achieve this, you can use the ``--restart`` option within the ``n3fit`` command, e.g.,: .. code-block:: bash