From 9ca6a550a6226337331ebbb6f982ea1b68e5358f Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Wed, 28 Feb 2024 16:00:54 +0100 Subject: [PATCH 1/8] Added convert_string_to_numpy function --- validphys2/src/validphys/hyperoptplot.py | 51 ++++++++++++++++-------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index c09b42f041..ad0a063b28 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -30,6 +30,37 @@ regex_not_op = re.compile(r"[\w\.]+") +def convert_string_to_numpy(matrix_string: str) -> np.ndarray: + """Process arrays given as strings and transform them into numpy arrays. + + Args: + matrix_string (str): array as string. + + Returns + numpy array. + """ + # Step 1: Remove newline characters and extra spaces + matrix_string = matrix_string.replace('\n', ' ') + matrix_string = ' '.join(matrix_string.split()) + + # Step 2: Split the string into rows + rows = matrix_string.split('] [') + + # Prepare an empty list to store the parsed numbers + matrix_list = [] + + # Step 3: Process each row + for row in rows: + # Remove any brackets and split the row into numbers + cleaned_row = row.replace('[', '').replace(']', '') + row_numbers = cleaned_row.split(' ') + # Convert numbers to floats and append to matrix_list + matrix_list.append([float(0) if num == 'nan' else float(num) for num in row_numbers if num]) + + # Step 4: Convert the list of lists to a NumPy array + return np.array(matrix_list) + + class HyperoptTrial: """ Hyperopt trial class. @@ -317,8 +348,8 @@ def parse_statistics(trial): # std = results["kfold_meta"]["hyper_std"] # dict_out["avg"] = average # dict_out["std"] = std - dict_out["hlosses"] = results["kfold_meta"]["hyper_losses"] - dict_out["vlosses"] = results["kfold_meta"]["validation_losses"] + dict_out["hlosses"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses"]) + dict_out["vlosses"] = convert_string_to_numpy(results["kfold_meta"]["validation_losses"]) return dict_out @@ -727,22 +758,10 @@ def plot_scans(df, best_df, plotting_parameter, include_best=True): best_df[key] = original_best.apply(lambda x: x[0]) ordering_true, best_x = order_axis(df, best_df, key=key) ax = sns.violinplot( - x=key, - y=loss_k, - data=df, - ax=ax, - palette="Set2", - cut=0.0, - order=ordering_true, + x=key, y=loss_k, data=df, ax=ax, palette="Set2", cut=0.0, order=ordering_true ) ax = sns.stripplot( - x=key, - y=loss_k, - data=df, - ax=ax, - color="gray", - order=ordering_true, - alpha=0.4, + x=key, y=loss_k, data=df, ax=ax, color="gray", order=ordering_true, alpha=0.4 ) # Finally plot the "best" one, which will be first From 3aa64ee54e3dffd4b9c024b6cb2a8b6dc5000672 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 4 Mar 2024 11:51:28 +0100 Subject: [PATCH 2/8] Added 'best_chi2_worst_phi2' loss type --- validphys2/src/validphys/hyperoptplot.py | 25 +++++++++++-- .../src/validphys/scripts/vp_hyperoptplot.py | 36 ++++++++++--------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index ad0a063b28..6dbac5ac1f 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -350,6 +350,7 @@ def parse_statistics(trial): # dict_out["std"] = std dict_out["hlosses"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses"]) dict_out["vlosses"] = convert_string_to_numpy(results["kfold_meta"]["validation_losses"]) + dict_out["hlosses_phi2"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses_phi2"]) return dict_out @@ -383,6 +384,9 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe test_loss = np.array(trial_dict["hlosses"]).max() elif loss_target == "std": test_loss = np.array(trial_dict["hlosses"]).std() + elif loss_target == "min_chi2_max_phi2": + test_loss = np.array(trial_dict["hlosses"]).mean() + phi2 = np.array(trial_dict["hlosses_phi2"]).mean() loss = val_loss * validation_multiplier + test_loss * test_f if ( @@ -396,6 +400,8 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe loss *= 10 trial_dict["loss"] = loss + if loss_target == "min_chi2_max_phi2": + trial_dict["loss_inverse_phi2"] = np.reciprocal(phi2) def generate_dictionary( @@ -570,9 +576,22 @@ def hyperopt_dataframe(commandline_args): # Now select the best one best_idx = dataframe.loss.idxmin() - best_trial_series = dataframe.loc[best_idx] - # Make into a dataframe and transpose or the plotting code will complain - best_trial = best_trial_series.to_frame().T + + if args.loss_target == "min_chi2_max_phi2": + minimum = dataframe.loss[best_idx] + std = np.std(dataframe.loss) + lim_max = dataframe.loss[best_idx] + std + # select rows with chi2 losses within the best point and lim_max + selected_chi2 = dataframe[(dataframe.loss >= minimum) & (dataframe.loss <= lim_max)] + # among the selected points, select the nth lowest in 1/phi2 + selected_phi2 = selected_chi2.loss_inverse_phi2.nsmallest(args.max_phi2_n_models) + # find the location of these points in the dataframe + indices = dataframe[dataframe['loss_inverse_phi2'].isin(selected_phi2)].index + best_trial = dataframe.loc[indices] + else: + best_trial_series = dataframe.loc[best_idx] + # Make into a dataframe and transpose or the plotting code will complain + best_trial = best_trial_series.to_frame().T log.info("Best setup:") with pd.option_context("display.max_rows", None, "display.max_columns", None): diff --git a/validphys2/src/validphys/scripts/vp_hyperoptplot.py b/validphys2/src/validphys/scripts/vp_hyperoptplot.py index 1faa875070..19c187181b 100644 --- a/validphys2/src/validphys/scripts/vp_hyperoptplot.py +++ b/validphys2/src/validphys/scripts/vp_hyperoptplot.py @@ -1,30 +1,35 @@ -from validphys.app import App -from validphys.loader import Loader, HyperscanNotFound -from validphys import hyperplottemplates -from reportengine.compat import yaml -import pwd +import logging import os +import pwd -import logging +from reportengine.compat import yaml +from validphys import hyperplottemplates +from validphys.app import App +from validphys.loader import HyperscanNotFound, Loader log = logging.getLogger(__name__) class HyperoptPlotApp(App): def add_positional_arguments(self, parser): - """ Wrapper around argumentparser """ + """Wrapper around argumentparser""" # Hyperopt settings parser.add_argument( - "hyperopt_name", - help="Folder of the hyperopt fit to generate the report for", + "hyperopt_name", help="Folder of the hyperopt fit to generate the report for" ) parser.add_argument( "-l", "--loss_target", help="Choice for the definition of target loss", - choices=['average', 'best_worst', 'std'], + choices=['average', 'best_worst', 'std', 'min_chi2_max_phi2'], default='average', ) + parser.add_argument( + "--max_phi2_n_models", + help="If --loss_target=best_chi2_worst_phi2, outputs n models with the highest phi2.", + type=int, + default=1, + ) parser.add_argument( "-v", "--val_multiplier", @@ -73,16 +78,12 @@ def add_positional_arguments(self, parser): type=str, default=pwd.getpwuid(os.getuid())[4].replace(",", ""), ) - parser.add_argument( - "--title", - help="Add custom title to the report's meta data", - type=str, - ) + parser.add_argument("--title", help="Add custom title to the report's meta data", type=str) parser.add_argument( "--keywords", help="Add keywords to the report's meta data. The keywords must be provided as a list", type=list, - default=[] + default=[], ) args = parser.parse_args() @@ -127,7 +128,8 @@ def complete_mapping(self): "combine": args["combine"], "autofilter": args["autofilter"], "debug": args["debug"], - "loss_target": args["loss_target"] + "loss_target": args["loss_target"], + "max_phi2_n_models": args["max_phi2_n_models"], } try: From bba511a37e144323e1ea9f6a8388e44048042816 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Fri, 8 Mar 2024 10:31:22 +0100 Subject: [PATCH 3/8] Replace phi2 to phi and set std as the spread of chi2 among the replicas of the best fit --- validphys2/src/validphys/hyperoptplot.py | 25 +++++++++++-------- .../src/validphys/scripts/vp_hyperoptplot.py | 8 +++--- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index 6dbac5ac1f..735b3bfac0 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -350,7 +350,7 @@ def parse_statistics(trial): # dict_out["std"] = std dict_out["hlosses"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses"]) dict_out["vlosses"] = convert_string_to_numpy(results["kfold_meta"]["validation_losses"]) - dict_out["hlosses_phi2"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses_phi2"]) + dict_out["hlosses_phi"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses_phi"]) return dict_out @@ -384,9 +384,12 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe test_loss = np.array(trial_dict["hlosses"]).max() elif loss_target == "std": test_loss = np.array(trial_dict["hlosses"]).std() - elif loss_target == "min_chi2_max_phi2": + elif loss_target == "min_chi2_max_phi": test_loss = np.array(trial_dict["hlosses"]).mean() - phi2 = np.array(trial_dict["hlosses_phi2"]).mean() + phi = np.array(trial_dict["hlosses_phi"]).mean() + else: + raise ValueError(f"Loss target {loss_target} not recognized.") + loss = val_loss * validation_multiplier + test_loss * test_f if ( @@ -400,8 +403,9 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe loss *= 10 trial_dict["loss"] = loss - if loss_target == "min_chi2_max_phi2": - trial_dict["loss_inverse_phi2"] = np.reciprocal(phi2) + if loss_target == "min_chi2_max_phi": + trial_dict["loss_phi"] = phi + trial_dict["loss_inverse_phi"] = np.reciprocal(phi) def generate_dictionary( @@ -577,16 +581,17 @@ def hyperopt_dataframe(commandline_args): # Now select the best one best_idx = dataframe.loss.idxmin() - if args.loss_target == "min_chi2_max_phi2": + if args.loss_target == "min_chi2_max_phi": minimum = dataframe.loss[best_idx] - std = np.std(dataframe.loss) + # set std to the spread of chi2 among the replicas of the best fit + std = np.std(dataframe.hlosses[best_idx]) lim_max = dataframe.loss[best_idx] + std # select rows with chi2 losses within the best point and lim_max selected_chi2 = dataframe[(dataframe.loss >= minimum) & (dataframe.loss <= lim_max)] - # among the selected points, select the nth lowest in 1/phi2 - selected_phi2 = selected_chi2.loss_inverse_phi2.nsmallest(args.max_phi2_n_models) + # among the selected points, select the nth lowest in 1/phi + selected_phi = selected_chi2.loss_inverse_phi.nsmallest(args.max_phi_n_models) # find the location of these points in the dataframe - indices = dataframe[dataframe['loss_inverse_phi2'].isin(selected_phi2)].index + indices = dataframe[dataframe['loss_inverse_phi'].isin(selected_phi)].index best_trial = dataframe.loc[indices] else: best_trial_series = dataframe.loc[best_idx] diff --git a/validphys2/src/validphys/scripts/vp_hyperoptplot.py b/validphys2/src/validphys/scripts/vp_hyperoptplot.py index 19c187181b..25e5cedd9a 100644 --- a/validphys2/src/validphys/scripts/vp_hyperoptplot.py +++ b/validphys2/src/validphys/scripts/vp_hyperoptplot.py @@ -21,12 +21,12 @@ def add_positional_arguments(self, parser): "-l", "--loss_target", help="Choice for the definition of target loss", - choices=['average', 'best_worst', 'std', 'min_chi2_max_phi2'], + choices=['average', 'best_worst', 'std', 'min_chi2_max_phi'], default='average', ) parser.add_argument( - "--max_phi2_n_models", - help="If --loss_target=best_chi2_worst_phi2, outputs n models with the highest phi2.", + "--max_phi_n_models", + help="If --loss_target=best_chi2_worst_phi, outputs n models with the highest phi.", type=int, default=1, ) @@ -129,7 +129,7 @@ def complete_mapping(self): "autofilter": args["autofilter"], "debug": args["debug"], "loss_target": args["loss_target"], - "max_phi2_n_models": args["max_phi2_n_models"], + "max_phi_n_models": args["max_phi_n_models"], } try: From 0e8b688711418e00d432c4c7f7f0d2a6e6379ca7 Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Fri, 8 Mar 2024 16:52:00 +0100 Subject: [PATCH 4/8] Added template plot and best set up --- validphys2/src/validphys/hyperoptplot.py | 149 ++++++++++++++++-- .../validphys/hyperplottemplates/report.md | 3 + 2 files changed, 139 insertions(+), 13 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index 735b3bfac0..157acfecf0 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -406,6 +406,9 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe if loss_target == "min_chi2_max_phi": trial_dict["loss_phi"] = phi trial_dict["loss_inverse_phi"] = np.reciprocal(phi) + else: + trial_dict["loss_phi"] = pd.NA + trial_dict["loss_inverse_phi"] = pd.NA def generate_dictionary( @@ -580,6 +583,7 @@ def hyperopt_dataframe(commandline_args): # Now select the best one best_idx = dataframe.loss.idxmin() + best_models = pd.DataFrame() if args.loss_target == "min_chi2_max_phi": minimum = dataframe.loss[best_idx] @@ -592,7 +596,14 @@ def hyperopt_dataframe(commandline_args): selected_phi = selected_chi2.loss_inverse_phi.nsmallest(args.max_phi_n_models) # find the location of these points in the dataframe indices = dataframe[dataframe['loss_inverse_phi'].isin(selected_phi)].index - best_trial = dataframe.loc[indices] + # save the best models + best_models = dataframe.loc[indices] + # add to the best models the one with the lowest chi2 in case it is not included + # if best_idx not in best_models.index: + # best_models = pd.concat([best_models, dataframe.loc[[best_idx]]]) + # set best trial to the model with the lowest 1/phi among the best chi2 models + best_inverse_phi_idx = best_models.loss_inverse_phi.idxmin() + best_trial = best_models.loc[best_inverse_phi_idx].to_frame().T else: best_trial_series = dataframe.loc[best_idx] # Make into a dataframe and transpose or the plotting code will complain @@ -602,7 +613,7 @@ def hyperopt_dataframe(commandline_args): with pd.option_context("display.max_rows", None, "display.max_columns", None): log.info(best_trial) - return dataframe, best_trial + return dataframe, best_trial, best_models @table @@ -610,7 +621,7 @@ def best_setup(hyperopt_dataframe, hyperscan_config, commandline_args): """ Generates a clean table with information on the hyperparameter settings of the best setup. """ - _, best_trial = hyperopt_dataframe + _, best_trial, _ = hyperopt_dataframe best_idx = best_trial.index[0] best_trial = best_trial.rename(index={best_idx: "parameter settings"}) best_trial = best_trial[ @@ -627,20 +638,132 @@ def best_setup(hyperopt_dataframe, hyperscan_config, commandline_args): "initializer", "dropout", "loss", + "loss_inverse_phi", ] ] - best_trial.insert(11, "loss type", commandline_args["loss_target"]) + best_trial.insert(12, "loss type", commandline_args["loss_target"]) best_trial = best_trial.T return best_trial +@figure +def plot_models_for_min_chi2_max_phi(hyperopt_dataframe, commandline_args): + """ + Generates plot of the model index as a function of loss and 1/phi + """ + if commandline_args["loss_target"] != 'min_chi2_max_phi': + fig, _ = plotutils.subplots() + return fig + + dataframe, _, best_models = hyperopt_dataframe + fig = plot_models(dataframe, best_models) + return fig + + +def plot_models(dataframe, best_models): + """ + Model plot called by `plot_models_for_min_chi2_max_phi`. + """ + # Reset the index of the dataframe + dataframe = dataframe.reset_index(drop=True) + # dataframe.sort_values(by=["loss"], inplace=True) + + figs, ax = plotutils.subplots() + + # define x and y + x = dataframe.index + y = dataframe['loss'] + z = dataframe['loss_inverse_phi'] + + indices = [] + # Iterate over each value in best_models['iteration'] + for value in best_models['iteration']: + # Find the index of this value in dataframe['iteration'] + index = dataframe[dataframe['iteration'] == value].index + # Append the index to the list + indices.extend(index) + + x_best_chi2_worst_phi = indices + y_best_chi2_worst_phi = best_models['loss'] + + # minimum chi2 point + min_idx = dataframe.loss.idxmin() + y_min = dataframe.loss[min_idx] + x_min = x[min_idx] + + print(f"Minimum chi2: x={x_min} y={y_min}") + + print("Selected points from the min_chi2_max_phi:") + print(y_best_chi2_worst_phi) + + # some color definitions + color_chi2 = 'red' + color_phi = 'blue' + color_min = 'gray' + + ax.plot(x, y, marker="o", label='', linewidth=0, markersize=10, color=color_chi2) + + # highlight the min chi2 point + ax.scatter( + x_min, + y_min, + color=color_min, + s=300, + marker='o', + alpha=0.5, + label=r'Min $\left<\chi^{2}\right>$', + ) + + # highlight all selected points from the best_chi2_worst_phi algorithm + ax.scatter( + x_best_chi2_worst_phi, + y_best_chi2_worst_phi, + color='cyan', + s=300, + marker='o', + alpha=0.5, + label=r'Min $\left<\chi^{2}\right>$ Max $\left<\varphi\right>$ models', + ) + + # highlight area for all x and y up to std + ax.axhspan( + y_min, + y_min + np.std(dataframe.hlosses[min_idx]), + color='yellow', + alpha=0.3, + label=r'Min $\left<\chi^{2}\right>$ + $\sigma$ band', + ) + + ax.set_xlabel('trial') + ax.set_ylabel(r'Loss', color=color_chi2) + ax.tick_params(axis='y', colors=color_chi2) + ax.set_title('') + # add a legend with a solid white background + ax.legend(facecolor='white', framealpha=1, shadow=True) + ax.grid(True) + + # second plot phi + ax2 = ax.twinx() + ax2.plot( + x, z, marker="s", label='', linewidth=0, markersize=5, color=color_phi, fillstyle='none' + ) + ax2.set_ylabel(r'$1/\left<\varphi\right>$', color=color_phi) + ax2.tick_params(axis='y', colors=color_phi) + ax2.legend() + ax2.grid(False) + ax2.spines['right'].set_visible(True) + + figs.tight_layout() + return figs + + @table def hyperopt_table(hyperopt_dataframe): """ Generates a table containing complete information on all the tested setups that passed the filters set in the commandline arguments. """ - dataframe, _ = hyperopt_dataframe + dataframe, _, _ = hyperopt_dataframe dataframe.sort_values(by=["loss"], inplace=True) return dataframe @@ -650,7 +773,7 @@ def plot_iterations(hyperopt_dataframe): """ Generates a scatter plot of the loss as a function of the iteration index. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe fig = plot_scans(dataframe, best_trial, "iteration") return fig @@ -660,7 +783,7 @@ def plot_optimizers(hyperopt_dataframe): """ Generates a violin plot of the loss per optimizer. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe fig = plot_scans(dataframe, best_trial, "optimizer") return fig @@ -670,7 +793,7 @@ def plot_clipnorm(hyperopt_dataframe, optimizer_name): """ Generates a scatter plot of the loss as a function of the clipnorm for a given optimizer. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe filtered_dataframe = dataframe[dataframe.optimizer == optimizer_name] best_filtered_idx = filtered_dataframe.loss.idxmin() best_idx = best_trial.iteration.iloc[0] @@ -687,7 +810,7 @@ def plot_learning_rate(hyperopt_dataframe, optimizer_name): """ Generates a scatter plot of the loss as a function of the learning rate for a given optimizer. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe filtered_dataframe = dataframe[dataframe.optimizer == optimizer_name] best_filtered_idx = filtered_dataframe.loss.idxmin() best_idx = best_trial.iteration.iloc[0] @@ -704,7 +827,7 @@ def plot_initializer(hyperopt_dataframe): """ Generates a violin plot of the loss per initializer. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe fig = plot_scans(dataframe, best_trial, "initializer") return fig @@ -714,7 +837,7 @@ def plot_epochs(hyperopt_dataframe): """ Generates a scatter plot of the loss as a function the number of epochs. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe fig = plot_scans(dataframe, best_trial, "epochs") return fig @@ -724,7 +847,7 @@ def plot_number_of_layers(hyperopt_dataframe): """ Generates a violin plot of the loss as a function of the number of layers of the model. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe fig = plot_scans(dataframe, best_trial, "number_of_layers") return fig @@ -734,7 +857,7 @@ def plot_activation_per_layer(hyperopt_dataframe): """ Generates a violin plot of the loss per activation function. """ - dataframe, best_trial = hyperopt_dataframe + dataframe, best_trial, _ = hyperopt_dataframe fig = plot_scans(dataframe, best_trial, "activation_per_layer") return fig diff --git a/validphys2/src/validphys/hyperplottemplates/report.md b/validphys2/src/validphys/hyperplottemplates/report.md index c0f5a76ada..344a28009f 100644 --- a/validphys2/src/validphys/hyperplottemplates/report.md +++ b/validphys2/src/validphys/hyperplottemplates/report.md @@ -3,6 +3,9 @@ ## Best Setup {@ best_setup @} +## Models +{@ plot_models_for_min_chi2_max_phi @} + ## Optimizers {@ plot_optimizers @} From 13f43c4235625ec785eb60926f947cf2b5b6046c Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Mon, 18 Mar 2024 09:15:41 +0100 Subject: [PATCH 5/8] Remove convert_string_to_numpy function --- validphys2/src/validphys/hyperoptplot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index 157acfecf0..4c781cf677 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -348,9 +348,9 @@ def parse_statistics(trial): # std = results["kfold_meta"]["hyper_std"] # dict_out["avg"] = average # dict_out["std"] = std - dict_out["hlosses"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses"]) - dict_out["vlosses"] = convert_string_to_numpy(results["kfold_meta"]["validation_losses"]) - dict_out["hlosses_phi"] = convert_string_to_numpy(results["kfold_meta"]["hyper_losses_phi"]) + dict_out["hlosses"] = results["kfold_meta"]["hyper_losses"] + dict_out["vlosses"] = results["kfold_meta"]["validation_losses"] + dict_out["hlosses_phi"] = results["kfold_meta"]["hyper_losses_phi"] return dict_out From 535ea801aa6ed5c6de702635e61f4857d19323bc Mon Sep 17 00:00:00 2001 From: Cmurilochem Date: Fri, 14 Jun 2024 13:27:35 +0200 Subject: [PATCH 6/8] Replace phi by phi2 --- validphys2/src/validphys/hyperoptplot.py | 43 ++++++++++--------- .../validphys/hyperplottemplates/report.md | 2 +- .../src/validphys/scripts/vp_hyperoptplot.py | 8 ++-- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index 4c781cf677..8c88adf9ad 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -384,9 +384,10 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe test_loss = np.array(trial_dict["hlosses"]).max() elif loss_target == "std": test_loss = np.array(trial_dict["hlosses"]).std() - elif loss_target == "min_chi2_max_phi": + elif loss_target == "min_chi2_max_phi2": test_loss = np.array(trial_dict["hlosses"]).mean() - phi = np.array(trial_dict["hlosses_phi"]).mean() + phi_per_fold = np.array(trial_dict["hlosses_phi"]) + phi2 = np.square(phi_per_fold).mean() else: raise ValueError(f"Loss target {loss_target} not recognized.") @@ -403,12 +404,12 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe loss *= 10 trial_dict["loss"] = loss - if loss_target == "min_chi2_max_phi": - trial_dict["loss_phi"] = phi - trial_dict["loss_inverse_phi"] = np.reciprocal(phi) + if loss_target == "min_chi2_max_phi2": + trial_dict["loss_phi2"] = phi2 + trial_dict["loss_inverse_phi2"] = np.reciprocal(phi2) else: - trial_dict["loss_phi"] = pd.NA - trial_dict["loss_inverse_phi"] = pd.NA + trial_dict["loss_phi2"] = pd.NA + trial_dict["loss_inverse_phi2"] = pd.NA def generate_dictionary( @@ -585,7 +586,7 @@ def hyperopt_dataframe(commandline_args): best_idx = dataframe.loss.idxmin() best_models = pd.DataFrame() - if args.loss_target == "min_chi2_max_phi": + if args.loss_target == "min_chi2_max_phi2": minimum = dataframe.loss[best_idx] # set std to the spread of chi2 among the replicas of the best fit std = np.std(dataframe.hlosses[best_idx]) @@ -593,17 +594,17 @@ def hyperopt_dataframe(commandline_args): # select rows with chi2 losses within the best point and lim_max selected_chi2 = dataframe[(dataframe.loss >= minimum) & (dataframe.loss <= lim_max)] # among the selected points, select the nth lowest in 1/phi - selected_phi = selected_chi2.loss_inverse_phi.nsmallest(args.max_phi_n_models) + selected_phi2 = selected_chi2.loss_inverse_phi2.nsmallest(args.max_phi2_n_models) # find the location of these points in the dataframe - indices = dataframe[dataframe['loss_inverse_phi'].isin(selected_phi)].index + indices = dataframe[dataframe['loss_inverse_phi2'].isin(selected_phi2)].index # save the best models best_models = dataframe.loc[indices] # add to the best models the one with the lowest chi2 in case it is not included # if best_idx not in best_models.index: # best_models = pd.concat([best_models, dataframe.loc[[best_idx]]]) # set best trial to the model with the lowest 1/phi among the best chi2 models - best_inverse_phi_idx = best_models.loss_inverse_phi.idxmin() - best_trial = best_models.loc[best_inverse_phi_idx].to_frame().T + best_inverse_phi2_idx = best_models.loss_inverse_phi2.idxmin() + best_trial = best_models.loc[best_inverse_phi2_idx].to_frame().T else: best_trial_series = dataframe.loc[best_idx] # Make into a dataframe and transpose or the plotting code will complain @@ -638,7 +639,7 @@ def best_setup(hyperopt_dataframe, hyperscan_config, commandline_args): "initializer", "dropout", "loss", - "loss_inverse_phi", + "loss_inverse_phi2", ] ] best_trial.insert(12, "loss type", commandline_args["loss_target"]) @@ -647,11 +648,11 @@ def best_setup(hyperopt_dataframe, hyperscan_config, commandline_args): @figure -def plot_models_for_min_chi2_max_phi(hyperopt_dataframe, commandline_args): +def plot_models_for_min_chi2_max_phi2(hyperopt_dataframe, commandline_args): """ - Generates plot of the model index as a function of loss and 1/phi + Generates plot of the model index as a function of loss and 1/phi2 """ - if commandline_args["loss_target"] != 'min_chi2_max_phi': + if commandline_args["loss_target"] != 'min_chi2_max_phi2': fig, _ = plotutils.subplots() return fig @@ -662,7 +663,7 @@ def plot_models_for_min_chi2_max_phi(hyperopt_dataframe, commandline_args): def plot_models(dataframe, best_models): """ - Model plot called by `plot_models_for_min_chi2_max_phi`. + Model plot called by `plot_models_for_min_chi2_max_phi2`. """ # Reset the index of the dataframe dataframe = dataframe.reset_index(drop=True) @@ -673,7 +674,7 @@ def plot_models(dataframe, best_models): # define x and y x = dataframe.index y = dataframe['loss'] - z = dataframe['loss_inverse_phi'] + z = dataframe['loss_inverse_phi2'] indices = [] # Iterate over each value in best_models['iteration'] @@ -693,7 +694,7 @@ def plot_models(dataframe, best_models): print(f"Minimum chi2: x={x_min} y={y_min}") - print("Selected points from the min_chi2_max_phi:") + print("Selected points from the min_chi2_max_phi2:") print(y_best_chi2_worst_phi) # some color definitions @@ -722,7 +723,7 @@ def plot_models(dataframe, best_models): s=300, marker='o', alpha=0.5, - label=r'Min $\left<\chi^{2}\right>$ Max $\left<\varphi\right>$ models', + label=r'Min $\left<\chi^{2}\right>$ Max $\left<\varphi^{2}\right>$ models', ) # highlight area for all x and y up to std @@ -747,7 +748,7 @@ def plot_models(dataframe, best_models): ax2.plot( x, z, marker="s", label='', linewidth=0, markersize=5, color=color_phi, fillstyle='none' ) - ax2.set_ylabel(r'$1/\left<\varphi\right>$', color=color_phi) + ax2.set_ylabel(r'$1/\left<\varphi^{2}\right>$', color=color_phi) ax2.tick_params(axis='y', colors=color_phi) ax2.legend() ax2.grid(False) diff --git a/validphys2/src/validphys/hyperplottemplates/report.md b/validphys2/src/validphys/hyperplottemplates/report.md index 344a28009f..9d9db48d4d 100644 --- a/validphys2/src/validphys/hyperplottemplates/report.md +++ b/validphys2/src/validphys/hyperplottemplates/report.md @@ -4,7 +4,7 @@ {@ best_setup @} ## Models -{@ plot_models_for_min_chi2_max_phi @} +{@ plot_models_for_min_chi2_max_phi2 @} ## Optimizers {@ plot_optimizers @} diff --git a/validphys2/src/validphys/scripts/vp_hyperoptplot.py b/validphys2/src/validphys/scripts/vp_hyperoptplot.py index 25e5cedd9a..b55738e2e9 100644 --- a/validphys2/src/validphys/scripts/vp_hyperoptplot.py +++ b/validphys2/src/validphys/scripts/vp_hyperoptplot.py @@ -21,12 +21,12 @@ def add_positional_arguments(self, parser): "-l", "--loss_target", help="Choice for the definition of target loss", - choices=['average', 'best_worst', 'std', 'min_chi2_max_phi'], + choices=['average', 'best_worst', 'std', 'min_chi2_max_phi2'], default='average', ) parser.add_argument( - "--max_phi_n_models", - help="If --loss_target=best_chi2_worst_phi, outputs n models with the highest phi.", + "--max_phi2_n_models", + help="If --loss_target=min_chi2_max_phi2, outputs n models with the highest phi2.", type=int, default=1, ) @@ -129,7 +129,7 @@ def complete_mapping(self): "autofilter": args["autofilter"], "debug": args["debug"], "loss_target": args["loss_target"], - "max_phi_n_models": args["max_phi_n_models"], + "max_phi2_n_models": args["max_phi2_n_models"], } try: From e2b77a76118f6c4622e06c22ab68cbc39895f97d Mon Sep 17 00:00:00 2001 From: Radonirinaunimi Date: Wed, 2 Oct 2024 22:58:15 +0200 Subject: [PATCH 7/8] Various fixes to plot hyperopt scans --- validphys2/src/validphys/hyperoptplot.py | 165 ++++++++++++----------- 1 file changed, 83 insertions(+), 82 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index 8c88adf9ad..abfa5610e4 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -9,6 +9,7 @@ import glob +from hashlib import new import json import logging import os @@ -62,8 +63,8 @@ def convert_string_to_numpy(matrix_string: str) -> np.ndarray: class HyperoptTrial: - """ - Hyperopt trial class. + """Hyperopt trial class. + Makes the dictionary-like output of ``hyperopt`` into an object that can be easily managed @@ -231,11 +232,27 @@ def link_trials(self, list_of_trials): } +PRINTOUT_KEYS = [ + "optimizer", + "learning_rate", + "clipnorm", + "nodes_per_layer", + "activation_per_layer", + "initializer", + "diff_losses", + "loss", + "loss_inverse_phi2", +] + + +# Number of standard deviation to select models +N_STD = 1.0 + + # Parse the different sections of the hyperoptimization # this should talk with hyper_scan somehow? def parse_optimizer(trial): - """ - This function parses the parameters that affect the optimization + """This function parses the parameters that affect the optimization. optimizer learning_rate (if it exists) @@ -258,9 +275,16 @@ def parse_optimizer(trial): return dict_out -def parse_stopping(trial): +def get_std(df, threshold, n_std=1.0): + """Given a dataframe of models, select the ones that pass the threshold + losss and compute the `n_std` standard deviation from these. """ - This function parses the parameters that affect the stopping + valid_models = df.loc[df.loss <= threshold].loss.values + return n_std * np.std(valid_models) + + +def parse_stopping(trial): + """This function parses the parameters that affect the stopping. epochs stopping_patience @@ -283,8 +307,8 @@ def parse_stopping(trial): def parse_architecture(trial): - """ - This function parses the family of parameters which regards the architecture of the NN + """This function parses the family of parameters which regards the architecture + of the NN. number_of_layers activation_per_layer @@ -325,8 +349,7 @@ def parse_architecture(trial): def parse_statistics(trial): - """ - Parse the statistical information of the trial + """Parse the statistical information of the trial. validation loss testing loss @@ -355,9 +378,8 @@ def parse_statistics(trial): def parse_trial(trial): - """ - Trials are very convoluted object, very branched inside - The goal of this function is to separate said branching so we can create hierarchies + """Trials are very convoluted object, very branched inside. The goal of this + function is to separate said branching so we can create hierarchies. """ # Is this a true trial? if trial["state"] != 2: @@ -373,25 +395,11 @@ def parse_trial(trial): def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_target): + """Read a trial dictionary and compute the true loss and decide whether the + run passes or not. """ - Read a trial dictionary and compute the true loss and decide whether the run passes or not - """ - test_f = 1.0 - validation_multiplier val_loss = float(trial_dict[KEYWORDS["vl"]]) - if loss_target == "average": - test_loss = np.array(trial_dict["hlosses"]).mean() - elif loss_target == "best_worst": - test_loss = np.array(trial_dict["hlosses"]).max() - elif loss_target == "std": - test_loss = np.array(trial_dict["hlosses"]).std() - elif loss_target == "min_chi2_max_phi2": - test_loss = np.array(trial_dict["hlosses"]).mean() - phi_per_fold = np.array(trial_dict["hlosses_phi"]) - phi2 = np.square(phi_per_fold).mean() - else: - raise ValueError(f"Loss target {loss_target} not recognized.") - - loss = val_loss * validation_multiplier + test_loss * test_f + test_loss = loss = trial_dict["loss"] if ( loss > fail_threshold @@ -399,12 +407,16 @@ def evaluate_trial(trial_dict, validation_multiplier, fail_threshold, loss_targe or test_loss > fail_threshold or np.isnan(loss) ): - trial_dict["good"] = False - # Set the loss an order of magnitude above the result so it shows obviously on the plots - loss *= 10 + # trial_dict["good"] = False + # # Set the loss an order of magnitude above the result so it shows obviously on the plots + # loss *= 10 + pass trial_dict["loss"] = loss + trial_dict["diff_losses"] = abs(loss - val_loss) if loss_target == "min_chi2_max_phi2": + phi_per_fold = np.array(trial_dict["hlosses_phi"]) + phi2 = np.square(phi_per_fold).mean() trial_dict["loss_phi2"] = phi2 trial_dict["loss_inverse_phi2"] = np.reciprocal(phi2) else: @@ -420,15 +432,15 @@ def generate_dictionary( val_multiplier=0.5, fail_threshold=10.0, ): - """ - Reads a json file and returns a list of dictionaries + """Reads a json file and returns a list of dictionaries. # Arguments: - `replica_path`: folder in which the tries.json file can be found - `starting_index`: if the trials are to be added to an already existing set, make sure the id has the correct index! - `val_multiplier`: validation multipler - - `fail_threhsold`: threshold for the loss to consider a configuration as a failure + - `fail_threhsold`: threshold for the loss to consider a configuration + as a failure """ filename = "{0}/{1}".format(replica_path, json_name) @@ -452,9 +464,8 @@ def generate_dictionary( def filter_by_string(filter_string): - """ - Receives a data_dict (a parsed trial) and a filter string, - returns True if the trial passes the filter + """Receives a data_dict (a parsed trial) and a filter string, + returns True if the trial passes the filter. filter string must have the format: keystring where can be any of !=, =, >, < @@ -581,16 +592,19 @@ def hyperopt_dataframe(commandline_args): dataframe = dataframe_raw else: dataframe = dataframe_raw[dataframe_raw["good"]] + dataframe = dataframe_raw + # dataframe = dataframe_raw[dataframe_raw['loss'] <= 10] # Now select the best one + print(dataframe[PRINTOUT_KEYS].sort_values(by=["diff_losses"], ascending=False)) best_idx = dataframe.loss.idxmin() best_models = pd.DataFrame() if args.loss_target == "min_chi2_max_phi2": minimum = dataframe.loss[best_idx] # set std to the spread of chi2 among the replicas of the best fit - std = np.std(dataframe.hlosses[best_idx]) - lim_max = dataframe.loss[best_idx] + std + std = get_std(dataframe, args.threshold, n_std=N_STD) + lim_max = dataframe.loss[best_idx] + N_STD * std # select rows with chi2 losses within the best point and lim_max selected_chi2 = dataframe[(dataframe.loss >= minimum) & (dataframe.loss <= lim_max)] # among the selected points, select the nth lowest in 1/phi @@ -599,28 +613,20 @@ def hyperopt_dataframe(commandline_args): indices = dataframe[dataframe['loss_inverse_phi2'].isin(selected_phi2)].index # save the best models best_models = dataframe.loc[indices] - # add to the best models the one with the lowest chi2 in case it is not included - # if best_idx not in best_models.index: - # best_models = pd.concat([best_models, dataframe.loc[[best_idx]]]) - # set best trial to the model with the lowest 1/phi among the best chi2 models best_inverse_phi2_idx = best_models.loss_inverse_phi2.idxmin() best_trial = best_models.loc[best_inverse_phi2_idx].to_frame().T else: best_trial_series = dataframe.loc[best_idx] - # Make into a dataframe and transpose or the plotting code will complain + # Make into a dataframe and transpose best_trial = best_trial_series.to_frame().T - log.info("Best setup:") - with pd.option_context("display.max_rows", None, "display.max_columns", None): - log.info(best_trial) - return dataframe, best_trial, best_models @table def best_setup(hyperopt_dataframe, hyperscan_config, commandline_args): - """ - Generates a clean table with information on the hyperparameter settings of the best setup. + """Generates a clean table with information on the hyperparameter settings + of the best setup. """ _, best_trial, _ = hyperopt_dataframe best_idx = best_trial.index[0] @@ -649,22 +655,18 @@ def best_setup(hyperopt_dataframe, hyperscan_config, commandline_args): @figure def plot_models_for_min_chi2_max_phi2(hyperopt_dataframe, commandline_args): - """ - Generates plot of the model index as a function of loss and 1/phi2 - """ + """Generates plot of the model index as a function of loss and 1/phi2.""" if commandline_args["loss_target"] != 'min_chi2_max_phi2': fig, _ = plotutils.subplots() return fig dataframe, _, best_models = hyperopt_dataframe - fig = plot_models(dataframe, best_models) + fig = plot_models(dataframe, best_models, commandline_args) return fig -def plot_models(dataframe, best_models): - """ - Model plot called by `plot_models_for_min_chi2_max_phi2`. - """ +def plot_models(dataframe, best_models, commandline_args): + """Model plot called by `plot_models_for_min_chi2_max_phi2`.""" # Reset the index of the dataframe dataframe = dataframe.reset_index(drop=True) # dataframe.sort_values(by=["loss"], inplace=True) @@ -692,61 +694,60 @@ def plot_models(dataframe, best_models): y_min = dataframe.loss[min_idx] x_min = x[min_idx] + # Extract the number of layers + n_layers = [len(i) - 1 for i in best_models["nodes_per_layer"]] + best_models["nb_layers"] = n_layers print(f"Minimum chi2: x={x_min} y={y_min}") - print("Selected points from the min_chi2_max_phi2:") - print(y_best_chi2_worst_phi) + print(best_models[["loss", "loss_inverse_phi2", "nb_layers"]]) + print(len(best_models)) # some color definitions - color_chi2 = 'red' - color_phi = 'blue' - color_min = 'gray' + color_phi = "C0" + color_min = "C2" + color_chi2 = "C1" - ax.plot(x, y, marker="o", label='', linewidth=0, markersize=10, color=color_chi2) + ax.plot(x, y, marker="o", label='', linewidth=0, markersize=8, color=color_chi2) # highlight the min chi2 point ax.scatter( - x_min, - y_min, - color=color_min, - s=300, - marker='o', - alpha=0.5, - label=r'Min $\left<\chi^{2}\right>$', + x_min, y_min, color=color_min, s=200, marker='o', alpha=0.5, label=r'Lowest $L(\theta)$' ) # highlight all selected points from the best_chi2_worst_phi algorithm ax.scatter( x_best_chi2_worst_phi, y_best_chi2_worst_phi, - color='cyan', - s=300, + color=color_phi, + s=200, marker='o', alpha=0.5, - label=r'Min $\left<\chi^{2}\right>$ Max $\left<\varphi^{2}\right>$ models', + label=r'10 Selected Models', ) # highlight area for all x and y up to std + nstd = get_std(dataframe, commandline_args["threshold"], n_std=N_STD) ax.axhspan( y_min, - y_min + np.std(dataframe.hlosses[min_idx]), + y_min + nstd, color='yellow', - alpha=0.3, - label=r'Min $\left<\chi^{2}\right>$ + $\sigma$ band', + alpha=0.25, + label=r'Lowest $L(\theta)$ + 1$\sigma$ band', ) + ax.set_ylim(top=6) ax.set_xlabel('trial') ax.set_ylabel(r'Loss', color=color_chi2) ax.tick_params(axis='y', colors=color_chi2) ax.set_title('') # add a legend with a solid white background - ax.legend(facecolor='white', framealpha=1, shadow=True) + ax.legend() ax.grid(True) - # second plot phi + # PLOTTING OF PHI2 ax2 = ax.twinx() ax2.plot( - x, z, marker="s", label='', linewidth=0, markersize=5, color=color_phi, fillstyle='none' + x, z, marker="s", label='', linewidth=0, markersize=6, color=color_phi, fillstyle='none' ) ax2.set_ylabel(r'$1/\left<\varphi^{2}\right>$', color=color_phi) ax2.tick_params(axis='y', colors=color_phi) From a6409d451c8e7187f32de59b4cba7358b65a4e7c Mon Sep 17 00:00:00 2001 From: Radonirinaunimi Date: Mon, 7 Oct 2024 17:03:05 +0200 Subject: [PATCH 8/8] Further clean-ups --- validphys2/src/validphys/hyperoptplot.py | 31 ++++++++---------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/validphys2/src/validphys/hyperoptplot.py b/validphys2/src/validphys/hyperoptplot.py index abfa5610e4..86e2cb5fea 100644 --- a/validphys2/src/validphys/hyperoptplot.py +++ b/validphys2/src/validphys/hyperoptplot.py @@ -4,12 +4,11 @@ """ # Within this file you can find the "more modern" vp-integrated hyperopt stuff -# and the older pre-vp hyperopt stuff, which can be considered deprecated but it is -# still used for the plotting script +# and the older pre-vp hyperopt stuff, which can be considered deprecated but +# it is still used for the plotting script import glob -from hashlib import new import json import logging import os @@ -366,11 +365,6 @@ def parse_statistics(trial): dict_out[KEYWORDS["vl"]] = validation_loss dict_out[KEYWORDS["tl"]] = testing_loss - # Kfolding information - # average = results["kfold_meta"]["hyper_avg"] - # std = results["kfold_meta"]["hyper_std"] - # dict_out["avg"] = average - # dict_out["std"] = std dict_out["hlosses"] = results["kfold_meta"]["hyper_losses"] dict_out["vlosses"] = results["kfold_meta"]["validation_losses"] dict_out["hlosses_phi"] = results["kfold_meta"]["hyper_losses_phi"] @@ -593,10 +587,9 @@ def hyperopt_dataframe(commandline_args): else: dataframe = dataframe_raw[dataframe_raw["good"]] dataframe = dataframe_raw - # dataframe = dataframe_raw[dataframe_raw['loss'] <= 10] + dataframe.dropna(subset=["loss_inverse_phi2"], inplace=True) # Now select the best one - print(dataframe[PRINTOUT_KEYS].sort_values(by=["diff_losses"], ascending=False)) best_idx = dataframe.loss.idxmin() best_models = pd.DataFrame() @@ -607,7 +600,7 @@ def hyperopt_dataframe(commandline_args): lim_max = dataframe.loss[best_idx] + N_STD * std # select rows with chi2 losses within the best point and lim_max selected_chi2 = dataframe[(dataframe.loss >= minimum) & (dataframe.loss <= lim_max)] - # among the selected points, select the nth lowest in 1/phi + # among the selected points, select the nth lowest in 1/phi^2 selected_phi2 = selected_chi2.loss_inverse_phi2.nsmallest(args.max_phi2_n_models) # find the location of these points in the dataframe indices = dataframe[dataframe['loss_inverse_phi2'].isin(selected_phi2)].index @@ -694,13 +687,9 @@ def plot_models(dataframe, best_models, commandline_args): y_min = dataframe.loss[min_idx] x_min = x[min_idx] - # Extract the number of layers - n_layers = [len(i) - 1 for i in best_models["nodes_per_layer"]] - best_models["nb_layers"] = n_layers - print(f"Minimum chi2: x={x_min} y={y_min}") - print("Selected points from the min_chi2_max_phi2:") - print(best_models[["loss", "loss_inverse_phi2", "nb_layers"]]) - print(len(best_models)) + # Print out the selected models + print(f"Selected {commandline_args['max_phi2_n_models']} Models:") + print(best_models[PRINTOUT_KEYS]) # some color definitions color_phi = "C0" @@ -735,13 +724,13 @@ def plot_models(dataframe, best_models, commandline_args): label=r'Lowest $L(\theta)$ + 1$\sigma$ band', ) - ax.set_ylim(top=6) + ax.set_ylim(bottom=1.0, top=5.0) ax.set_xlabel('trial') ax.set_ylabel(r'Loss', color=color_chi2) ax.tick_params(axis='y', colors=color_chi2) ax.set_title('') # add a legend with a solid white background - ax.legend() + ax.legend(loc="upper left") ax.grid(True) # PLOTTING OF PHI2 @@ -751,7 +740,7 @@ def plot_models(dataframe, best_models, commandline_args): ) ax2.set_ylabel(r'$1/\left<\varphi^{2}\right>$', color=color_phi) ax2.tick_params(axis='y', colors=color_phi) - ax2.legend() + # ax2.legend() ax2.grid(False) ax2.spines['right'].set_visible(True)