From a0acf77ddd4d5e3328f812dd6b1ddc36b771bc65 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Thu, 17 Dec 2020 11:25:53 +0100 Subject: [PATCH 1/5] add preprocessing factors to json file --- n3fit/src/n3fit/io/writer.py | 2 +- n3fit/src/n3fit/performfit.py | 2 +- n3fit/src/n3fit/vpinterface.py | 43 +++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/n3fit/src/n3fit/io/writer.py b/n3fit/src/n3fit/io/writer.py index f9458be57a..5cedcd8011 100644 --- a/n3fit/src/n3fit/io/writer.py +++ b/n3fit/src/n3fit/io/writer.py @@ -119,7 +119,7 @@ def jsonfit(stopping_object, pdf_object, tr_chi2, vl_chi2, true_chi2, timing): """ all_info = {} # Generate preprocessing information - all_info["preprocessing"] = "" + all_info["preprocessing"] = pdf_object.get_preprocessing_factors() # .fitinfo-like info all_info["epoch_of_the_stop"] = stopping_object.epoch_of_the_stop all_info["best_epoch"] = stopping_object.e_best_chi2 diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index eb928d72e4..32f5318a40 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -317,7 +317,7 @@ def performfit( ) # Create a pdf instance - pdf_instance = N3PDF(pdf_model) + pdf_instance = N3PDF(pdf_model, fit_basis=fitting.get("basis")) # Generate the writer wrapper writer_wrapper = WriterWrapper( diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 36810dc0a9..ab37d4b545 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -44,8 +44,22 @@ class N3PDF(PDF): - def __init__(self, pdf_model, name="n3fit"): + """ + Creates a N3PDF object, extension of the validphys PDF object to perform calculation + with a n3fit generated model. + + Parameters + ---------- + pdf_model: n3fit.backend.MetaModel + PDF trained with n3fit, x -> f(x)_{i} where i are the flavours in the evol basis + fit_basis: list(dict) + basis of the training, used for reporting + name: str + name of the N3PDF object + """ + def __init__(self, pdf_model, fit_basis = None, name="n3fit"): self.model = pdf_model + self.fit_basis = fit_basis self.basis = check_basis("evolution", EVOL_LIST)["basis"] # Set the number of members to two for legacy compatibility # in this case replica 0 and replica 1 are the same @@ -64,6 +78,33 @@ def load(self): """ return self + def get_nn_weights(self): + """ Outputs all weights of the NN as numpy arrays """ + return self.model.get_weights() + + def get_preprocessing_factors(self): + """ Loads the preprocessing alpha and beta arrays from the PDF trained model. + If a ``fit_basis`` given in the format of `n3fit` runcards is given it will be used + to generate a new dictionary with the names, the exponent and whether they are trainable + otherwise outputs a Nx2 array where [:,0] are alphas and [:,1] betas + """ + factors = self.model.get_layer("pdf_prepro").get_weights() + all_fac = np.concatenate(factors) + alphas_and_betas = all_fac.reshape((-1,2)) + if self.fit_basis is not None and len(alphas_and_betas) == len(self.fit_basis): + output_dictionaries = [] + for ab, d in zip(alphas_and_betas, self.fit_basis): + output_dictionaries.append({ + "fl": d["fl"], + "smallx": float(ab[0]), + "largex": float(ab[1]), + "trainable": d.get("trainable", True) + }) + alphas_and_betas = output_dictionaries + return alphas_and_betas + + + def __call__(self, xarr, flavours=None): """Uses the internal model to produce pdf values. The output is on the evolution basis. From 710a85d1f2d08af671fdba70e6d37a73ba2eb0e1 Mon Sep 17 00:00:00 2001 From: Juacrumar Date: Tue, 12 Jan 2021 15:31:39 +0100 Subject: [PATCH 2/5] Apply suggestions from code review Co-authored-by: siranipour <43517072+siranipour@users.noreply.github.com> --- n3fit/src/n3fit/vpinterface.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index ab37d4b545..2404797c58 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -79,18 +79,18 @@ def load(self): return self def get_nn_weights(self): - """ Outputs all weights of the NN as numpy arrays """ + """Outputs all weights of the NN as numpy arrays """ return self.model.get_weights() def get_preprocessing_factors(self): - """ Loads the preprocessing alpha and beta arrays from the PDF trained model. - If a ``fit_basis`` given in the format of `n3fit` runcards is given it will be used + """Loads the preprocessing alpha and beta arrays from the PDF trained model. + If a ``fit_basis`` given in the format of ``n3fit`` runcards is given it will be used to generate a new dictionary with the names, the exponent and whether they are trainable otherwise outputs a Nx2 array where [:,0] are alphas and [:,1] betas """ factors = self.model.get_layer("pdf_prepro").get_weights() all_fac = np.concatenate(factors) - alphas_and_betas = all_fac.reshape((-1,2)) + alphas_and_betas = all_fac.reshape((-1, 2)) if self.fit_basis is not None and len(alphas_and_betas) == len(self.fit_basis): output_dictionaries = [] for ab, d in zip(alphas_and_betas, self.fit_basis): From 90a04560517f4019b3b2e939a302a3bd44c2e734 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Tue, 12 Jan 2021 16:43:28 +0100 Subject: [PATCH 3/5] review comments + get weight by name --- .../n3fit/backends/keras_backend/MetaLayer.py | 20 ++++++ n3fit/src/n3fit/vpinterface.py | 66 ++++++++++--------- 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/n3fit/src/n3fit/backends/keras_backend/MetaLayer.py b/n3fit/src/n3fit/backends/keras_backend/MetaLayer.py index 8e01fd43fa..47264b517c 100644 --- a/n3fit/src/n3fit/backends/keras_backend/MetaLayer.py +++ b/n3fit/src/n3fit/backends/keras_backend/MetaLayer.py @@ -61,6 +61,26 @@ def builder_helper( self.weight_inits.append((kernel, initializer)) return kernel + def get_weight_by_name(self, weight_name, internal_count=0): + """ + Returns a weight of the layer by name, returns None if the layer does not include + the named weight. + + Note that internally weights of a layer are prefaced by the name of the layer, this + should not be added to the input of this function. i.e., if the internal name is + "layer/weight:0", the argument to this method should be just "weight". + + Parameters + ---------- + weight_name: str + Name of the weight + """ + check_name = f"{self.name}/{weight_name}:{internal_count}" + for weight in self.weights: + if weight.name == check_name: + return weight + return None + # Implemented initializers @staticmethod def init_constant(value): diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 2404797c58..729fb6d55a 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -45,19 +45,20 @@ class N3PDF(PDF): """ - Creates a N3PDF object, extension of the validphys PDF object to perform calculation - with a n3fit generated model. - - Parameters - ---------- - pdf_model: n3fit.backend.MetaModel - PDF trained with n3fit, x -> f(x)_{i} where i are the flavours in the evol basis - fit_basis: list(dict) - basis of the training, used for reporting - name: str - name of the N3PDF object + Creates a N3PDF object, extension of the validphys PDF object to perform calculation + with a n3fit generated model. + + Parameters + ---------- + pdf_model: :py:class:`n3fit.backends.MetaModel` + PDF trained with n3fit, x -> f(x)_{i} where i are the flavours in the evol basis + fit_basis: list(dict) + basis of the training, used for reporting + name: str + name of the N3PDF object """ - def __init__(self, pdf_model, fit_basis = None, name="n3fit"): + + def __init__(self, pdf_model, fit_basis=None, name="n3fit"): self.model = pdf_model self.fit_basis = fit_basis self.basis = check_basis("evolution", EVOL_LIST)["basis"] @@ -79,7 +80,7 @@ def load(self): return self def get_nn_weights(self): - """Outputs all weights of the NN as numpy arrays """ + """Outputs all weights of the NN as numpy.ndarrays """ return self.model.get_weights() def get_preprocessing_factors(self): @@ -88,37 +89,38 @@ def get_preprocessing_factors(self): to generate a new dictionary with the names, the exponent and whether they are trainable otherwise outputs a Nx2 array where [:,0] are alphas and [:,1] betas """ - factors = self.model.get_layer("pdf_prepro").get_weights() - all_fac = np.concatenate(factors) - alphas_and_betas = all_fac.reshape((-1, 2)) - if self.fit_basis is not None and len(alphas_and_betas) == len(self.fit_basis): + preprocessing_layer = self.model.get_layer("pdf_prepro") + if self.fit_basis is not None: output_dictionaries = [] - for ab, d in zip(alphas_and_betas, self.fit_basis): - output_dictionaries.append({ - "fl": d["fl"], - "smallx": float(ab[0]), - "largex": float(ab[1]), - "trainable": d.get("trainable", True) - }) + for d in self.fit_basis: + flavour = d["fl"] + alpha = preprocessing_layer.get_weight_by_name(f"alpha_{flavour}") + beta = preprocessing_layer.get_weight_by_name(f"beta_{flavour}") + output_dictionaries.append( + { + "fl": flavour, + "smallx": float(alpha), + "largex": float(beta), + "trainable": d.get("trainable", True), + } + ) alphas_and_betas = output_dictionaries return alphas_and_betas - - def __call__(self, xarr, flavours=None): """Uses the internal model to produce pdf values. The output is on the evolution basis. Parameters ---------- - xarr: numpy.array + xarr: numpy.ndarray x-points with shape (xgrid_size,) (size-1 dimensions are removed) flavours: list list of flavours to output Returns ------- - numpy.array + numpy.ndarray (xgrid_size, flavours) pdf result """ if flavours is None: @@ -136,16 +138,16 @@ def grid_values(self, flavours, xarr, qmat=None): """ Parameters ---------- - flavours: numpy.array + flavours: numpy.ndarray flavours to compute - xarr: numpy.array + xarr: numpy.ndarray x-points to compute, dim: (xgrid_size,) - qmat: numpy.array + qmat: numpy.ndarray q-points to compute (not used by n3fit, used only for shaping purposes) Returns ------ - numpy.array + numpy.ndarray array of shape (1, flavours, xgrid_size, qmat) with the values of the ``pdf_model`` evaluated in ``xarr`` """ From 45233acbe8e4dccb0665809e35aad99e7ff18f82 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Tue, 12 Jan 2021 16:47:09 +0100 Subject: [PATCH 4/5] workaround for mismatchs --- n3fit/src/n3fit/vpinterface.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 729fb6d55a..9c5f827457 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -96,11 +96,15 @@ def get_preprocessing_factors(self): flavour = d["fl"] alpha = preprocessing_layer.get_weight_by_name(f"alpha_{flavour}") beta = preprocessing_layer.get_weight_by_name(f"beta_{flavour}") + if alpha is not None: + alpha = float(alpha) + if beta is not None: + beta = float(beta) output_dictionaries.append( { "fl": flavour, - "smallx": float(alpha), - "largex": float(beta), + "smallx": alpha, + "largex": beta, "trainable": d.get("trainable", True), } ) From a9e0eebf41dab1420de009273ee273ca1d8a6d61 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Wed, 13 Jan 2021 00:05:39 +0100 Subject: [PATCH 5/5] small fix --- n3fit/src/n3fit/vpinterface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/n3fit/src/n3fit/vpinterface.py b/n3fit/src/n3fit/vpinterface.py index 9c5f827457..dd5077e8c5 100644 --- a/n3fit/src/n3fit/vpinterface.py +++ b/n3fit/src/n3fit/vpinterface.py @@ -97,9 +97,9 @@ def get_preprocessing_factors(self): alpha = preprocessing_layer.get_weight_by_name(f"alpha_{flavour}") beta = preprocessing_layer.get_weight_by_name(f"beta_{flavour}") if alpha is not None: - alpha = float(alpha) + alpha = float(alpha.numpy()) if beta is not None: - beta = float(beta) + beta = float(beta.numpy()) output_dictionaries.append( { "fl": flavour,