From 04c5ad8993b38ed8ea1a036c745009fe11ef46e9 Mon Sep 17 00:00:00 2001 From: Tanishq Sharma <99052355+t7phy@users.noreply.github.com> Date: Fri, 10 Mar 2023 22:16:54 +0100 Subject: [PATCH 1/9] Create commondata_utils.py --- validphys2/src/validphys/commondata_utils.py | 158 +++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 validphys2/src/validphys/commondata_utils.py diff --git a/validphys2/src/validphys/commondata_utils.py b/validphys2/src/validphys/commondata_utils.py new file mode 100644 index 0000000000..558afee918 --- /dev/null +++ b/validphys2/src/validphys/commondata_utils.py @@ -0,0 +1,158 @@ +""" +A useful set of functions for implementation of datasets. + +@author: Tanishq Sharma +""" + +import numpy as np + +from math import sqrt +from numpy.linalg import eig + +def symmetrize_errors(delta_plus, delta_minus): + r"""Compute the symmterized uncertainty and the shift in data point. + + Parameters + ---------- + delta_plus : float + The top/plus uncertainty with sign + delta_minus : float + The bottom/minus uncertainty with sign + + Returns + ------- + se_delta : float + The value to be added to the data point + se_sigma : float + The symmetrized uncertainty to be used in commondata + + """ + semi_diff = (delta_plus + delta_minus)/2 + average = (delta_plus - delta_minus)/2 + se_delta = semi_diff + se_sigma = sqrt(average*average + 2*semi_diff*semi_diff) + return se_delta, se_sigma + +def percentage_to_absolute(percentage, value): + r"""Compute the absolute value of uncertainty from percentage. + + Parameters + ---------- + percentage : string + Experimental datasets can provide the percentage + uncertainties with a % sign or without one. + In case a % sign is used, it will be automatically + stripped and the remaining string converted to a + float. If a % sign is not present, just input the + percentage uncertainty as a string, i.e. str(value). + This is important as this functions uses replace() + function which requires a string input. + value : float + The data point + + Returns + ------- + absolute : float + The absolute value of the uncertainty + + """ + percentage = float(percentage.replace("%", "")) + absolute = percentage * value * 0.01 + return absolute + +def corMat_to_covMat(ndata, errList, corMatList): + r"""Converts correlation matrix elements to covariance + matrix elements. + + Parameters + ---------- + ndata : integer + Number of data points + errList : list + A one dimensional list which contains the uncertainty + associated to each data point in order. + corMatList : list + A one dimensional list which contains the elements of + the correlation matrix row by row. Since experimental + datasets provide these matrices in a list form, this + simplifies the implementation for the user. + + Returns + ------- + covMatList : list + A one dimensional list which contains the elements of + the covariance matrix row by row. + + """ + covMatList = [] + for i in range(len(corMatList)): + a = i // ndata + b = i % ndata + covMatList.append(corMatList[i] * errList[a] * errList[b]) + return covMatList + +def covMat_to_artUnc(ndata, covMatList, is_normalized): + r"""Converts the covariance matrix to a matrix of + artificial uncertainties. + + Parameters + ---------- + ndata : integer + Number of data points + covMatList : list + A one dimensional list which contains the elements of + the covariance matrix row by row. Since experimental + datasets provide these matrices in a list form, this + simplifies the implementation for the user. + is_normalized : boolean + True if the dataset contains normalized values, False + if the dataset contains absolute values. Needed for + proper determination of whether the matrix is postive- + semidefinite. + + Returns + ------- + artUnc : numpy.ndarray + A two dimensional matrix which contains artificial + uncertainties to be added to the commondata. + i^th row contains the artificial uncertainties of + the i^th data point. + + """ + epsilon = -0.0000000001 + negEValCount = 0 + psdCheck = True + covMat = np.zeros((ndata, ndata)) + artUnc = np.zeros((ndata, ndata)) + for i in range(len(covMatList)): + a = i // ndata + b = i % ndata + covMat[a][b] = covMatList[i] + eigVal, eigVec = eig(covMat) + if is_normalized == False: + for i in range(ndata): + for j in range(ndata): + if eigVal[j] <= 0: + raise ValueError('The covariance matrix is not positive-semidefinite') + else: + artUnc[i][j] = eigVec[i][j] * sqrt(eigVal[j]) + elif is_normalized == True: + for j in range(len(eigVal)): + if eigVal[j] < epsilon: + psdCheck = False + elif eigVal[j] > epsilon and eigVal[j] <= 0: + negEValCount = negEValCount + 1 + if negEValCount == 2: + psdCheck = False + elif eigVal[j] > 0: + continue + if psdCheck == False: + raise ValueError('The covariance matrix is not positive-semidefinite') + else: + for i in range(ndata): + for j in range(ndata): + if eigVal[j] < 0: + continue + else: + artUnc[i][j] = eigVec[i][j] * sqrt(eigVal[j]) + return artUnc From ee2eb7b82867002a08eb49f0f7432760e62afa25 Mon Sep 17 00:00:00 2001 From: Tanishq Sharma <99052355+t7phy@users.noreply.github.com> Date: Fri, 10 Mar 2023 22:17:55 +0100 Subject: [PATCH 2/9] Update commondata_utils.py --- validphys2/src/validphys/commondata_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validphys2/src/validphys/commondata_utils.py b/validphys2/src/validphys/commondata_utils.py index 558afee918..62ec5d8284 100644 --- a/validphys2/src/validphys/commondata_utils.py +++ b/validphys2/src/validphys/commondata_utils.py @@ -1,5 +1,5 @@ """ -A useful set of functions for implementation of datasets. +A useful set of functions for implementation of datasets in commondata. @author: Tanishq Sharma """ From 77d5b4e604e27577df946d5bfff4f5a3900bd89a Mon Sep 17 00:00:00 2001 From: Tanishq Sharma <99052355+t7phy@users.noreply.github.com> Date: Thu, 23 Mar 2023 00:21:12 +0100 Subject: [PATCH 3/9] more util functions added --- validphys2/src/validphys/commondata_utils.py | 230 +++++++++++++++++-- 1 file changed, 216 insertions(+), 14 deletions(-) diff --git a/validphys2/src/validphys/commondata_utils.py b/validphys2/src/validphys/commondata_utils.py index 62ec5d8284..9e94e6a175 100644 --- a/validphys2/src/validphys/commondata_utils.py +++ b/validphys2/src/validphys/commondata_utils.py @@ -1,5 +1,15 @@ """ -A useful set of functions for implementation of datasets in commondata. +Python utilities for commondata implementation. + +This module provides helpful functions that automate a few +tasks that are regularly needed for the implementation of +experimental data to the commondata format. If there are +any additional functions that should be added here as they +could simply some repetitve tasks, please do suggest. + +Before the usage of any functions, it is recommended to read +the docstrings of the function to understand the inputs and +outputs. @author: Tanishq Sharma """ @@ -60,14 +70,12 @@ def percentage_to_absolute(percentage, value): absolute = percentage * value * 0.01 return absolute -def corMat_to_covMat(ndata, errList, corMatList): - r"""Converts correlation matrix elements to covariance +def corMat_to_covMat(errList, corMatList): + r"""Convert correlation matrix elements to covariance matrix elements. Parameters ---------- - ndata : integer - Number of data points errList : list A one dimensional list which contains the uncertainty associated to each data point in order. @@ -86,13 +94,13 @@ def corMat_to_covMat(ndata, errList, corMatList): """ covMatList = [] for i in range(len(corMatList)): - a = i // ndata - b = i % ndata + a = i // len(errList) + b = i % len(errList) covMatList.append(corMatList[i] * errList[a] * errList[b]) return covMatList def covMat_to_artUnc(ndata, covMatList, is_normalized): - r"""Converts the covariance matrix to a matrix of + r"""Convert the covariance matrix to a matrix of artificial uncertainties. Parameters @@ -112,11 +120,11 @@ def covMat_to_artUnc(ndata, covMatList, is_normalized): Returns ------- - artUnc : numpy.ndarray - A two dimensional matrix which contains artificial - uncertainties to be added to the commondata. - i^th row contains the artificial uncertainties of - the i^th data point. + artUnc : list + A two dimensional matrix (given as a list of lists) + which contains artificial uncertainties to be added + to the commondata. i^th row (or list) contains the + artificial uncertainties of the i^th data point. """ epsilon = -0.0000000001 @@ -155,4 +163,198 @@ def covMat_to_artUnc(ndata, covMatList, is_normalized): continue else: artUnc[i][j] = eigVec[i][j] * sqrt(eigVal[j]) - return artUnc + return artUnc.tolist() + +def cross_corMat_to_covMat(rowErrList, colErrList, corMatList): + r"""Convert cross correlation matrix elements + (i.e. those between different different variables or + observables) to covariance matrix elements. + + Parameters + ---------- + rowErrList : list + A one dimensional list which contains the uncertainty + associated to each data point of the variable that is + given on the vertical axis. + colErrList : list + A one dimensional list which contains the uncertainty + associated to each data point of the variable that is + given on the horizontal axis. + corMatList : list + A one dimensional list which contains the elements of + the correlation matrix row by row. Since experimental + datasets provide these matrices in a list form, this + simplifies the implementation for the user. + + Returns + ------- + covMatList : list + A one dimensional list which contains the elements of + the covariance matrix row by row. + + """ + covMatList = [] + for i in range(len(corMatList)): + a = i // len(colErrList) + b = i % len(colErrList) + covMatList.append(corMatList[i] * rowErrList[a] * colErrList[b]) + return covMatList + +def matList_to_matrix(rows, columns, matList): + r"""Convert a 1d list to a 2d matrix. + + Note: This utils function is not strictly needed for + data implementation, however, it is provided for + the aid of the user due to how matrices are treated + throughout all the other functions. This function + allows the user to convert a list that contains the + elemnets of matrix row by row to a proper matrix, if + need be for any reason. + + Parameters + ---------- + rows : int + No. of rows in the matrix + columns : int + No. of columns in the matrix + matList : list + A one dimensional list which contains the elements of + the matrix row by row. + + Returns + ------- + matrix : numpy.ndarray + The matrix as a numpy 2d array. + + """ + if rows * columns == len(matList): + matrix = np.zeros((rows, columns)) + for i in range(rows): + for j in range(columns): + matrix[i][j] = matList[j + i * columns] + matrix = np.array(matrix) + return matrix + else: + raise Exception('rows * columns != len(matList)') + +def concatMatrices(rows, columns, listOfMatrices): + r"""Join smaller matrices into a large matrix. + + This function aims to simplify the process of joining multiple + smaller matrices into one large matrix. Such a need could arise, + for instance, when cross variable covariance matrices are provided + but the user needs to join all the matrices to generate the full + covariance matrix corresponding to the entire dataset. + + Parameters + ---------- + rows : int + No. of rows of matrices to be concatenated. E.g., if 6 + matrices: A, B, C, D, E, F need to be joined as + [[A, B, C], + [D, E, F]], + the number of rows would be 2. + columns : int + No. of columns of matrices to be concatenated. In the + above example, this would be 3. + listOfMatrices : list + A list of the matrices that have to concatenated row by + row. In the above example, this would be [A, B, C, D, E, F]. + The matrices themselves need to be provided as a list of lists, + or a numpy 2d array. If the user has the matrix in a 1d row by + row form, use matList_to_matrix() to convert it. It is assumed + the user verifies that all the input matrices have the correct + dimensions. Matrices with incompatible dimensions will lead to + undesired behavior. + + Returns + ------- + finalMatList : list + A one dimensional list which contains the elements of + the final, fully concatenated matrix row by row. + + """ + for i in range(len(listOfMatrices)): + listOfMatrices[i] = np.array(listOfMatrices[i]) + colList = [] + for i in range(rows): + rowList = [] + for j in range(columns): + rowList.append(listOfMatrices[j + i * columns]) + colList.append(np.concatenate(tuple(rowList), axis=1)) + finalMat = np.concatenate(tuple(colList), axis=0) + finalMatList = [] + for i in range(len(finalMat)): + for j in range(len(finalMat[i])): + finalMatList.append(finalMat[i][j]) + return finalMatList + +def triMat_to_fullMat(mode, triMatList): + r"""Convert a list of values of a triangular matrix + to a symmetric matrix. + + Experimental datasets can provide the entries of + correlation or covariance matrices as a triangular + matrix, as these matrices are symmetric by their + very nature. This function can convert these list to + a complete symmetric matrix, that can be used for the + dataset implementation. + + mode : bool + Enter 0 or 1 based on the following scenarios: + Use mode 0 if matrix entries are given row by + row such as: + 0 1 2 3 + 4 5 6 + 7 8 + 9 + Use mode 1 if the matrix entries are given column + by column such as: + 0 1 3 6 + 2 4 7 + 5 8 + 9 + Please note that the numbers above (0-9) are not + entries of the matrix but rather the index of the + entries of the list which contains the elements of + the triangular matrix. + triMatList : list + A list containing the elements of the triangular matrix, + for example, for a 4*4 matrix, the list of + triangular matrix entries could be: + [a, b, c, d, e, f, g, h, i, j] + + Returns + ------- + matList : list + A one dimensional list which contains the elements of + the fully populated, symmetric matrix row by row. + + """ + dim = int((np.sqrt(1 + 8*len(triMatList)) - 1)/2) + matrix = np.zeros((dim, dim)) + if mode == 0: + for i in range(dim): + for j in range(i + 1): + listEl = len(triMatList) - 1 - ((i*(i + 1))//2 + j) + if i == j: + matrix[dim - 1 - i][dim - 1 - j] = triMatList[listEl] + else: + matrix[dim - 1 - i][dim - 1 - j] = triMatList[listEl] + matrix[dim - 1 - j][dim - 1 - i] = triMatList[listEl] + elif mode == 1: + for i in range(dim): + for j in range(i + 1): + listEl = (i*(i + 1))//2 + j + if i == j: + matrix[i][j] = triMatList[listEl] + else: + matrix[i][j] = triMatList[listEl] + matrix[j][i] = triMatList[listEl] + else: + raise Exception('Mode should be 0 or 1, refer to the function for usage') + matList = [] + for i in range(dim): + for j in range(dim): + matList.append(matrix[i][j]) + return matList From f434e67b0ab6f7cd7e083bc031f845d517cc0b59 Mon Sep 17 00:00:00 2001 From: Tanishq Sharma <99052355+t7phy@users.noreply.github.com> Date: Thu, 23 Mar 2023 00:25:49 +0100 Subject: [PATCH 4/9] docstring corrections --- validphys2/src/validphys/commondata_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validphys2/src/validphys/commondata_utils.py b/validphys2/src/validphys/commondata_utils.py index 9e94e6a175..19bcbe822a 100644 --- a/validphys2/src/validphys/commondata_utils.py +++ b/validphys2/src/validphys/commondata_utils.py @@ -4,8 +4,8 @@ This module provides helpful functions that automate a few tasks that are regularly needed for the implementation of experimental data to the commondata format. If there are -any additional functions that should be added here as they -could simply some repetitve tasks, please do suggest. +any additional functions that could be added here as they +could simplify some repetitve tasks, please do suggest. Before the usage of any functions, it is recommended to read the docstrings of the function to understand the inputs and From 3102e950d739b74d9545894e73eadae62ae9876b Mon Sep 17 00:00:00 2001 From: Tanishq Sharma <99052355+t7phy@users.noreply.github.com> Date: Fri, 24 Mar 2023 21:06:32 +0100 Subject: [PATCH 5/9] percentage_to_absolute made more user friendly --- validphys2/src/validphys/commondata_utils.py | 22 +++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/validphys2/src/validphys/commondata_utils.py b/validphys2/src/validphys/commondata_utils.py index 19bcbe822a..10e47b24af 100644 --- a/validphys2/src/validphys/commondata_utils.py +++ b/validphys2/src/validphys/commondata_utils.py @@ -48,15 +48,13 @@ def percentage_to_absolute(percentage, value): Parameters ---------- - percentage : string + percentage : string/float Experimental datasets can provide the percentage uncertainties with a % sign or without one. - In case a % sign is used, it will be automatically - stripped and the remaining string converted to a - float. If a % sign is not present, just input the - percentage uncertainty as a string, i.e. str(value). - This is important as this functions uses replace() - function which requires a string input. + The function will autostrip % sign and convert to + a float type in case the percentage uncertainty + comes with a % sign. Else, it will directly perform + the computation. value : float The data point @@ -66,9 +64,13 @@ def percentage_to_absolute(percentage, value): The absolute value of the uncertainty """ - percentage = float(percentage.replace("%", "")) - absolute = percentage * value * 0.01 - return absolute + if type(percentage) is str: + percentage = float(percentage.replace("%", "")) + absolute = percentage * value * 0.01 + return absolute + else: + absolute = percentage * value * 0.01 + return absolute def corMat_to_covMat(errList, corMatList): r"""Convert correlation matrix elements to covariance From 703ca1845221cd8ba593ffa635728154f36a6cf3 Mon Sep 17 00:00:00 2001 From: Tanishq Sharma <99052355+t7phy@users.noreply.github.com> Date: Wed, 29 Mar 2023 19:36:12 +0200 Subject: [PATCH 6/9] Formatting changes as per suggestions and more robust covmat_to_artunc --- validphys2/src/validphys/commondata_utils.py | 189 +++++++++---------- 1 file changed, 94 insertions(+), 95 deletions(-) diff --git a/validphys2/src/validphys/commondata_utils.py b/validphys2/src/validphys/commondata_utils.py index 10e47b24af..b0f990263e 100644 --- a/validphys2/src/validphys/commondata_utils.py +++ b/validphys2/src/validphys/commondata_utils.py @@ -72,16 +72,16 @@ def percentage_to_absolute(percentage, value): absolute = percentage * value * 0.01 return absolute -def corMat_to_covMat(errList, corMatList): +def cormat_to_covmat(err_list, cormat_list): r"""Convert correlation matrix elements to covariance matrix elements. Parameters ---------- - errList : list + err_list : list A one dimensional list which contains the uncertainty associated to each data point in order. - corMatList : list + cormat_list : list A one dimensional list which contains the elements of the correlation matrix row by row. Since experimental datasets provide these matrices in a list form, this @@ -89,19 +89,19 @@ def corMat_to_covMat(errList, corMatList): Returns ------- - covMatList : list + covmat_list : list A one dimensional list which contains the elements of the covariance matrix row by row. """ - covMatList = [] - for i in range(len(corMatList)): - a = i // len(errList) - b = i % len(errList) - covMatList.append(corMatList[i] * errList[a] * errList[b]) - return covMatList + covmat_list = [] + for i in range(len(cormat_list)): + a = i // len(err_list) + b = i % len(err_list) + covmat_list.append(cormat_list[i] * err_list[a] * err_list[b]) + return covmat_list -def covMat_to_artUnc(ndata, covMatList, is_normalized): +def covmat_to_artunc(ndata, covmat_list, no_of_norm_mat=0): r"""Convert the covariance matrix to a matrix of artificial uncertainties. @@ -109,20 +109,27 @@ def covMat_to_artUnc(ndata, covMatList, is_normalized): ---------- ndata : integer Number of data points - covMatList : list + covmat_list : list A one dimensional list which contains the elements of the covariance matrix row by row. Since experimental datasets provide these matrices in a list form, this simplifies the implementation for the user. - is_normalized : boolean - True if the dataset contains normalized values, False - if the dataset contains absolute values. Needed for - proper determination of whether the matrix is postive- - semidefinite. + no_of_norm_mat : int + Normalized covariance matrices may have an eigenvalue + of 0 due to the last data point not being linearly + independent. To allow for this, the user should input + the number of normalized matrices that are being treated + in an instance. For example, if a single covariance matrix + of a normalized distribution is being processed, the input + would be 1. If a covariance matrix contains pertains to + 3 normalized datasets (i.e. cross covmat for 3 + distributions), the input would be 3. The default value is + 0 for when the covariance matrix pertains to an absolute + distribution. Returns ------- - artUnc : list + artunc : list A two dimensional matrix (given as a list of lists) which contains artificial uncertainties to be added to the commondata. i^th row (or list) contains the @@ -130,59 +137,51 @@ def covMat_to_artUnc(ndata, covMatList, is_normalized): """ epsilon = -0.0000000001 - negEValCount = 0 - psdCheck = True - covMat = np.zeros((ndata, ndata)) - artUnc = np.zeros((ndata, ndata)) - for i in range(len(covMatList)): + neg_eval_count = 0 + psd_check = True + covmat = np.zeros((ndata, ndata)) + artunc = np.zeros((ndata, ndata)) + for i in range(len(covmat_list)): a = i // ndata b = i % ndata - covMat[a][b] = covMatList[i] - eigVal, eigVec = eig(covMat) - if is_normalized == False: + covmat[a][b] = covmat_list[i] + eigval, eigvec = eig(covmat) + for j in range(len(eigval)): + if eigval[j] < epsilon: + psd_check = False + elif eigval[j] > epsilon and eigval[j] <= 0: + neg_eval_count = neg_eval_count + 1 + if neg_eval_count == (no_of_norm_mat + 1): + psd_check = False + elif eigval[j] > 0: + continue + if psd_check == False: + raise ValueError('The covariance matrix is not positive-semidefinite') + else: for i in range(ndata): for j in range(ndata): - if eigVal[j] <= 0: - raise ValueError('The covariance matrix is not positive-semidefinite') + if eigval[j] < 0: + continue else: - artUnc[i][j] = eigVec[i][j] * sqrt(eigVal[j]) - elif is_normalized == True: - for j in range(len(eigVal)): - if eigVal[j] < epsilon: - psdCheck = False - elif eigVal[j] > epsilon and eigVal[j] <= 0: - negEValCount = negEValCount + 1 - if negEValCount == 2: - psdCheck = False - elif eigVal[j] > 0: - continue - if psdCheck == False: - raise ValueError('The covariance matrix is not positive-semidefinite') - else: - for i in range(ndata): - for j in range(ndata): - if eigVal[j] < 0: - continue - else: - artUnc[i][j] = eigVec[i][j] * sqrt(eigVal[j]) - return artUnc.tolist() + artunc[i][j] = eigvec[i][j] * sqrt(eigval[j]) + return artunc.tolist() -def cross_corMat_to_covMat(rowErrList, colErrList, corMatList): +def cross_cormat_to_covmat(row_err_list, col_err_list, cormat_list): r"""Convert cross correlation matrix elements (i.e. those between different different variables or observables) to covariance matrix elements. Parameters ---------- - rowErrList : list + row_err_list : list A one dimensional list which contains the uncertainty associated to each data point of the variable that is given on the vertical axis. - colErrList : list + col_err_list : list A one dimensional list which contains the uncertainty associated to each data point of the variable that is given on the horizontal axis. - corMatList : list + cormat_list : list A one dimensional list which contains the elements of the correlation matrix row by row. Since experimental datasets provide these matrices in a list form, this @@ -190,19 +189,19 @@ def cross_corMat_to_covMat(rowErrList, colErrList, corMatList): Returns ------- - covMatList : list + covmat_list : list A one dimensional list which contains the elements of the covariance matrix row by row. """ - covMatList = [] - for i in range(len(corMatList)): - a = i // len(colErrList) - b = i % len(colErrList) - covMatList.append(corMatList[i] * rowErrList[a] * colErrList[b]) - return covMatList + covmat_list = [] + for i in range(len(cormat_list)): + a = i // len(col_err_list) + b = i % len(col_err_list) + covmat_list.append(cormat_list[i] * row_err_list[a] * col_err_list[b]) + return covmat_list -def matList_to_matrix(rows, columns, matList): +def matlist_to_matrix(rows, columns, mat_list): r"""Convert a 1d list to a 2d matrix. Note: This utils function is not strictly needed for @@ -219,7 +218,7 @@ def matList_to_matrix(rows, columns, matList): No. of rows in the matrix columns : int No. of columns in the matrix - matList : list + mat_list : list A one dimensional list which contains the elements of the matrix row by row. @@ -229,17 +228,17 @@ def matList_to_matrix(rows, columns, matList): The matrix as a numpy 2d array. """ - if rows * columns == len(matList): + if rows * columns == len(mat_list): matrix = np.zeros((rows, columns)) for i in range(rows): for j in range(columns): - matrix[i][j] = matList[j + i * columns] + matrix[i][j] = mat_list[j + i * columns] matrix = np.array(matrix) return matrix else: - raise Exception('rows * columns != len(matList)') + raise Exception('rows * columns != len(mat_list)') -def concatMatrices(rows, columns, listOfMatrices): +def concat_matrices(rows, columns, list_of_matrices): r"""Join smaller matrices into a large matrix. This function aims to simplify the process of joining multiple @@ -259,7 +258,7 @@ def concatMatrices(rows, columns, listOfMatrices): columns : int No. of columns of matrices to be concatenated. In the above example, this would be 3. - listOfMatrices : list + list_of_matrices : list A list of the matrices that have to concatenated row by row. In the above example, this would be [A, B, C, D, E, F]. The matrices themselves need to be provided as a list of lists, @@ -271,27 +270,27 @@ def concatMatrices(rows, columns, listOfMatrices): Returns ------- - finalMatList : list + final_mat_list : list A one dimensional list which contains the elements of the final, fully concatenated matrix row by row. """ - for i in range(len(listOfMatrices)): - listOfMatrices[i] = np.array(listOfMatrices[i]) - colList = [] + for i in range(len(list_of_matrices)): + list_of_matrices[i] = np.array(list_of_matrices[i]) + col_list = [] for i in range(rows): - rowList = [] + row_list = [] for j in range(columns): - rowList.append(listOfMatrices[j + i * columns]) - colList.append(np.concatenate(tuple(rowList), axis=1)) - finalMat = np.concatenate(tuple(colList), axis=0) - finalMatList = [] - for i in range(len(finalMat)): - for j in range(len(finalMat[i])): - finalMatList.append(finalMat[i][j]) - return finalMatList + row_list.append(list_of_matrices[j + i * columns]) + col_list.append(np.concatenate(tuple(row_list), axis=1)) + final_mat = np.concatenate(tuple(col_list), axis=0) + final_mat_list = [] + for i in range(len(final_mat)): + for j in range(len(final_mat[i])): + final_mat_list.append(final_mat[i][j]) + return final_mat_list -def triMat_to_fullMat(mode, triMatList): +def trimat_to_fullmat(mode, tri_mat_list): r"""Convert a list of values of a triangular matrix to a symmetric matrix. @@ -320,7 +319,7 @@ def triMat_to_fullMat(mode, triMatList): entries of the matrix but rather the index of the entries of the list which contains the elements of the triangular matrix. - triMatList : list + tri_mat_list : list A list containing the elements of the triangular matrix, for example, for a 4*4 matrix, the list of triangular matrix entries could be: @@ -328,35 +327,35 @@ def triMat_to_fullMat(mode, triMatList): Returns ------- - matList : list + mat_list : list A one dimensional list which contains the elements of the fully populated, symmetric matrix row by row. """ - dim = int((np.sqrt(1 + 8*len(triMatList)) - 1)/2) + dim = int((np.sqrt(1 + 8*len(tri_mat_list)) - 1)/2) matrix = np.zeros((dim, dim)) if mode == 0: for i in range(dim): for j in range(i + 1): - listEl = len(triMatList) - 1 - ((i*(i + 1))//2 + j) + list_el = len(tri_mat_list) - 1 - ((i*(i + 1))//2 + j) if i == j: - matrix[dim - 1 - i][dim - 1 - j] = triMatList[listEl] + matrix[dim - 1 - i][dim - 1 - j] = tri_mat_list[list_el] else: - matrix[dim - 1 - i][dim - 1 - j] = triMatList[listEl] - matrix[dim - 1 - j][dim - 1 - i] = triMatList[listEl] + matrix[dim - 1 - i][dim - 1 - j] = tri_mat_list[list_el] + matrix[dim - 1 - j][dim - 1 - i] = tri_mat_list[list_el] elif mode == 1: for i in range(dim): for j in range(i + 1): - listEl = (i*(i + 1))//2 + j + list_el = (i*(i + 1))//2 + j if i == j: - matrix[i][j] = triMatList[listEl] + matrix[i][j] = tri_mat_list[list_el] else: - matrix[i][j] = triMatList[listEl] - matrix[j][i] = triMatList[listEl] + matrix[i][j] = tri_mat_list[list_el] + matrix[j][i] = tri_mat_list[list_el] else: raise Exception('Mode should be 0 or 1, refer to the function for usage') - matList = [] + mat_list = [] for i in range(dim): for j in range(dim): - matList.append(matrix[i][j]) - return matList + mat_list.append(matrix[i][j]) + return mat_list From 585d35aadfbdf2ce4bad0f10fc069e7f9b509ce8 Mon Sep 17 00:00:00 2001 From: t7phy Date: Wed, 5 Apr 2023 00:48:40 +0200 Subject: [PATCH 7/9] tests added --- .../validphys/tests/test_commondata_utils.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 validphys2/src/validphys/tests/test_commondata_utils.py diff --git a/validphys2/src/validphys/tests/test_commondata_utils.py b/validphys2/src/validphys/tests/test_commondata_utils.py new file mode 100644 index 0000000000..80b85302d7 --- /dev/null +++ b/validphys2/src/validphys/tests/test_commondata_utils.py @@ -0,0 +1,75 @@ +import pytest + +import numpy as np + +from validphys import commondata_utils as cdu + + +def test_symmetrize_errors(): + right = [1, 2, -1, -2] + left = [-2, -1, -1, 1] + delta = [-1/2, 1/2, -1, -1/2] + sigma = [1.65831239518, 1.65831239518, + 1.41421356237, 1.65831239518] + for i in range(4): + se_del, se_sig = cdu.symmetrize_errors(right[i], left[i]) + np.testing.assert_equal(delta[i], se_del) + np.testing.assert_almost_equal(sigma[i], se_sig) + +def test_percentage_to_absolute(): + perc = [5, '5%', -5, '-5%'] + value = 50 + res = [2.5, 2.5, -2.5, -2.5] + for i in range(4): + np.testing.assert_equal(cdu.percentage_to_absolute(perc[i], value), res[i]) + +def test_cormat_to_covmat(): + cormat = [1, 0.625, 0.25, 0.625, 1, 0.5, 0.25, 0.5, 1] + errlist = [4, 2, 2] + res = [16, 5, 2, 5, 4, 2, 2, 2, 4] + covmat = cdu.cormat_to_covmat(errlist, cormat) + np.testing.assert_allclose(res, covmat) + +def test_covmat_to_artunc_and_matlist_to_matrix(): + covmat_list = [16, 5, 2, 5, 4, 2, 2, 2, 4] + artunc = np.array(cdu.covmat_to_artunc(3, covmat_list)) + artuncT = artunc.transpose() + covmat = cdu.matlist_to_matrix(3, 3, covmat_list) + np.testing.assert_array_almost_equal(covmat, np.matmul(artunc, artuncT)) + +def test_cross_cormat_to_covmat(): + rowerrlist = [5, 4, 3, 5] + colerrlist = [1, 2, 2] + cormat = [0.2, 0.3, 0.4, 0.25, 0.35, 0.45, + 0.15, 0.2, 0.25, 0.6, 0.4, 0.2] + res = [1.0, 3.0, 4.0, 1.0, 2.8, 3.6, + 0.45, 1.2, 1.5, 3.0, 4.0, 2.0] + covmat = cdu.cross_cormat_to_covmat(rowerrlist, colerrlist, cormat) + np.testing.assert_allclose(res, covmat, atol=10e-10) + +def test_concat_matrices(): + arrA = np.array([[1, 2], + [3, 4]]) + arrB = np.array([[5, 6], + [7, 8]]) + arrC = np.array([[9, 10], + [11, 12]]) + arrD = np.array([[13, 14], + [15, 16]]) + arrE = np.array([[17, 18], + [19, 20]]) + arrF = np.array([[21, 22], + [23, 24]]) + res1 = [1, 2, 5, 6, 3, 4, 7, 8, 9, 10, 13, 14, 11, + 12, 15, 16, 17, 18, 21, 22, 19, 20, 23, 24] + res2 = [1, 2, 5, 6, 9, 10, 3, 4, 7, 8, 11, 12, 13, + 14, 17, 18, 21, 22, 15, 16, 19, 20, 23, 24] + np.testing.assert_allclose(res1, cdu.concat_matrices(3, 2, [arrA, arrB, arrC, arrD, arrE, arrF])) + np.testing.assert_allclose(res2, cdu.concat_matrices(2, 3, [arrA, arrB, arrC, arrD, arrE, arrF])) + +def test_trimat_to_fullmat(): + trimat = [1, 2, 3, 4, 5, 6] + mode0 = [1, 2, 3, 2, 4, 5, 3, 5, 6] + mode1 = [1, 2, 4, 2, 3, 5, 4, 5, 6] + np.testing.assert_allclose(mode0, cdu.trimat_to_fullmat(0, trimat)) + np.testing.assert_allclose(mode1, cdu.trimat_to_fullmat(1, trimat)) From df20a8d9a623632389222c36d35063eaa4634a73 Mon Sep 17 00:00:00 2001 From: t7phy Date: Wed, 5 Apr 2023 00:53:16 +0200 Subject: [PATCH 8/9] tests formatting --- validphys2/src/validphys/tests/test_commondata_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/validphys2/src/validphys/tests/test_commondata_utils.py b/validphys2/src/validphys/tests/test_commondata_utils.py index 80b85302d7..157a5f74ff 100644 --- a/validphys2/src/validphys/tests/test_commondata_utils.py +++ b/validphys2/src/validphys/tests/test_commondata_utils.py @@ -64,8 +64,10 @@ def test_concat_matrices(): 12, 15, 16, 17, 18, 21, 22, 19, 20, 23, 24] res2 = [1, 2, 5, 6, 9, 10, 3, 4, 7, 8, 11, 12, 13, 14, 17, 18, 21, 22, 15, 16, 19, 20, 23, 24] - np.testing.assert_allclose(res1, cdu.concat_matrices(3, 2, [arrA, arrB, arrC, arrD, arrE, arrF])) - np.testing.assert_allclose(res2, cdu.concat_matrices(2, 3, [arrA, arrB, arrC, arrD, arrE, arrF])) + np.testing.assert_allclose(res1, cdu.concat_matrices(3, 2, + [arrA, arrB, arrC, arrD, arrE, arrF])) + np.testing.assert_allclose(res2, cdu.concat_matrices(2, 3, + [arrA, arrB, arrC, arrD, arrE, arrF])) def test_trimat_to_fullmat(): trimat = [1, 2, 3, 4, 5, 6] From d8100966c3936ebe543c43c21a0512bddcc0885d Mon Sep 17 00:00:00 2001 From: t7phy Date: Tue, 12 Mar 2024 12:43:02 +0100 Subject: [PATCH 9/9] suggestions by RS --- .../new_commondata}/commondata_utils.py | 0 .../validphys/tests/test_commondata_utils.py | 77 ------------------- 2 files changed, 77 deletions(-) rename validphys2/src/validphys/{ => datafiles/new_commondata}/commondata_utils.py (100%) delete mode 100644 validphys2/src/validphys/tests/test_commondata_utils.py diff --git a/validphys2/src/validphys/commondata_utils.py b/validphys2/src/validphys/datafiles/new_commondata/commondata_utils.py similarity index 100% rename from validphys2/src/validphys/commondata_utils.py rename to validphys2/src/validphys/datafiles/new_commondata/commondata_utils.py diff --git a/validphys2/src/validphys/tests/test_commondata_utils.py b/validphys2/src/validphys/tests/test_commondata_utils.py deleted file mode 100644 index 157a5f74ff..0000000000 --- a/validphys2/src/validphys/tests/test_commondata_utils.py +++ /dev/null @@ -1,77 +0,0 @@ -import pytest - -import numpy as np - -from validphys import commondata_utils as cdu - - -def test_symmetrize_errors(): - right = [1, 2, -1, -2] - left = [-2, -1, -1, 1] - delta = [-1/2, 1/2, -1, -1/2] - sigma = [1.65831239518, 1.65831239518, - 1.41421356237, 1.65831239518] - for i in range(4): - se_del, se_sig = cdu.symmetrize_errors(right[i], left[i]) - np.testing.assert_equal(delta[i], se_del) - np.testing.assert_almost_equal(sigma[i], se_sig) - -def test_percentage_to_absolute(): - perc = [5, '5%', -5, '-5%'] - value = 50 - res = [2.5, 2.5, -2.5, -2.5] - for i in range(4): - np.testing.assert_equal(cdu.percentage_to_absolute(perc[i], value), res[i]) - -def test_cormat_to_covmat(): - cormat = [1, 0.625, 0.25, 0.625, 1, 0.5, 0.25, 0.5, 1] - errlist = [4, 2, 2] - res = [16, 5, 2, 5, 4, 2, 2, 2, 4] - covmat = cdu.cormat_to_covmat(errlist, cormat) - np.testing.assert_allclose(res, covmat) - -def test_covmat_to_artunc_and_matlist_to_matrix(): - covmat_list = [16, 5, 2, 5, 4, 2, 2, 2, 4] - artunc = np.array(cdu.covmat_to_artunc(3, covmat_list)) - artuncT = artunc.transpose() - covmat = cdu.matlist_to_matrix(3, 3, covmat_list) - np.testing.assert_array_almost_equal(covmat, np.matmul(artunc, artuncT)) - -def test_cross_cormat_to_covmat(): - rowerrlist = [5, 4, 3, 5] - colerrlist = [1, 2, 2] - cormat = [0.2, 0.3, 0.4, 0.25, 0.35, 0.45, - 0.15, 0.2, 0.25, 0.6, 0.4, 0.2] - res = [1.0, 3.0, 4.0, 1.0, 2.8, 3.6, - 0.45, 1.2, 1.5, 3.0, 4.0, 2.0] - covmat = cdu.cross_cormat_to_covmat(rowerrlist, colerrlist, cormat) - np.testing.assert_allclose(res, covmat, atol=10e-10) - -def test_concat_matrices(): - arrA = np.array([[1, 2], - [3, 4]]) - arrB = np.array([[5, 6], - [7, 8]]) - arrC = np.array([[9, 10], - [11, 12]]) - arrD = np.array([[13, 14], - [15, 16]]) - arrE = np.array([[17, 18], - [19, 20]]) - arrF = np.array([[21, 22], - [23, 24]]) - res1 = [1, 2, 5, 6, 3, 4, 7, 8, 9, 10, 13, 14, 11, - 12, 15, 16, 17, 18, 21, 22, 19, 20, 23, 24] - res2 = [1, 2, 5, 6, 9, 10, 3, 4, 7, 8, 11, 12, 13, - 14, 17, 18, 21, 22, 15, 16, 19, 20, 23, 24] - np.testing.assert_allclose(res1, cdu.concat_matrices(3, 2, - [arrA, arrB, arrC, arrD, arrE, arrF])) - np.testing.assert_allclose(res2, cdu.concat_matrices(2, 3, - [arrA, arrB, arrC, arrD, arrE, arrF])) - -def test_trimat_to_fullmat(): - trimat = [1, 2, 3, 4, 5, 6] - mode0 = [1, 2, 3, 2, 4, 5, 3, 5, 6] - mode1 = [1, 2, 4, 2, 3, 5, 4, 5, 6] - np.testing.assert_allclose(mode0, cdu.trimat_to_fullmat(0, trimat)) - np.testing.assert_allclose(mode1, cdu.trimat_to_fullmat(1, trimat))