From 63110d21bc02fc889258cedee63e24bdaa47138b Mon Sep 17 00:00:00 2001 From: Luis Fabregas <48292540+luisfabib@users.noreply.github.com> Date: Wed, 10 Aug 2022 09:41:38 -0700 Subject: [PATCH 01/10] freeze sphinx-gallery version to 0.9.0 to fix current docs build (#371) --- .github/workflows/deploy_ghpages.yml | 2 +- .github/workflows/docs_PR.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_ghpages.yml b/.github/workflows/deploy_ghpages.yml index 7568f771..7b89f15e 100644 --- a/.github/workflows/deploy_ghpages.yml +++ b/.github/workflows/deploy_ghpages.yml @@ -28,7 +28,7 @@ jobs: python -m pip install --upgrade pip python -m pip install pydata-sphinx-theme==0.7.1 python -m pip install numpydoc - python -m pip install sphinx-gallery + python -m pip install sphinx-gallery==0.9.0 python -m pip install sphinxcontrib-httpdomain python -m pip install sphinxcontrib-ghcontributors python -m pip install sphinx-issues diff --git a/.github/workflows/docs_PR.yml b/.github/workflows/docs_PR.yml index 11be219d..dbeaf7e6 100644 --- a/.github/workflows/docs_PR.yml +++ b/.github/workflows/docs_PR.yml @@ -32,7 +32,7 @@ jobs: python -m pip install --upgrade pip python -m pip install pydata-sphinx-theme==0.7.1 python -m pip install numpydoc - python -m pip install sphinx-gallery + python -m pip install sphinx-gallery==0.9.0 python -m pip install sphinxcontrib-httpdomain python -m pip install sphinxcontrib-ghcontributors python -m pip install sphinx-copybutton From 36be2fab967d3511e07ad120c00cff7bfd0efff3 Mon Sep 17 00:00:00 2001 From: Luis Fabregas Date: Fri, 9 Dec 2022 12:58:59 +0100 Subject: [PATCH 02/10] Correct behavior of masking during fitting (#394) --- deerlab/model.py | 6 ++- deerlab/solvers.py | 18 ++++---- deerlab/utils/__init__.py | 2 - deerlab/utils/utils.py | 96 +++++++++++++++++++++++++++++++++++---- test/test_model_class.py | 47 +++++++++++++++++++ test/test_snlls.py | 52 +++++++++++++++------ 6 files changed, 188 insertions(+), 33 deletions(-) diff --git a/deerlab/model.py b/deerlab/model.py index 8a0538ef..e0de3f21 100644 --- a/deerlab/model.py +++ b/deerlab/model.py @@ -1127,6 +1127,8 @@ def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl= Uncertainty quantification of the fitted model response. regparam : scalar Regularization parameter value used for the regularization of the linear parameters. + penweights : scalar or list thereof + Penalty weight value(s) used for the penalties specified through ``penalties``. plot : callable Function to display the results. It will display the fitted data. The function returns the figure object (``matplotlib.figure.Figure``) @@ -1143,13 +1145,13 @@ def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl= * ``stats['bic']`` - Bayesian information criterion cost : float Value of the cost function at the solution. - + noiselvl : ndarray + Estimated or user-given noise standard deviations of the individual datasets. evaluate(model, constants) : callable Function to evaluate a model at the fitted parameter values. Takes a model object or callable function ``model`` to be evaluated. All the parameters in the model or in the callable definition must match their corresponding parameter names in the ``FitResult`` object. Any model constants present required by the model must be specified as a second argument ``constants``. It returns the model's response at the fitted parameter values as an ndarray. - propagate(model,constants,lb,ub) : callable Function to propagate the uncertainty in the fit results to a model's response. Takes a model object or callable function ``model`` to be evaluated. All the parameters in the model or in the callable definition must match their corresponding parameter names in the ``FitResult`` object. diff --git a/deerlab/solvers.py b/deerlab/solvers.py index 719a8365..e1f11f50 100644 --- a/deerlab/solvers.py +++ b/deerlab/solvers.py @@ -249,7 +249,7 @@ def _model_evaluation(ymodels,parfit,paruq,uq): # =========================================================================================== # =========================================================================================== -def _goodness_of_fit_stats(ys,yfits,noiselvl,nParam): +def _goodness_of_fit_stats(ys,yfits,noiselvl,nParam,masks): """ Evaluation of goodness-of-fit statistics ======================================== @@ -258,9 +258,9 @@ def _goodness_of_fit_stats(ys,yfits,noiselvl,nParam): and returns a list of dictionaries with the statistics for each dataset. """ stats = [] - for y,yfit,sigma in zip(ys,yfits,noiselvl): - Ndof = len(y) - nParam - stats.append(goodness_of_fit(y, yfit, Ndof, sigma)) + for y,yfit,sigma,mask in zip(ys,yfits,noiselvl,masks): + Ndof = len(y[mask]) - nParam + stats.append(goodness_of_fit(y[mask], yfit[mask], Ndof, sigma)) return stats # =========================================================================================== @@ -533,7 +533,8 @@ def snlls(y, Amodel, par0=None, lb=None, ub=None, lbl=None, ubl=None, nnlsSolver Value of the cost function at the solution. residuals : ndarray Vector of residuals at the solution. - + noiselvl : ndarray + Estimated or user-given noise standard deviations. """ if verbose>0: @@ -542,7 +543,7 @@ def snlls(y, Amodel, par0=None, lb=None, ub=None, lbl=None, ubl=None, nnlsSolver # Ensure that all arrays are numpy.nparray par0 = np.atleast_1d(par0) - + # Parse multiple datsets and non-linear operators into a single concatenated vector/matrix y, Amodel, weights, mask, subsets, noiselvl = parse_multidatasets(y, Amodel, weights, noiselvl, masks=mask, subsets=subsets) @@ -904,6 +905,7 @@ def ymodel(n): # Make lists of data and fits ys = [y[subset] for subset in subsets] + masks = [mask[subset] for subset in subsets] if complexy: ys = [ys[n] + 1j*y[imagsubset] for n,imagsubset in enumerate(imagsubsets)] yfits = modelfit.copy() @@ -915,7 +917,7 @@ def ymodel(n): # Goodness-of-fit # --------------- Ndof = Nnonlin + Ndof_lin - stats = _goodness_of_fit_stats(ys,yfits,noiselvl,Ndof) + stats = _goodness_of_fit_stats(ys,yfits,noiselvl,Ndof,masks) # Display function plotfcn = partial(_plot,ys,yfits,yuq) @@ -927,7 +929,7 @@ def ymodel(n): return FitResult(nonlin=nonlinfit, lin=linfit, param=parfit, model=modelfit, nonlinUncert=paramuq_nonlin, linUncert=paramuq_lin, paramUncert=paramuq, modelUncert=modelfituq, regparam=alpha, plot=plotfcn, - stats=stats, cost=fvals, residuals=res) + stats=stats, cost=fvals, residuals=res, noiselvl=noiselvl) # =========================================================================================== diff --git a/deerlab/utils/__init__.py b/deerlab/utils/__init__.py index 260ef484..8fa398b3 100644 --- a/deerlab/utils/__init__.py +++ b/deerlab/utils/__init__.py @@ -1,4 +1,2 @@ # __init__.py from .utils import * -from .gof import goodness_of_fit - diff --git a/deerlab/utils/utils.py b/deerlab/utils/utils.py index 372b8575..5b562cf8 100644 --- a/deerlab/utils/utils.py +++ b/deerlab/utils/utils.py @@ -46,22 +46,24 @@ def parse_multidatasets(V_, K, weights, noiselvl, precondition=False, masks=None else: subset = subsets.copy() + # Parse the masks + if masks is None: + masks = [np.full_like(V,True).astype(bool) for V in Vlist] + elif not isinstance(masks,list): + masks = [masks] + if len(masks)!= len(Vlist): + raise SyntaxError('The number of masks does not match the number of signals.') + mask = np.concatenate(masks, axis=0) + # Noise level estimation/parsing sigmas = np.zeros(len(subset)) if noiselvl is None: for i in range(len(subset)): - sigmas[i] = der_snr(Vlist[i]) + sigmas[i] = der_snr(Vlist[i][masks[i]]) else: noiselvl = np.atleast_1d(noiselvl) sigmas = noiselvl.copy() - if masks is None: - masks = [np.full_like(V,True).astype(bool) for V in Vlist] - elif not isinstance(masks,list): - masks = [masks] - if len(masks)!= len(Vlist): - raise SyntaxError('The number of masks does not match the number of signals.') - mask = np.concatenate(masks, axis=0) # Concatenate the datasets along the list axis V = np.concatenate(Vlist, axis=0) @@ -157,6 +159,84 @@ def formatted_table(table,align=None): #=============================================================================== +#=============================================================================== +def goodness_of_fit(x,xfit,Ndof,noiselvl): + r""" + Goodness of Fit statistics + ========================== + + Computes multiple statistical indicators of goodness of fit. + + Usage: + ------ + stats = goodness_of_fit(x,xfit,Ndof) + + Arguments: + ---------- + x (N-element array) + Original data + xfit (N-element array) + Fit + Ndog (scalar, int) + Number of degrees of freedom + noiselvl (scalar) + Standard dexiation of the noise in x. + + Returns: + -------- + stats (dict) + Statistical indicators: + stats['chi2red'] - Reduced chi-squared + stats['rmsd'] - Root mean-squared dexiation + stats['R2'] - R-squared test + stats['aic'] - Akaike information criterion + stats['aicc'] - Corrected Akaike information criterion + stats['bic'] - Bayesian information criterion + + """ + sigma = noiselvl + Ndof = np.maximum(Ndof,1) + residuals = x - xfit + + # Special case: no noise, and perfect fit + if np.isclose(np.sum(residuals),0): + return {'chi2red':1,'R2':1,'rmsd':0,'aic':-np.inf,'aicc':-np.inf,'bic':-np.inf,'autocorr':0} + + # Get number of xariables + N = len(x) + # Extrapolate number of parameters + Q = Ndof - N + + # Reduced Chi-squared test + chi2red = 1/Ndof*np.linalg.norm(residuals)**2/sigma**2 + + # Autocorrelation based on Durbin–Watson statistic + autocorr_DW = abs( 2 - np.sum((residuals[1:-1] - residuals[0:-2])**2)/np.sum(residuals**2) ) + + # R-squared test + R2 = 1 - np.sum((residuals)**2)/np.sum((xfit-np.mean(xfit))**2) + + # Root-mean square dexiation + rmsd = np.sqrt(np.sum((residuals)**2)/N) + + # Log-likelihood + loglike = N*np.log(np.sum((residuals)**2)) + + # Akaike information criterion + aic = loglike + 2*Q + + # Corrected Akaike information criterion + aicc = loglike + 2*Q + 2*Q*(Q+1)/(N-Q-1) + + # Bayesian information criterion + bic = loglike + Q*np.log(N) + + return {'chi2red':chi2red,'R2':R2,'rmsd':rmsd,'aic':aic,'aicc':aicc,'bic':bic,'autocorr':autocorr_DW} +#=============================================================================== + + + + #=============================================================================== def der_snr(y): """ diff --git a/test/test_model_class.py b/test/test_model_class.py index d2fc9680..27359cda 100644 --- a/test/test_model_class.py +++ b/test/test_model_class.py @@ -1141,3 +1141,50 @@ def test_pickle_model(): os.remove("pickled_model.pkl") raise exception # ====================================================================== + + +# Construct mask +mask = np.ones_like(x).astype(bool) +mask[(x>3.8) & (x<4.2)] = False +# Distort data outside of mask +yref = mock_data_fcn(x) +ycorrupted = yref.copy() +ycorrupted[~mask] = 0 + +def test_fit_masking(): +#================================================================ + "Check that masking works" + model = _getmodel_axis('parametric') + + result = fit(model,ycorrupted,x) + resultmasked = fit(model,ycorrupted,x,mask=mask) + + assert np.allclose(resultmasked.model,yref) and not np.allclose(result.model,yref) +#================================================================ + +def test_masking_noiselvl(): +# ====================================================================== + "Check that masking leads to correct noise level estimates" + model = _getmodel_axis('parametric') + + noiselvl = 0.02 + yexp = ycorrupted + whitegaussnoise(x,noiselvl,seed=1) + + result = fit(model,yexp,x) + resultmasked = fit(model,yexp,x,mask=mask) + + assert resultmasked.noiselvl!=result.noiselvl and abs(resultmasked.noiselvl/noiselvl-1)<0.1 +# ====================================================================== + +def test_masking_chi2red(): +# ====================================================================== + "Check that masking is accounted for by the goodness-of-fit" + model = _getmodel_axis('parametric') + + yexp = ycorrupted + whitegaussnoise(x,0.02,seed=1) + + result = fit(model,yexp,x) + resultmasked = fit(model,yexp,x,mask=mask) + + assert resultmasked.stats['chi2red']2.5) & (x<3.5)] = False +# Distort data outside of mask +y = model([3, 0.5]) +yref = y.copy() +y[~mask] = 0 + def test_masking(): # ====================================================================== "Check that datapoints can be masked out" - x = np.linspace(0,7,100) - def model(p): - center, std = p - y = dd_gauss(x,center, std) - return y - - mask = np.ones_like(x).astype(bool) - mask[(x>2.5) & (x<3.5)] = False - - y = model([3, 0.5]) - yref = y.copy() - y[~mask] = 0 - fitmasked = snlls(y,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5], mask=mask) fit = snlls(y,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5]) assert np.allclose(fitmasked.model,yref) and not np.allclose(fit.model,yref) # ====================================================================== + +def test_masking_noiselvl(): +# ====================================================================== + "Check that masking leads to correct noise level estimates" + + noiselvl = 0.02 + yexp = y + whitegaussnoise(x,noiselvl,seed=1) + + fitmasked = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5], mask=mask) + fit = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5]) + + assert fitmasked.noiselvl!=fit.noiselvl and abs(fitmasked.noiselvl/noiselvl-1)<0.1 +# ====================================================================== + +def test_masking_chi2red(): +# ====================================================================== + "Check that masking is accounted for by the goodness-of-fit" + + yexp = y + whitegaussnoise(x,0.02,seed=1) + + fitmasked = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5], mask=mask) + fit = snlls(yexp,model,par0=[4,0.2],lb=[1,0.05],ub=[6,5]) + + assert fitmasked.stats['chi2red'] Date: Fri, 9 Dec 2022 13:00:03 +0100 Subject: [PATCH 03/10] Deprecate distribution through Anaconda (#400) - Remove all CI/CD files associated to the build and release process - Adjust the installation instructions - Adjust the CI/CD workflow to just run through PyPI --- .../conda_build_publish_package/Dockerfile | 11 ---- .../conda_build_publish_package/README.md | 63 ------------------- .../conda_build_publish_package/action.yml | 21 ------- .../conda_build_publish_package/entrypoint.sh | 50 --------------- .github/workflows/package_upload.yml | 27 +------- README.md | 7 +-- conda.recipe/build.bat | 2 - conda.recipe/build.sh | 1 - conda.recipe/meta.yaml | 36 ----------- conda.recipe/package_conda.bat | 44 ------------- docsrc/source/installation.rst | 22 +------ 11 files changed, 3 insertions(+), 281 deletions(-) delete mode 100644 .github/actions/conda_build_publish_package/Dockerfile delete mode 100644 .github/actions/conda_build_publish_package/README.md delete mode 100644 .github/actions/conda_build_publish_package/action.yml delete mode 100644 .github/actions/conda_build_publish_package/entrypoint.sh delete mode 100644 conda.recipe/build.bat delete mode 100644 conda.recipe/build.sh delete mode 100644 conda.recipe/meta.yaml delete mode 100644 conda.recipe/package_conda.bat diff --git a/.github/actions/conda_build_publish_package/Dockerfile b/.github/actions/conda_build_publish_package/Dockerfile deleted file mode 100644 index 1acbcfd8..00000000 --- a/.github/actions/conda_build_publish_package/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM continuumio/miniconda3:4.7.10 - -LABEL "repository"="https://github.com/m0nhawk/conda-package-publish-action" -LABEL "maintainer"="Andrew Prokhorenkov " - -RUN conda install -y anaconda-client conda-build - - -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/conda_build_publish_package/README.md b/.github/actions/conda_build_publish_package/README.md deleted file mode 100644 index da267334..00000000 --- a/.github/actions/conda_build_publish_package/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Build and Publish Anaconda Package - -A Github Action to publish your software package to an Anaconda repository. - -### Example workflow to publish to conda every time you make a new release - -```yaml -name: publish_conda - -on: - release: - types: [published] - -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: publish-to-conda - uses: maxibor/conda-package-publish-action@v1.1 - with: - subDir: 'conda' - AnacondaToken: ${{ secrets.ANACONDA_TOKEN }} - platforms: 'win-64 osx-64 linux-64' - python: '3.6 3.8' -``` - -### Example project structure - -``` -. -├── LICENSE -├── README.md -├── myproject -│   ├── __init__.py -│   └── myproject.py -├── conda -│   ├── build.sh -│   └── meta.yaml -├── .github -│   └── workflows -│   └── publish_conda.yml -├── .gitignore -``` -### Inputs - -The action takes the following - -- `AnacondaToken` - Anaconda access Token (see below) - -- `subDir` - (Optional) Sub-directory with conda recipe. Default: `.` - -- `platforms` - (Optional) Platforms to build and publish. Default: `win-64 osx-64 linux-64`. - -- `python` - (Optional) Python versions to build and publish. Default: `3.8`. - -### ANACONDA_TOKEN - -1. Get an Anaconda token (with read and write API access) at `anaconda.org/USERNAME/settings/access` -2. Add it to the Secrets of the Github repository as `ANACONDA_TOKEN` - -### Build Channels -By Default, this Github Action will search for conda build dependancies (on top of the standard channels) in `conda-forge` and `bioconda` diff --git a/.github/actions/conda_build_publish_package/action.yml b/.github/actions/conda_build_publish_package/action.yml deleted file mode 100644 index 877c90c5..00000000 --- a/.github/actions/conda_build_publish_package/action.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: 'Publish Conda package to Anaconda.org' -description: 'Build and Publish conda package to Anaconda' -author: 'Andrew Prokhorenkov, modified by Maxime Borry, modified by Luis Fabregas' -branding: - icon: 'package' - color: 'purple' -inputs: - subDir: - description: 'Sub-directory with conda recipe' - default: '.' - AnacondaToken: - description: 'Anaconda access Token' - platforms: - description: 'Platforms to build and publish [osx linux win]' - default: 'win-64 osx-64 linux-64' - python: - description: 'Python version to build and publish' - default: '3.8' -runs: - using: 'docker' - image: 'Dockerfile' diff --git a/.github/actions/conda_build_publish_package/entrypoint.sh b/.github/actions/conda_build_publish_package/entrypoint.sh deleted file mode 100644 index 309b2045..00000000 --- a/.github/actions/conda_build_publish_package/entrypoint.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -set -ex -set -o pipefail - -go_to_build_dir() { - if [ ! -z $INPUT_SUBDIR ]; then - cd $INPUT_SUBDIR - fi -} - -check_if_meta_yaml_file_exists() { - if [ ! -f meta.yaml ]; then - echo "meta.yaml must exist in the directory that is being packaged and published." - exit 1 - fi -} - -build_package(){ - - IFS=' ' read -ra PYTHON <<< "$INPUT_PYTHON" - IFS=' ' read -ra PLATFORMS <<< "$INPUT_PLATFORMS" - - for python in "${PYTHON[@]}"; do - conda build -c conda-forge -c bioconda --output-folder . --python $python . - done - for platform in "${PLATFORMS[@]}"; do - for filename in /$platform/*.tar.bz2; do - conda convert /$platform/$filename -p $platform linux-64/*.tar.bz2 -o . - done - done -} - -upload_package(){ - - IFS=' ' read -ra PLATFORMS <<< "$INPUT_PLATFORMS" - - export ANACONDA_API_TOKEN=$INPUT_ANACONDATOKEN - - for platform in "${PLATFORMS[@]}"; do - for filename in ./$platform/*.tar.bz2; do - anaconda upload $filename - done - done -} - -go_to_build_dir -check_if_meta_yaml_file_exists -build_package -upload_package diff --git a/.github/workflows/package_upload.yml b/.github/workflows/package_upload.yml index 6b74adec..69be3f25 100644 --- a/.github/workflows/package_upload.yml +++ b/.github/workflows/package_upload.yml @@ -1,4 +1,4 @@ -name: Build & Upload Python Package +name: Build & Upload DeerLab to PyPI on: release: @@ -26,28 +26,3 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - - conda-build-n-publish: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Get DeerLab version - run: echo "DEERLAB_VERSION=$(cat VERSION)" >> $GITHUB_ENV - - name: Update version in Conda metadata - uses: jacobtomlinson/gha-find-replace@master - with: - find: "VERSION" - replace: ${{env.DEERLAB_VERSION}} - include: "conda.recipe/meta.yaml" - - name: Build & Publish to Anaconda - uses: ./.github/actions/conda_build_publish_package - with: - subdir: 'conda.recipe' - anacondatoken: ${{ secrets.ANACONDA_TOKEN }} - platforms: 'osx-64 linux-32 linux-64 win-32 win-64' - python: '3.8 3.9 3.10' \ No newline at end of file diff --git a/README.md b/README.md index 39348c0f..ae4c1ec9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # DeerLab [![https://jeschkelab.github.io/DeerLab/](https://img.shields.io/pypi/v/deerlab)](https://pypi.org/project/DeerLab/) -[![https://img.shields.io/conda/v/JeschkeLab/deerlab](https://img.shields.io/conda/v/JeschkeLab/deerlab)](https://anaconda.org/jeschkelab/deerlab) [![Website](https://img.shields.io/website?down_message=offline&label=Documentation&up_message=online&url=https%3A%2F%2Fjeschkelab.github.io%2FDeerLab%2Findex.html)](https://jeschkelab.github.io/DeerLab/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/deerlab)](https://www.python.org/downloads/) ![PyPI - Downloads](https://img.shields.io/pypi/dm/deerlab?color=brightgreen) @@ -23,16 +22,12 @@ All additional dependencies are automatically downloaded and installed during th ## Setup -A pre-built distribution can be installed from the PyPI repository using `pip` or from the Anaconda repository using `conda`. +A pre-built distribution can be installed from the PyPI repository using `pip`. From a terminal (preferably with admin privileges) use the following command to install from PyPI: python -m pip install deerlab -or the following command to install from Anaconda: - - conda install deerlab -c JeschkeLab - More details on the installation and updating of DeerLab can be found [here](https://jeschkelab.github.io/DeerLab/installation.html). ## Citing DeerLab diff --git a/conda.recipe/build.bat b/conda.recipe/build.bat deleted file mode 100644 index 8930aeee..00000000 --- a/conda.recipe/build.bat +++ /dev/null @@ -1,2 +0,0 @@ -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 \ No newline at end of file diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh deleted file mode 100644 index fec5047c..00000000 --- a/conda.recipe/build.sh +++ /dev/null @@ -1 +0,0 @@ -$PYTHON setup.py install # Python command to install the script. \ No newline at end of file diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml deleted file mode 100644 index b8f26b3e..00000000 --- a/conda.recipe/meta.yaml +++ /dev/null @@ -1,36 +0,0 @@ -package: - name: deerlab - version: VERSION - -source: - git_rev: VERSION - git_url: https://github.com/JeschkeLab/DeerLab.git - -requirements: - build: - - python - - setuptools - - run: - - python - - memoization - - matplotlib - - tqdm - - joblib - - numpy - - scipy - - cvxopt - - dill - - pytest - -about: - home: https://jeschkelab.github.io/DeerLab/index.html - license: MIT - summary: 'Comprehensive package for data analysis of dipolar EPR spectroscopy' - description: | - DeerLab is a comprehensive free scientific software package for Python focused on modelling, - penalized least-squares regression, and uncertainty quantification. It provides highly - specialized on the analysis of dipolar EPR (electron paramagnetic resonance) spectroscopy data. - Dipolar EPR spectroscopy techniques include DEER (double electron-electron resonance), - RIDME (relaxation-induced dipolar modulation enhancement), and others. - dev_url: https://jeschkelab.github.io/DeerLab diff --git a/conda.recipe/package_conda.bat b/conda.recipe/package_conda.bat deleted file mode 100644 index 4ba2393c..00000000 --- a/conda.recipe/package_conda.bat +++ /dev/null @@ -1,44 +0,0 @@ -@echo off - -set versions=3.6 3.7 3.8 3.9 3.10 -set platforms=osx-64 linux-32 linux-64 win-32 win-64 - -:: Get the path to the Anaconda executable -for %%i in (_conda.exe) do ( - set anaconda=%%~p$PATH:i -) -set conda_build=C:%anaconda%conda-bld - -echo Activating Anaconda environment... -Powershell.exe -executionpolicy remotesigned C:%anaconda%shell\condabin\conda-hook.ps1 ; conda activate 'C:%anaconda%' - -echo Activating Anaconda client... -call anaconda login - -echo Building conda package... -:: Delete existing tarball files -for %%f in (%conda_build%\win-64\*.tar.bz2) do ( - del %%f -) - -:: Build the conda packages for the supported Python versions -for %%v in (%versions%) do ( - call conda-build --python %%v . -) - -:: Convert packages to all supported platforms -for %%f in (%conda_build%\win-64\*.tar.bz2) do ( - echo "Converting %%f" - for %%p in (%platforms%) do ( - call conda-convert --platform %%p %%f -o %conda_build% - ) -) - -:: Upload packages to Anaconda -for %%p in (%platforms%) do ( - for %%f in (%conda_build%\%%p\*.tar.bz2) do ( - call anaconda upload --user JeschkeLab %%f - ) -) - -echo "Building conda package finished!" \ No newline at end of file diff --git a/docsrc/source/installation.rst b/docsrc/source/installation.rst index f71e8556..d3294fc8 100644 --- a/docsrc/source/installation.rst +++ b/docsrc/source/installation.rst @@ -46,18 +46,6 @@ DeerLab installs the following packages: * `tqdm `_ - A lightweight package for smart progress meters. * `dill `_ - An extension of Python's pickle module for serializing and de-serializing python objects. -Installing from Anaconda -************************* - -DeerLab is also distributed via the Anaconda repository and the ``conda`` package manager. - -Open the Anaconda prompt (preferably with administrative privileges) or activate the Anaconda environment. Next install DeerLab via the ``conda`` package manager as follows:: - - conda install deerlab -c JeschkeLab - -The package manager will automatically take care of installing all DeerLab dependencies. - - Importing DeerLab ------------------ @@ -75,10 +63,6 @@ To upgrade an existing DeerLab installation to the latest released version, use python -m pip install --upgrade deerlab -or if you are using Anaconda use the following command from the Anaconda prompt:: - - conda update deerlab - Other installations ------------------- @@ -88,11 +72,7 @@ Installing specific versions Any DeerLab version released after v0.10.0 can be installed via ``pip`` using the following command matching the x.y.z to the desired version:: python -m pip install deerlab==x.y.z - -or via ``conda`` if you use Anaconda as follows:: - - conda install deerlab=x.y.z - + DeerLab version prior to 0.10 are written in MATLAB and are still available from an `archived repository `_. Download and installation instruction for the MATLAB environment are provided in the released documentation. MATLAB releases have been deprecated and no further support is provided. From 15821e2391bc6c4b120e5cf1a4982e5ef8164adf Mon Sep 17 00:00:00 2001 From: Luis Fabregas Date: Thu, 8 Dec 2022 16:05:14 +0100 Subject: [PATCH 04/10] examples: fix background plot error in 4-pulse DEER examples --- examples/basic/ex_bootstrapping.py | 6 ++---- examples/basic/ex_compactness_with_without.py | 4 ++-- examples/basic/ex_fitting_4pdeer.py | 4 ++-- examples/basic/ex_fitting_4pdeer_compactness.py | 4 ++-- examples/basic/ex_fitting_4pdeer_gauss.py | 4 ++-- examples/basic/ex_profileanalysis.py | 4 ++-- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/examples/basic/ex_bootstrapping.py b/examples/basic/ex_bootstrapping.py index b5d32f02..587dd184 100644 --- a/examples/basic/ex_bootstrapping.py +++ b/examples/basic/ex_bootstrapping.py @@ -66,8 +66,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) @@ -77,9 +77,7 @@ plt.plot(t,Vexp,'.',color='grey',label='Data') # Plot the fitted signal plt.plot(t,Vfit,linewidth=3,label='Bootstrap median',color=violet) -plt.fill_between(t,Vci[:,0],Vci[:,1],alpha=0.3,color=violet) plt.plot(t,Bfit,'--',linewidth=3,color=violet,label='Unmodulated contribution') -plt.fill_between(t,Bci[:,0],Bci[:,1],alpha=0.3,color=violet) plt.legend(frameon=False,loc='best') plt.xlabel('Time $t$ (μs)') plt.ylabel('$V(t)$ (arb.u.)') diff --git a/examples/basic/ex_compactness_with_without.py b/examples/basic/ex_compactness_with_without.py index 1dbf18c0..76ed47da 100644 --- a/examples/basic/ex_compactness_with_without.py +++ b/examples/basic/ex_compactness_with_without.py @@ -69,8 +69,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution - Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) - Bfit = Bfcn(results.mod,results.conc) + Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) + Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.subplot(2,2,n+1) diff --git a/examples/basic/ex_fitting_4pdeer.py b/examples/basic/ex_fitting_4pdeer.py index f6e358a6..8ca07775 100644 --- a/examples/basic/ex_fitting_4pdeer.py +++ b/examples/basic/ex_fitting_4pdeer.py @@ -55,8 +55,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) diff --git a/examples/basic/ex_fitting_4pdeer_compactness.py b/examples/basic/ex_fitting_4pdeer_compactness.py index fda85779..6ac43173 100644 --- a/examples/basic/ex_fitting_4pdeer_compactness.py +++ b/examples/basic/ex_fitting_4pdeer_compactness.py @@ -58,8 +58,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) diff --git a/examples/basic/ex_fitting_4pdeer_gauss.py b/examples/basic/ex_fitting_4pdeer_gauss.py index b2b48aa2..099e5933 100644 --- a/examples/basic/ex_fitting_4pdeer_gauss.py +++ b/examples/basic/ex_fitting_4pdeer_gauss.py @@ -59,8 +59,8 @@ Pci50 = Puncert.ci(50)/scale # Extract the unmodulated contribution -Bfcn = lambda mod,conc: scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[6,7]) diff --git a/examples/basic/ex_profileanalysis.py b/examples/basic/ex_profileanalysis.py index 6aaef435..f44113f0 100644 --- a/examples/basic/ex_profileanalysis.py +++ b/examples/basic/ex_profileanalysis.py @@ -55,8 +55,8 @@ Pci50 = results.PUncert.ci(50) # Extract the unmodulated contribution -Bfcn = lambda mod,conc: results.P_scale*(1-mod)*dl.bg_hom3d(t,conc,mod) -Bfit = Bfcn(results.mod,results.conc) +Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) plt.figure(figsize=[9,7]) From 7fb71cac38f488e718eac5b9562ac131c28611af Mon Sep 17 00:00:00 2001 From: Luis Fabregas <48292540+luisfabib@users.noreply.github.com> Date: Tue, 22 Nov 2022 18:25:04 +0100 Subject: [PATCH 05/10] bootstrap_analysis: fix error when analyzing scalar variables (#402) --- deerlab/bootstrap_analysis.py | 2 +- test/test_bootstrap_analysis.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deerlab/bootstrap_analysis.py b/deerlab/bootstrap_analysis.py index 368571a3..f3c277ed 100644 --- a/deerlab/bootstrap_analysis.py +++ b/deerlab/bootstrap_analysis.py @@ -129,7 +129,7 @@ def sample(): # Assert that all outputs are strictly numerical for var in varargout: - if not all(isnumeric(x) for x in var): + if not all(isnumeric(x) for x in np.atleast_1d(var)): raise ValueError('Non-numeric output arguments by the analyzed function are not accepted.') # Check that the full bootstrap analysis will not exceed the memory limits diff --git a/test/test_bootstrap_analysis.py b/test/test_bootstrap_analysis.py index e0e6e552..e356bb67 100644 --- a/test/test_bootstrap_analysis.py +++ b/test/test_bootstrap_analysis.py @@ -1,6 +1,6 @@ import numpy as np -from deerlab import dipolarkernel, whitegaussnoise, bootstrap_analysis, snlls +from deerlab import whitegaussnoise, bootstrap_analysis, snlls from deerlab.dd_models import dd_gauss from deerlab.utils import assert_docstring @@ -25,7 +25,7 @@ def fitfcn_global(ys): def fitfcn_multiout(yexp): fit = snlls(yexp,model,[1,3],uq=False) - return fit.nonlin*fit.lin, fit.model + return fit.nonlin*fit.lin, fit.model, fit.nonlin[0] def fitfcn_complex(yexp): fit = snlls(yexp,model,[1,3],uq=False) @@ -58,10 +58,10 @@ def test_multiple_ouputs(): # ====================================================================== "Check that both bootstrap handles the correct number outputs" - parfit,yfit = fitfcn_multiout(yexp) + parfit,yfit,_ = fitfcn_multiout(yexp) paruq = bootstrap_analysis(fitfcn_multiout,yexp,model(parfit),10) - assert len(paruq)==2 and all(abs(paruq[0].mean - parfit)) and all(abs(paruq[1].mean - yfit)) + assert len(paruq)==3 and all(abs(paruq[0].mean - parfit)) and all(abs(paruq[1].mean - yfit)) # ====================================================================== def test_multiple_datasets(): From 9bbf10567ce5463565283875cdffb07370bd2e36 Mon Sep 17 00:00:00 2001 From: Luis Fabregas <48292540+luisfabib@users.noreply.github.com> Date: Tue, 25 Oct 2022 18:36:15 +0200 Subject: [PATCH 06/10] fit: Expose `cores` option of bootstrap_analysis (#387) --- deerlab/model.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deerlab/model.py b/deerlab/model.py index e0de3f21..da5bca6f 100644 --- a/deerlab/model.py +++ b/deerlab/model.py @@ -1018,7 +1018,7 @@ def decorator(func): #============================================================================================== @insert_snlls_optionals_docstrings() def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl=None, mask=None, weights=None, - regparam='aic',reg='auto',regparamrange=None,**kwargs): + regparam='aic',reg='auto',regparamrange=None, bootcores=1,**kwargs): r""" Fit the model(s) to the dataset(s) @@ -1051,6 +1051,10 @@ def fit(model_, y, *constants, par0=None, penalties=None, bootstrap=0, noiselvl= bootstrap : scalar, optional, Bootstrap samples for uncertainty quantification. If ``bootstrap>0``, the uncertainty quantification will be performed via the boostrapping method with based on the number of samples specified as the argument. + + bootcores : scalar, optional + Number of CPU cores/processes for parallelization of the bootstrap uncertainty quantification. If ``cores=1`` no parallel + computing is used. If ``cores=-1`` all available CPUs are used. The default is one core (no parallelization). reg : boolean or string, optional Determines the use of regularization on the solution of the linear problem. @@ -1265,7 +1269,7 @@ def bootstrap_fcn(ysim): if not isinstance(fit.model,list): fit.model = [fit.model] return (fit.param,*fit.model) # Bootstrapped uncertainty quantification - param_uq = bootstrap_analysis(bootstrap_fcn,ysplit,fitresults.model,samples=bootstrap,noiselvl=noiselvl) + param_uq = bootstrap_analysis(bootstrap_fcn,ysplit,fitresults.model,samples=bootstrap,noiselvl=noiselvl,cores=bootcores) # Include information on the boundaries for better uncertainty estimates paramlb = model._vecsort(model._getvector('lb'))[np.concatenate(param_idx)] paramub = model._vecsort(model._getvector('ub'))[np.concatenate(param_idx)] From ed0d5e5c490d5fb1882b2e30a02d31fd4166ebd9 Mon Sep 17 00:00:00 2001 From: Luis Fabregas Date: Fri, 9 Dec 2022 13:16:10 +0100 Subject: [PATCH 07/10] examples: minor fix --- examples/basic/ex_fitting_4pdeer_gauss.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/basic/ex_fitting_4pdeer_gauss.py b/examples/basic/ex_fitting_4pdeer_gauss.py index 099e5933..106711d3 100644 --- a/examples/basic/ex_fitting_4pdeer_gauss.py +++ b/examples/basic/ex_fitting_4pdeer_gauss.py @@ -59,7 +59,7 @@ Pci50 = Puncert.ci(50)/scale # Extract the unmodulated contribution -Bfcn = lambda mod,conc,reftime: results.P_scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) +Bfcn = lambda mod,conc,reftime: scale*(1-mod)*dl.bg_hom3d(t-reftime,conc,mod) Bfit = results.evaluate(Bfcn) Bci = results.propagate(Bfcn).ci(95) From 3762645dc3b679d82c890e38954f9025a92964ab Mon Sep 17 00:00:00 2001 From: Luis Fabregas Date: Fri, 9 Dec 2022 13:16:24 +0100 Subject: [PATCH 08/10] update changelog --- docsrc/source/changelog.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docsrc/source/changelog.rst b/docsrc/source/changelog.rst index ad91d4ad..d91caafb 100644 --- a/docsrc/source/changelog.rst +++ b/docsrc/source/changelog.rst @@ -15,6 +15,20 @@ Release Notes - |api| : This will require changes in your scripts or code. +Release v0.14.5 - December 2022 +--------------------------------- + +- |fix| The distribution of DeerLab through Anaconda and its ``conda`` manager has been deprecated as of this release (:pr:`400`). +- |fix| Fix errors in the background function plots used in the examples showing 4-pulse DEER analyses. + +.. rubric:: ``fit`` +- |fix| Expose the ``cores`` option of ``bootstrap_analysis`` to parallelize bootstrap analysis from the ``fit`` function. +- |fix| Correct behavior of masking during fitting (:pr:`394`). When using the ``mask`` option of the ``fit`` function, certain steps such as noise estimation and goodness-of-fit assessment were not taking into account the mask during the analysis. + +.. rubric:: ``bootstrap_analysis`` +- |fix| Fix error prompted when analyzing scalar variables (:pr:`402`). + + Release v0.14.4 - August 2022 --------------------------------- From d0404168ab0b777a0a033c9b4a756b9c8eee370a Mon Sep 17 00:00:00 2001 From: Luis Fabregas Date: Fri, 9 Dec 2022 13:16:33 +0100 Subject: [PATCH 09/10] bump version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 6cf9fa53..574be600 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v0.14.4 +v0.14.5 From 0b26ff5b6347407c438f53aedec85df12954145f Mon Sep 17 00:00:00 2001 From: Luis Fabregas Date: Fri, 9 Dec 2022 13:18:34 +0100 Subject: [PATCH 10/10] changelog: minor edit --- docsrc/source/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docsrc/source/changelog.rst b/docsrc/source/changelog.rst index d91caafb..95298daa 100644 --- a/docsrc/source/changelog.rst +++ b/docsrc/source/changelog.rst @@ -22,7 +22,7 @@ Release v0.14.5 - December 2022 - |fix| Fix errors in the background function plots used in the examples showing 4-pulse DEER analyses. .. rubric:: ``fit`` -- |fix| Expose the ``cores`` option of ``bootstrap_analysis`` to parallelize bootstrap analysis from the ``fit`` function. +- |fix| Expose the ``cores`` option of ``bootstrap_analysis`` to parallelize bootstrap analysis from the ``fit`` function (:pr:`387`). - |fix| Correct behavior of masking during fitting (:pr:`394`). When using the ``mask`` option of the ``fit`` function, certain steps such as noise estimation and goodness-of-fit assessment were not taking into account the mask during the analysis. .. rubric:: ``bootstrap_analysis``