diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a954409..187fb61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: - ubuntu-latest - macos-latest - windows-latest - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] fail-fast: true steps: diff --git a/.gitignore b/.gitignore index 91cb119..2b962bb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__/ dist/ build/ _build/ +poetry.lock +uv.lock # anything hidden should be manually added .* @@ -28,6 +30,7 @@ _build/ # notebooks *.ipynb +notebooks/data-examples/ # singularity containers *.sif diff --git a/README.md b/README.md index 0184a08..aa4d4f1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ An attempt to rewrite the project documentation is available as a [separate proj ## Installation -You will need Python >= 3.8. +You will need Python >= 3.9. ### Windows users diff --git a/containers/available_images.md b/containers/available_images.md index 624c006..bc160d5 100644 --- a/containers/available_images.md +++ b/containers/available_images.md @@ -1,196 +1,6 @@ -Available singularity images are: +Available Apptainer images are: -* tramway-hpc-220312-py3{6,7,[8](http://dl.pasteur.fr/fop/n2SSFwr5/tramway-hpc-220312-py38.sif),[9](http://dl.pasteur.fr/fop/W6GcBTqk/tramway-hpc-220312-py39.sif),[10](http://dl.pasteur.fr/fop/Cmmobhil/tramway-hpc-220312-py310.sif)}.sif - - *tramway0.6.4* image with targets *hpc-minimal* and *animate*, and *scikit-learn* as an additional package. - Python is available as *python3.x*, with *x* any of *6*, *7*, *8*, *9*, *10* depending on the container file. - The run command admits option *-s*, passed to Python, - resulting in command (e.g.) `python3.8 -s -m tramway $@`. - The included Python dependencies are the following: - -``` -cached-property==1.5.2 # <=py38 -certifi==2019.11.28 -chardet==3.0.4 -cvxopt==1.3.0 -cycler==0.11.0 -dbus-python==1.2.16 -decorator==4.4.2 # py36 only -fonttools==4.30.0 # >=py37 -h5py==3.1.0 # py36 only -h5py==3.6.0 # >=py37 -idna==2.8 -imageio==2.15.0 # py36 only -imageio==2.16.1 # >=py37 -importlib-resources==5.4.0 # py36 only -joblib==1.1.0 -kiwisolver==1.3.1 # py36 only -kiwisolver==1.3.2 # >=py37 -matplotlib==3.3.4 # py36 only -matplotlib==3.5.1 # >=py37 -networkx==2.5.1 # py36 only -networkx==2.6.3 # py37 only -networkx==2.7.1 # >=py38 -numpy==1.19.5 # py36 only -numpy==1.21.5 # py37 only -numpy==1.22.3 # >=py38 -opencv-python==4.5.5.64 -packaging==21.3 # >=py37 -pandas==1.1.5 # py36 only -pandas==1.3.5 # py37 only -pandas==1.4.1 # >=py38 -Pillow==8.4.0 # py36 only -Pillow==9.0.1 # >=py37 -polytope==0.2.3 -PyGObject==3.36.0 -pyparsing==3.0.7 -python-apt==2.0.0+ubuntu0.20.4.7 -python-dateutil==2.8.2 -pytz==2021.3 -PyWavelets==1.1.1 # py36 only -PyWavelets==1.3.0 # >=py37 -requests==2.22.0 -requests-unixsocket==0.2.0 -rwa-python==0.9.1 -scikit-image==0.17.2 # py36 only -scikit-image==0.19.2 # >=py37 -scikit-learn==0.24.2 # py36 only -scikit-learn==1.0.2 # >=py37 -scipy==1.5.4 # py36 only -scipy==1.7.3 # py37 only -scipy==1.8.0 # >=py38 -six==1.14.0 -stopit==1.1.2 -threadpoolctl==3.1.0 -tifffile==2020.9.3 # py36 only -tifffile==2021.11.2 # py37 only -tifffile==2022.2.9 # >=py38 -tqdm==4.63.0 -urllib3==1.25.8 -zipp==3.6.0 # py36 only -``` - -* tramway-hpc-230609-py3{8,9,[10](https://dl.pasteur.fr/fop/keDoKINW/tramway-hpc-230609-py310.sif),[11](https://dl.pasteur.fr/fop/znbYr11f/tramway-hpc-230609-py311.sif)}.sif - - *tramway0.6.7* image with targets *hpc-minimal* and *animate*, and *scikit-learn* as an additional package. - Python is available as *python3.x*, with *x* any of *8*, *9*, *10*, *11* depending on the container file. - The run command admits option *-s*, passed to Python, - resulting in command (e.g.) `python3.x -s -m tramway $@`. - The included Python dependencies are the following: - -``` -certifi==2019.11.28 -chardet==3.0.4 -contourpy==1.0.7 -cvxopt==1.3.1 -cycler==0.11.0 -dbus-python==1.2.16 -fonttools==4.40.0 -h5py==3.8.0 -idna==2.8 -imageio==2.31.1 -importlib-resources==5.12.0 # py<310 -joblib==1.2.0 -kiwisolver==1.4.4 -lazy_loader==0.2 -matplotlib==3.7.1 -networkx==3.1 -numpy==1.24.3 -opencv-python==4.7.0.72 -packaging==23.1 -pandas==2.0.2 -Pillow==9.5.0 -polytope==0.2.3 -PyGObject==3.36.0 -pyparsing==3.0.9 -python-apt==2.0.1+ubuntu0.20.4.1 -python-dateutil==2.8.2 -pytz==2023.3 -PyWavelets==1.4.1 -requests==2.22.0 -requests-unixsocket==0.2.0 -rwa-python==0.9.3 -scikit-image==0.20.0 -scikit-learn==1.2.2 -scipy==1.9.1 # py<310 -scipy==1.10.1 # py>39 -six==1.16.0 -stopit==1.1.2 -threadpoolctl==3.1.0 -tifffile==2023.4.12 -tqdm==4.65.0 -tzdata==2023.3 -urllib3==1.25.8 -zipp==3.15.0 # py<310 -``` - -* tramway-hpc-240423-py3{[8](https://dl.pasteur.fr/fop/B3LCHpTL/tramway-hpc-240423-py38.sif),[9](https://dl.pasteur.fr/fop/crbAgtuu/tramway-hpc-240423-py39.sif),[10](https://dl.pasteur.fr/fop/zZ2nrnKP/tramway-hpc-240423-py310.sif),[11](https://dl.pasteur.fr/fop/7F8PPD7j/tramway-hpc-240423-py311.sif),[12](https://dl.pasteur.fr/fop/lRWEB8oR/tramway-hpc-240423-py312.sif)}.sif - - *tramway0.6.8* image with targets *hpc-minimal* and *animate*, and *scikit-learn* as an additional package. - Python is available as *python3.x*, with *x* any of *8*, *9*, *10*, *11*, *12* depending on the container file. - The run command admits option *-s*, passed to Python, - resulting in command (e.g.) `python3.x -s -m tramway $@`. - The included Python dependencies are the following: - -``` -certifi==2019.11.28 -chardet==3.0.4 -contourpy==1.1.1 # py38 only -contourpy==1.2.1 # py>38 -cvxopt==1.3.2 -cycler==0.12.1 -dbus-python==1.2.16 -fonttools==4.51.0 -h5py==3.11.0 -idna==2.8 -imageio==2.34.1 -importlib-resources==6.4.0 # py<312 -joblib==1.4.0 -kiwisolver==1.4.5 -lazy_loader==0.4 -matplotlib==3.7.5 # py38 only -matplotlib==3.8.4 # py>38 -networkx==3.1 # py38 only -networkx==3.2.1 # py39 only -networkx==3.3 # py>39 -numpy==1.24.4 # py38 only -numpy==1.26.4 # py>38 -opencv-python==4.9.0.80 -packaging==24.0 -pandas==2.0.3 # py38 only -pandas==2.2.2 # py>38 -pillow==10.3.0 -polytope==0.2.5 -PyGObject==3.36.0 -pyparsing==3.1.2 -python-apt==2.0.1+ubuntu0.20.4.1 -python-dateutil==2.9.0.post0 -pytz==2024.1 -PyWavelets==1.4.1 -requests==2.22.0 -requests-unixsocket==0.2.0 -rwa-python==0.9.3 -scikit-image==0.21.0 # py38 only -scikit-image==0.22.0 # py39 only -scikit-image==0.23.2 # py>39 -scikit-learn==1.3.2 # py38 only -scikit-learn==1.4.2 # py>38 -scipy==1.10.1 # py38 only -scipy==1.13.0 # py>38 -setuptools==69.5.1 # py312 only -six==1.16.0 -stopit==1.1.2 -threadpoolctl==3.4.0 -tifffile==2023.7.10 # py38 only -tifffile==2024.4.18 # py>38 -tqdm==4.66.2 -tzdata==2024.1 -urllib3==1.25.8 -wheel==0.43.0 # py312 only -zipp==3.18.1 # py<310 only -``` - -* tramway-hpc-250124-py3{[8](https://dl.pasteur.fr/fop/xVjMGhTa/tramway-hpc-2501024-py38.sif),[9](https://dl.pasteur.fr/fop/AyK3JTC0/tramway-hpc-2501024-py39.sif),[10](https://dl.pasteur.fr/fop/jEmrwWhp/tramway-hpc-2501024-py310.sif),[11](https://dl.pasteur.fr/fop/QmL7TE6w/tramway-hpc-2501024-py311.sif),[12](https://dl.pasteur.fr/fop/elytCrRB/tramway-hpc-2501024-py312.sif),[13](https://dl.pasteur.fr/fop/hSeIMslv/tramway-hpc-2501024-py313.sif)}.sif +* tramway-hpc-250124-py3{[8](https://dl.pasteur.fr/fop/fnQY9aI4/tramway-hpc-250124-py38.sif),[9](https://dl.pasteur.fr/fop/bqIvcQOu/tramway-hpc-250124-py39.sif),[10](https://dl.pasteur.fr/fop/CGSBWArs/tramway-hpc-250124-py310.sif),[11](https://dl.pasteur.fr/fop/AFE8uuyA/tramway-hpc-250124-py311.sif),[12](https://dl.pasteur.fr/fop/4KuTx4gX/tramway-hpc-250124-py312.sif),[13](https://dl.pasteur.fr/fop/5XftLW2h/tramway-hpc-250124-py313.sif)}.sif *tramway0.6.9* image with targets *hpc-minimal* and *animate*, and *scikit-learn* as an additional package. Python is available as *python3.x*, with *x* any of *8*, *9*, *10*, *11*, *12*, *13* depending on the container file. @@ -266,3 +76,103 @@ zipp==3.20.2 # py == 3.8 zipp==3.21.0 # py == 3.9 ``` +* tramway-hpc-260217-py3{[9](https://dl.pasteur.fr/fop/vk2BlcFV/tramway-hpc-260217-py39.sif),[10](https://dl.pasteur.fr/fop/vt7BJqDk/tramway-hpc-260217-py310.sif),[11](https://dl.pasteur.fr/fop/yZNQ1Wn4/tramway-hpc-260217-py311.sif),[12](https://dl.pasteur.fr/fop/X4Pqd1Bk/tramway-hpc-260217-py312.sif),[13](https://dl.pasteur.fr/fop/FBEYEP0E/tramway-hpc-260217-py313.sif),[14](https://dl.pasteur.fr/fop/NbJmUfmQ/tramway-hpc-260217-py314.sif)}.sif + + *tramway0.7* image with targets *hpc-minimal* and *animate*, and *scikit-learn* as an additional package. + Python is available as *python3.x*, with *x* any of *9*, *10*, *11*, *12*, *13*, *14* depending on the container file. + The run command admits option *-s*, passed to Python, + resulting in command (e.g.) `python3.x -s -m tramway $@`. + The included Python dependencies are the following: + +``` +blinker==1.4 # py >= 3.10 +certifi==2019.11.28 # py == 3.9 +chardet==3.0.4 # py == 3.9 +contourpy==1.3.0 # py == 3.9 +contourpy==1.3.2 # py == 3.10 +contourpy==1.3.3 # py >= 3.11 +cryptography==3.4.8 # py >= 3.10 +cvxopt==1.3.3 +cycler==0.12.1 +dbus-python==1.2.16 # py == 3.9 +dbus-python==1.2.18 # py >= 3.10 +dill==0.4.1 +distro==1.7.0 # py >= 3.10 +exceptiongroup==1.3.1 # py == 3.9 +fonttools==4.60.2 # py == 3.9 +fonttools==4.61.1 # py >= 3.10 +h5py==3.14.0 # py == 3.9 +h5py==3.15.1 # py >= 3.10 +httplib2==0.20.2 # py >=3.10 +idna==2.8 # py == 3.9 +ImageIO==2.37.2 +importlib_resources==6.5.2 # py == 3.9 +importlib-metadata==4.6.4 # py >= 3.10 +iniconfig==2.1.0 # py == 3.9 +jeepney==0.7.1 # py >= 3.10 +joblib==1.5.3 +keyring==23.5.0 # py >= 3.10 +kiwisolver==1.4.7 # py == 3.9 +kiwisolver==1.4.9 # py >= 3.10 +launchpadlib==1.10.16 # py >= 3.10 +lazr.restfulclient==0.14.4 # py >= 3.10 +lazr.uri==1.0.6 # py >= 3.10 +lazy_loader==0.4 +matplotlib==3.9.4 # py == 3.9 +matplotlib==3.10.8 # py >= 3.10 +more-itertools==8.10.0 # py >= 3.10 +multiprocess==0.70.19 +networkx==3.2.1 # py == 3.9 +networkx==3.4.2 # py == 3.10 +networkx==3.6.1 # py >= 3.11 +numpy==2.0.2 # py == 3.9 +numpy==2.2.6 # py == 3.10 +numpy==2.4.2 # py >= 3.11 +oauthlib==3.2.0 # py >= 3.10 +opencv-python==4.13.0.92 +packaging==26.0 +pandas==2.3.3 +pillow==11.3.0 # py == 3.9 +pillow==12.1.1 # py >= 3.10 +pluggy==1.6.0 # py == 3.9 +polytope==0.2.5 +Pygments==2.19.2 # py == 3.9 +PyGObject==3.36.0 # py == 3.9 +PyGObject==3.42.1 # py >= 3.10 +PyJWT==2.3.0 # py >= 3.10 +pyparsing==3.3.2 +pytest==8.4.2 # py == 3.9 +python-apt==2.0.1+ubuntu0.20.4.1 # py == 3.9 +python-apt==2.4.0+ubuntu4.1 # py >= 3.10 +python-dateutil==2.9.0.post0 +pytz==2025.2 +requests==2.22.0 # py == 3.9 +requests-unixsocket==0.2.0 # py == 3.9 +rwa-python==0.10 +scikit-image==0.24.0 # py == 3.9 +scikit-image==0.25.2 # py == 3.10 +scikit-image==0.26.0 # py >= 3.11 +scikit-learn==1.6.1 # py == 3.9 +scikit-learn==1.7.2 # py == 3.10 +scikit-learn==1.8.0 # py >= 3.11 +scipy==1.13.1 # py == 3.9 +scipy==1.15.3 # py == 3.10 +scipy==1.17.0 # py >= 3.11 +SecretStorage==3.3.1 # py >= 3.10 +setuptools==82.0.0 # py >= 3.12 +six==1.17.0 # py == 3.9 +six==1.16.0 # py >= 3.10 +stopit==1.1.2 +threadpoolctl==3.6.0 +tifffile==2024.8.30 # py == 3.9 +tifffile==2025.5.10 # py == 3.10 +tifffile==2026.2.16 # py >= 3.11 +tomli==2.4.0 # py == 3.9 +tqdm==4.67.3 +typing_extensions==4.15.0 # py == 3.9 +tzdata==2025.3 +urllib3==1.25.8 # py == 3.9 +wadllib==1.3.6 # py >= 3.10 +zipp==3.23.0 # py == 3.9 +zipp==1.0.0 # py >= 3.10 +``` diff --git a/containers/container-pip-freeze.sh b/containers/container-pip-freeze.sh index df10e02..8c7770c 100755 --- a/containers/container-pip-freeze.sh +++ b/containers/container-pip-freeze.sh @@ -1,7 +1,7 @@ #!/bin/bash if [ -z $1 ]; then - echo "usage: $0 CONTAINER_FILE [PATH_TO_SINGULARITY]" + echo "usage: $0 CONTAINER_FILE [PATH_TO_APPTAINER]" >&2 exit 1 fi @@ -9,7 +9,7 @@ CONTAINER=$1 shift if [ -z $1 ]; then - SINGULARITY="singularity exec" + SINGULARITY="apptainer exec" else SINGULARITY="$1 exec" fi diff --git a/containers/detect_python.sh b/containers/detect_python.sh index 58b8757..34fb15b 100755 --- a/containers/detect_python.sh +++ b/containers/detect_python.sh @@ -1,6 +1,6 @@ #!/bin/bash -for ((minor=13;6<=minor;minor--)); do +for ((minor=14;6<=minor;minor--)); do py=python3.$minor if command -v $py &>/dev/null; then if $py -m pip freeze | grep tramway &>/dev/null; then diff --git a/containers/test-maestro.sh b/containers/test-maestro.sh new file mode 100755 index 0000000..75ccecc --- /dev/null +++ b/containers/test-maestro.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +## to get and run the present script, on Maestro: +# cd $MYSCRATCH +# rm -rf TRamWAy; git clone --branch dev https://github.com/DecBayComp/TRamWAy && cd $_ +# tests/test-maestro.sh + +set -e + +version=260217 +pyminor=9 +container="containers/tramway-hpc-$version-py3$pyminor.sif" +testdata=tests/test_analyzer_200803 +slurmopts="-p dbc -q dbc" + +[ "`pwd`" = "$MYSCRATCH/TRamWAy" ] || exit 1 + +# TODO make the data available online again +[ -d "$testdata" ] || cp -r ~/Projects/Tramway/TRamWAy/"$testdata" tests/ + +module load apptainer +rm -f $container; srun $slurmopts apptainer build $container "containers/tramway-hpc-py3$pyminor" + +mkdir -p containers/tmp +srun $slurmopts apptainer exec \ + --bind "$testdata:/root/TRamWAy/$testdata" \ + --cwd=/root/TRamWAy --overlay=containers/tmp $container \ + "python3.$pyminor" -m pytest --ignore=tests/test_analyzer_pipeline.py diff --git a/containers/tramway-hpc-py310 b/containers/tramway-hpc-py310 index d6c9d7f..7b91201 100644 --- a/containers/tramway-hpc-py310 +++ b/containers/tramway-hpc-py310 @@ -1,10 +1,10 @@ Bootstrap: docker -From: ubuntu:focal +From: ubuntu:22.04 %help TRamWAy is available in the python3.10 environment: python3.10 -m tramway -The container OS is Ubuntu Focal Fossa. +The container OS is Ubuntu 22.04. %setup @@ -24,7 +24,7 @@ The container OS is Ubuntu Focal Fossa. if [ -d $LOCAL ]; then CONTAINED=${SINGULARITY_ROOTFS}/root/TRamWAy mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/requirements.txt ${LOCAL}/README.md + cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/pyproject.toml ${LOCAL}/README.md cp -ru -t ${CONTAINED}/ ${LOCAL}/tramway ${LOCAL}/scripts fi @@ -34,12 +34,10 @@ The container OS is Ubuntu Focal Fossa. apt-get update -y apt-get install -y --no-install-recommends locales locale-gen - apt-get install -y --no-install-recommends libhdf5-103 ffmpeg \ + apt-get install -y --no-install-recommends libhdf5-openmpi-dev ffmpeg \ build-essential git software-properties-common libopenblas0 glpk-utils # Python 3.10 - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update -y apt-get install -y --no-install-recommends python3.10 python3.10-venv rm -rf /var/lib/apt/lists/* python3.10 /root/get-pip.py diff --git a/containers/tramway-hpc-py311 b/containers/tramway-hpc-py311 index 675a2f8..0e7c095 100644 --- a/containers/tramway-hpc-py311 +++ b/containers/tramway-hpc-py311 @@ -1,10 +1,10 @@ Bootstrap: docker -From: ubuntu:focal +From: ubuntu:22.04 %help TRamWAy is available in the python3.11 environment: python3.11 -m tramway -The container OS is Ubuntu Focal Fossa. +The container OS is Ubuntu 22.04. %setup @@ -24,7 +24,7 @@ The container OS is Ubuntu Focal Fossa. if [ -d $LOCAL ]; then CONTAINED=${SINGULARITY_ROOTFS}/root/TRamWAy mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/requirements.txt ${LOCAL}/README.md + cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/pyproject.toml ${LOCAL}/README.md cp -ru -t ${CONTAINED}/ ${LOCAL}/tramway ${LOCAL}/scripts fi @@ -34,8 +34,9 @@ The container OS is Ubuntu Focal Fossa. apt-get update -y apt-get install -y --no-install-recommends locales locale-gen - apt-get install -y --no-install-recommends libhdf5-103 ffmpeg \ - build-essential git software-properties-common libopenblas0 glpk-utils + apt-get install -y --no-install-recommends libhdf5-openmpi-dev ffmpeg \ + build-essential git software-properties-common libopenblas0 \ + glpk-utils gpg-agent # Python 3.11 add-apt-repository -y ppa:deadsnakes/ppa diff --git a/containers/tramway-hpc-py312 b/containers/tramway-hpc-py312 index cded142..a3f0467 100644 --- a/containers/tramway-hpc-py312 +++ b/containers/tramway-hpc-py312 @@ -1,10 +1,10 @@ Bootstrap: docker -From: ubuntu:focal +From: ubuntu:22.04 %help TRamWAy is available in the python3.12 environment: python3.12 -m tramway -The container OS is Ubuntu Focal Fossa. +The container OS is Ubuntu 22.04. %setup @@ -24,7 +24,7 @@ The container OS is Ubuntu Focal Fossa. if [ -d $LOCAL ]; then CONTAINED=${SINGULARITY_ROOTFS}/root/TRamWAy mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/requirements.txt ${LOCAL}/README.md + cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/pyproject.toml ${LOCAL}/README.md cp -ru -t ${CONTAINED}/ ${LOCAL}/tramway ${LOCAL}/scripts fi @@ -34,8 +34,9 @@ The container OS is Ubuntu Focal Fossa. apt-get update -y apt-get install -y --no-install-recommends locales locale-gen - apt-get install -y --no-install-recommends libhdf5-103 ffmpeg \ - build-essential git software-properties-common libopenblas0 glpk-utils + apt-get install -y --no-install-recommends libhdf5-openmpi-dev ffmpeg \ + build-essential git software-properties-common libopenblas0 \ + glpk-utils gpg-agent # Python 3.12 add-apt-repository -y ppa:deadsnakes/ppa diff --git a/containers/tramway-hpc-py313 b/containers/tramway-hpc-py313 index dab6006..13b9d92 100644 --- a/containers/tramway-hpc-py313 +++ b/containers/tramway-hpc-py313 @@ -1,10 +1,10 @@ Bootstrap: docker -From: ubuntu:focal +From: ubuntu:22.04 %help TRamWAy is available in the python3.13 environment: python3.13 -m tramway -The container OS is Ubuntu Focal Fossa. +The container OS is Ubuntu 22.04. %setup @@ -24,7 +24,7 @@ The container OS is Ubuntu Focal Fossa. if [ -d $LOCAL ]; then CONTAINED=${SINGULARITY_ROOTFS}/root/TRamWAy mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/requirements.txt ${LOCAL}/README.md + cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/pyproject.toml ${LOCAL}/README.md cp -ru -t ${CONTAINED}/ ${LOCAL}/tramway ${LOCAL}/scripts fi @@ -34,8 +34,9 @@ The container OS is Ubuntu Focal Fossa. apt-get update -y apt-get install -y --no-install-recommends locales locale-gen - apt-get install -y --no-install-recommends libhdf5-103 ffmpeg \ - build-essential git software-properties-common libopenblas0 glpk-utils + apt-get install -y --no-install-recommends libhdf5-openmpi-dev ffmpeg \ + build-essential git software-properties-common libopenblas0 \ + glpk-utils gpg-agent # Python 3.13 add-apt-repository -y ppa:deadsnakes/ppa diff --git a/containers/tramway-hpc-py38 b/containers/tramway-hpc-py314 similarity index 69% rename from containers/tramway-hpc-py38 rename to containers/tramway-hpc-py314 index acb5ab2..b3c3b4a 100644 --- a/containers/tramway-hpc-py38 +++ b/containers/tramway-hpc-py314 @@ -1,10 +1,10 @@ Bootstrap: docker -From: ubuntu:focal +From: ubuntu:22.04 %help -TRamWAy is available in the python3.8 environment: - python3.8 -m tramway -The container OS is Ubuntu Focal Fossa. +TRamWAy is available in the python3.14 environment: + python3.14 -m tramway +The container OS is Ubuntu 22.04. %setup @@ -24,7 +24,7 @@ The container OS is Ubuntu Focal Fossa. if [ -d $LOCAL ]; then CONTAINED=${SINGULARITY_ROOTFS}/root/TRamWAy mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/requirements.txt ${LOCAL}/README.md + cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/pyproject.toml ${LOCAL}/README.md cp -ru -t ${CONTAINED}/ ${LOCAL}/tramway ${LOCAL}/scripts fi @@ -34,18 +34,19 @@ The container OS is Ubuntu Focal Fossa. apt-get update -y apt-get install -y --no-install-recommends locales locale-gen - apt-get install -y --no-install-recommends libhdf5-103 ffmpeg \ - build-essential git software-properties-common libopenblas0 glpk-utils + apt-get install -y --no-install-recommends libhdf5-openmpi-dev ffmpeg \ + build-essential git software-properties-common libopenblas0 \ + glpk-utils gpg-agent - # Python 3.8 + # Python 3.14 add-apt-repository -y ppa:deadsnakes/ppa apt-get update -y - apt-get install -y --no-install-recommends python3.8 python3.8-venv + apt-get install -y --no-install-recommends python3.14 python3.14-venv rm -rf /var/lib/apt/lists/* - python3.8 /root/get-pip.py + python3.14 /root/get-pip.py export LC_ALL=C - #python3.8 -m pip install --upgrade pip - python3.8 -m pip uninstall -qy tramway || true + #pip3.14 install --upgrade pip + pip3.14 uninstall -qy tramway || true # (2.) and (3.) cd /root @@ -53,19 +54,19 @@ The container OS is Ubuntu Focal Fossa. git clone https://github.com/DecBayComp/TRamWAy -b dev --single-branch --depth 1 --no-tags fi cd TRamWAy - python3.8 -m pip install .[hpc-minimal,animate] + pip3.14 install .[hpc-minimal,animate] # (1.) - #python3.8 -m pip install tramway[hpc-minimal,animate] + #pip3.14 install tramway[hpc-minimal,animate] - python3.8 -m pip install scikit-learn + pip3.14 install scikit-learn mkdir -p /pasteur %runscript cmd="tramway" - python="python3.8" + python="python3.14" if [ -n "$1" -a "$1" = "-s" ]; then cmd="${python} -s -m tramway" shift diff --git a/containers/tramway-hpc-py39 b/containers/tramway-hpc-py39 index ffb6130..a57b705 100644 --- a/containers/tramway-hpc-py39 +++ b/containers/tramway-hpc-py39 @@ -24,7 +24,7 @@ The container OS is Ubuntu Focal Fossa. if [ -d $LOCAL ]; then CONTAINED=${SINGULARITY_ROOTFS}/root/TRamWAy mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/requirements.txt ${LOCAL}/README.md + cp -u -t ${CONTAINED}/ ${LOCAL}/setup.py ${LOCAL}/pyproject.toml ${LOCAL}/README.md cp -ru -t ${CONTAINED}/ ${LOCAL}/tramway ${LOCAL}/scripts fi @@ -35,7 +35,8 @@ The container OS is Ubuntu Focal Fossa. apt-get install -y --no-install-recommends locales locale-gen apt-get install -y --no-install-recommends libhdf5-103 ffmpeg \ - build-essential git software-properties-common libopenblas0 glpk-utils + build-essential git software-properties-common libopenblas0 \ + glpk-utils hdf5-tools # Python 3.9 add-apt-repository -y ppa:deadsnakes/ppa @@ -58,7 +59,7 @@ The container OS is Ubuntu Focal Fossa. # (1.) #python3.9 -m pip install tramway[hpc-minimal,animate] - python3.9 -m pip install scikit-learn + python3.9 -m pip install scikit-learn pytest mkdir -p /pasteur diff --git a/pyproject.toml b/pyproject.toml index a62233a..c7d3930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "tramway" -version = "0.6.9" +version = "0.7" description = "" authors = ["François Laurent "] license = "CECILL-2.1" readme = "README.md" [tool.poetry.dependencies] -python = ">=3.8,<4" +python = ">=3.9,<4" setuptools = ">=69.1.0,<99" six = ">=1.16,<2" numpy = ">=1.24.3,<3" @@ -15,6 +15,7 @@ scipy = ">=1.9.1,<2" pandas = ">=2.0.2,<3" matplotlib = ">=3.7.1,<4" rwa-python = ">=0.9.3,<1" +multiprocess = ">=0.70.19,<1" polytope = { version = ">=0.2.3,<1", optional = true } cvxopt = { version = ">=1.3.1,<2", optional = true } @@ -36,10 +37,9 @@ pytest = "^7.3.1" [tool.poetry.extras] # `full` is for backward compatibility; use --all-extras instead -full = ["polytope", "cvxopt", "paramiko", "stopit", "nbconvert", "bokeh", "selenium", "plotly", "nbformat", "opencv-python", "scikit-image", "tqdm"] +full = ["polytope", "cvxopt", "paramiko", "stopit", "nbconvert", "opencv-python", "scikit-image", "tqdm"] roi = ["polytope", "cvxopt", "tqdm"] animate = ["opencv-python", "scikit-image", "tqdm"] -webui = ["bokeh", "selenium", "plotly", "nbformat"] hpc-minimal = ["polytope", "cvxopt", "stopit"] hpc = ["polytope", "cvxopt", "paramiko", "stopit", "nbconvert"] diff --git a/scripts/tramway-browse b/scripts/tramway-browse deleted file mode 100755 index 6bc127a..0000000 --- a/scripts/tramway-browse +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 - -import tempfile as tmp -import sys -import subprocess - -def browse(files=[], browser='Firefox', colormap=None): - if not files: - import glob - filepattern = '*.rwa' - if not glob.glob(filepattern): - filepattern = '*/*.rwa' - if not glob.glob(filepattern): - print('cannot find any matching rwa file') - return - files = [filepattern] - - script = tmp.NamedTemporaryFile('w', suffix='.py', encoding='utf-8', delete=False) - source = """\ -#!/usr/bin/env python3 - -from tramway.analyzer import * -from selenium import webdriver - -a = RWAnalyzer() -a.spt_data.from_rwa_files(['{}']) -a.env.script = '{}' -{}a.browser.show_maps(webdriver=webdriver.{}) -""".format("', '".join(files), script.name, - '' if colormap is None else "a.browser.colormap = '{}'\n".format(colormap), - browser) - script.write(source) - script.flush() - p = subprocess.Popen([sys.executable, script.name], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - ret = p.communicate() - except KeyboardInterrupt: - ret = p.communicate() - script.close() - def tostr(s): - if isinstance(s, bytes): - s = s.decode('utf8') - return s - ret = [ tostr(r) for r in ret ] - return [ r for r in ret if r ] - - -def main(): - import argparse - parser = argparse.ArgumentParser(prog='tramway-browse', - description='Browse TRamWAy-generated .rwa files') - parser.add_argument('files', nargs='*', help='for example: *.rwa or */*.rwa') - parser.add_argument('--browser', default='Firefox', choices=['Firefox','Chrome','Edge','Ie','Opera','Safari','WebKitGTK']) - parser.add_argument('--colormap', help="Matplotlib colormap name") - [ print(ret) for ret in browse(**parser.parse_args().__dict__) ] - -if __name__ == '__main__': - main() - diff --git a/setup.py b/setup.py index 9ad16ce..dbcfd85 100644 --- a/setup.py +++ b/setup.py @@ -1,20 +1,16 @@ # -*- coding: utf-8 -*- from setuptools import setup -from codecs import open from os import path -# requirements moved to requirements.txt -install_requires = ['six', 'numpy', 'scipy', 'pandas', 'matplotlib', 'rwa-python>=0.8', 'setuptools'] +install_requires = ['six', 'numpy', 'scipy', 'pandas', 'matplotlib', 'rwa-python>=0.8', 'setuptools', 'multiprocess'] extras_require = { 'animate': ['opencv-python', 'scikit-image', 'tqdm'], 'roi': ['polytope', 'tqdm'], - 'webui': ['bokeh >=2.0.2, <2.3.0', 'selenium', 'plotly', 'nbformat'], 'hpc-minimal': ['polytope', 'stopit'], 'hpc': ['polytope', 'paramiko', 'stopit', 'nbconvert'], 'full': [ 'polytope', 'paramiko', 'stopit', 'nbconvert', - 'bokeh >=2.0.2, <2.3.0', 'selenium', 'plotly', 'nbformat', 'opencv-python', 'scikit-image', 'tqdm', ]} tests_require = ['pytest'] @@ -28,7 +24,7 @@ setup( name = 'tramway', - version = '0.6.9', + version = '0.7', description = 'TRamWAy', long_description = long_description, url = 'https://github.com/DecBayComp/TRamWAy', @@ -38,12 +34,12 @@ classifiers = [ 'Intended Audience :: Science/Research', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', ], keywords = '', package_dir = {'tramway': 'tramway'}, @@ -91,7 +87,7 @@ 'tramway.analyzer.localizer', 'tramway.analyzer.tracker', ], - scripts = ['scripts/tramway', 'scripts/tramway-browse'], + scripts = ['scripts/tramway'], install_requires = install_requires, extras_require = extras_require, tests_require = tests_require, diff --git a/tests/test_analyzer_pipeline.py b/tests/test_analyzer_pipeline.py index bee8994..e39a3ee 100644 --- a/tests/test_analyzer_pipeline.py +++ b/tests/test_analyzer_pipeline.py @@ -14,6 +14,8 @@ logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler()) +# for tests deported onto Maestro, first copy the test data: +# scp -r tests/test_analyzer_200803 maestro:~/Projects/Tramway/TRamWAy/tests/ def run_script0(tmpdir, dynamicmesh, env='', options=''): tmpdir = tmpdir.strpath @@ -29,28 +31,18 @@ def script(env='', options=''): import os import numpy import pandas -a=RWAnalyzer() -logger = a.logger -a._logger = BasicLogger() # let subprocess.check_output catch the log output -assert os.path.isfile(os.path.expanduser('{input}')) -a.spt_data.from_ascii_file('{input}') -a.spt_data.localization_precision = 1e-4 -roi = [[.2,-.1],[-.3,.3],[0.,.1],[-.2,-0.]] -a.roi.from_squares(numpy.array(roi), .2, group_overlapping_roi=True) -assert len(list(a.roi.as_support_regions()))==2 -a.tesseller = tessellers.Hexagons -a.time.from_sliding_window(30) + def infer(cells): i, n = zip(*[ (cell.index, len(cell)) for cell in cells.values() ]) return pandas.DataFrame(dict(n=list(n)), index=list(i)) -a.mapper.from_plugin(infer) -{env} + def fresh_start(self): for f in self.spt_data: try: os.unlink(os.path.expanduser(f.source[:-3]+'rwa')) except FileNotFoundError: pass + def tessellate(self): dry_run = True for f in self.spt_data: @@ -69,9 +61,11 @@ def tessellate(self): if dry_run: self.logger.info('stage skipped') assert False + def reload(self): #self.logger.debug(os.path.splitext(self.spt_data.source)[0]+'.rwa') self.spt_data.reload_from_rwa_files() + def map(self): dry_run = True for r in self.roi.as_support_regions(): @@ -88,12 +82,26 @@ def map(self): if dry_run: self.logger.info('stage skipped') assert False + +a = RWAnalyzer() +logger = a.logger +a._logger = BasicLogger() # let subprocess.check_output catch the log output +assert os.path.isfile(os.path.expanduser('{input}')) +a.spt_data.from_ascii_file('{input}') +a.spt_data.localization_precision = 1e-4 +roi = [[.2,-.1],[-.3,.3],[0.,.1],[-.2,-0.]] +a.roi.from_squares(numpy.array(roi), .2, group_overlapping_roi=True) +assert len(list(a.roi.as_support_regions()))==2 +a.tesseller = tessellers.Hexagons +a.time.from_sliding_window(30) +a.mapper.from_plugin(infer) +{env} a.pipeline.append_stage(fresh_start, run_everywhere=True) a.pipeline.append_stage(tessellate, granularity='roi') a.pipeline.append_stage(reload, requires_mutability=True) a.pipeline.append_stage(map, granularity='time segment'{options}) """.format(input=dynamicmesh.replace('\\','/'), env=env, options=options) - # + # # test = """ @@ -113,7 +121,8 @@ def map(self): with open(script_name, 'w') as f: f.write(script(env)) - f.write('a.run()\n') + f.write("if __name__ == '__main__':\n") + f.write(' a.run()\n') out = subprocess.check_output([sys.executable, script_name], encoding='utf8', timeout=360) logger.info(out) with open(script_name, 'w') as f: @@ -156,7 +165,15 @@ def script(env='', options=''): import os.path import numpy import pandas -a=RWAnalyzer() + +def fresh_start(self): + for f in self.spt_data: + try: + os.unlink(os.path.expanduser(f.source[:-3]+'rwa')) + except FileNotFoundError: + pass + +a = RWAnalyzer() logger = a.logger a._logger = BasicLogger() # let subprocess.check_output catch the log output assert os.path.isfile(os.path.expanduser('{input}')) @@ -165,6 +182,7 @@ def script(env='', options=''): a.tesseller = tessellers.GWR a.time.from_sliding_window(20) a.time.window_shift = 10 + translocations = single(a.spt_data).dataframe trajectories = translocations_to_trajectories(translocations) def infer(cells): @@ -179,14 +197,9 @@ def infer(cells): ids.append(i) tlen.append(numpy.mean(traj_ls)) return pandas.DataFrame({{'traj. len.': tlen}}, index=ids) + a.mapper.from_plugin(infer) {env} -def fresh_start(self): - for f in self.spt_data: - try: - os.unlink(os.path.expanduser(f.source[:-3]+'rwa')) - except FileNotFoundError: - pass a.pipeline.append_stage(fresh_start, run_everywhere=True) a.pipeline.append_stage(stages.tessellate(label='gwr')) a.pipeline.append_stage(stages.reload()) @@ -208,7 +221,8 @@ def fresh_start(self): with open(script_name, 'w') as f: f.write(script(env)) - f.write('a.run()\n') + f.write("if __name__ == '__main__':\n") + f.write(' a.run()\n') out = subprocess.check_output([sys.executable, script_name], encoding='utf8', timeout=360) logger.info(out) with open(script_name, 'w') as f: @@ -240,7 +254,19 @@ def script(env='', options=''): import os import numpy import pandas -a=RWAnalyzer() + +def infer(cells): + i, n = zip(*[ (cell.index, len(cell)) for cell in cells.values() ]) + return pandas.DataFrame(dict(n=list(n)), index=list(i)) + +def fresh_start(self): + for f in self.spt_data: + try: + os.unlink(os.path.expanduser(f.source[:-3]+'rwa')) + except FileNotFoundError: + pass + +a = RWAnalyzer() logger = a.logger a._logger = BasicLogger() # let subprocess.check_output catch the log output assert os.path.isfile(os.path.expanduser('{input}')) @@ -251,21 +277,12 @@ def script(env='', options=''): assert len(list(a.roi.as_support_regions()))==2 a.tesseller = tessellers.Hexagons a.time.from_sliding_window(30) -def infer(cells): - i, n = zip(*[ (cell.index, len(cell)) for cell in cells.values() ]) - return pandas.DataFrame(dict(n=list(n)), index=list(i)) a.mapper.from_plugin(infer) {env} -def fresh_start(self): - for f in self.spt_data: - try: - os.unlink(os.path.expanduser(f.source[:-3]+'rwa')) - except FileNotFoundError: - pass a.pipeline.append_stage(stages.tessellate_and_infer(map_label='n', overwrite={overwrite})) """.format(input=dynamicmesh.replace('\\','/'), env=env, options=options, overwrite=overwrite) - # + # # test = """ @@ -281,7 +298,8 @@ def fresh_start(self): with open(script_name, 'w') as f: f.write(script(env)) - f.write('a.run()\n') + f.write("if __name__ == '__main__':\n") + f.write(' a.run()\n') out = subprocess.check_output([sys.executable, script_name], encoding='utf8', timeout=360) logger.info(out) with open(script_name, 'w') as f: @@ -323,28 +341,25 @@ def test_LocalHost0(self, tmpdir, dynamicmesh): run_script0(tmpdir, dynamicmesh, env) def prepare_Maestro(self, dynamicmesh): - try: - with open(os.path.join(os.path.dirname(__file__), 'maestro.credentials'), 'r') as f: + credential_file = os.path.join(os.path.dirname(__file__), 'maestro.credentials') + if os.path.isfile(credential_file): + with open(credential_file, 'r') as f: username = f.readline().rstrip() password = f.readline().rstrip() - except FileNotFoundError: - username = os.environ['USER'] env = f"""\ a.env = environments.Maestro a.env.username = '{username}' -a.env.sbatch_options.update(dict(p='dbc_pmo', qos='dbc'))\ +a.env.ssh._password = '{password}' +a.env.sbatch_options.update(dict(p='dbc', qos='dbc'))\ """ else: - # + username = os.environ['USER'] env = f"""\ a.env = environments.Maestro a.env.username = '{username}' -a.env.ssh._password = '{password}' -a.env.sbatch_options.update(dict(p='dbc_pmo', qos='dbc'))\ +a.env.sbatch_options.update(dict(p='dbc', qos='dbc'))\ """ - # input_file = os.path.join('~', os.path.relpath(dynamicmesh, os.path.expanduser('~'))) - #input_file = '~/Projects/TRamWAy/tests/test_analyzer_200803/test04_moving_potential_sink.txt' return input_file, env def test_Maestro0(self, tmpdir, dynamicmesh): diff --git a/tests/test_commandline.py b/tests/test_commandline.py index edf65cb..919c60d 100644 --- a/tests/test_commandline.py +++ b/tests/test_commandline.py @@ -31,9 +31,9 @@ assert sys.version_info[0] == 3 -py3_hash = 'zUhBbH55' +py3_hash = 'xbCi14Pp' data_server = f'http://dl.pasteur.fr/fop/{py3_hash}/' -data_update = '250124' +data_update = '260216' data_file = 'glycine_receptor.trxyt' data_dir = f'test_commandline_py3_{data_update}' @@ -120,6 +120,44 @@ def execute(cmd, *args): cmd = cmd.format(*args) return subprocess.call(cmd.split()) +def cmp(file1, file2, dir_=None): + if dir_ is not None: + file1, file2 = os.path.join(dir_, file1), os.path.join(dir_, file2) + p = subprocess.Popen(('cmp', file1, file2), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + assert not out and not err + +features = {'diffusivity': 'd', 'potential': 'v', 'force': 'f'} + +def compare_plots(output_file, reference, rtype, records=[]): + dir_ = os.path.dirname(output_file) + suffixes = {'generated': output_file, 'expected': reference} + for suffix, file in suffixes.items(): + status = execute('{} -m tramway draw {} -i {} -p png', + sys.executable, rtype, file) + assert status == 0 + if not records: + assert rtype == 'cells' + for suffix, file1 in suffixes.items(): + file2 = file1[:-4]+'.png' + assert os.path.isfile(file2) + os.rename(file2, os.path.join(dir_, suffix+'.png')) + else: + assert rtype == 'map' + for suffix, file1 in suffixes.items(): + for record in records: + if len(records) == 1: + file2 = file1[:-4]+'.png' + else: + file2 = file1[:-4]+'_'+features.get(record, record)+'.png' + assert os.path.isfile(file2) + os.rename(file2, os.path.join(dir_, record+'_'+suffix+'.png')) + if not records: + cmp('generated.png', 'expected.png', dir_=dir_) + for record in records: + cmp(record+'_generated.png', record+'_expected.png', dir_=dir_) + class TestTessellation(object): @@ -145,6 +183,10 @@ def common(self, tmpdir, datadir, cmd, reference=None): assert os.path.isfile(output_file) if reference: reference = self.rwafile(reference) + # new procedure (faster checks in case of unstability) + compare_plots(output_file, reference, 'cells') + return + # former procedure p = subprocess.Popen(('h5diff', reference, output_file), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() @@ -218,7 +260,7 @@ class TestInference(object): def print(self, *args, **kwargs): _print(self.tmpdir, *args, **kwargs) - def common(self, tmpdir, datadir, cmd, reference): + def common(self, tmpdir, datadir, cmd, reference, vars=[]): self.tmpdir = tmpdir initial_file = prepare_file('inference_input.rwa', datadir, tmpdir) input_file = prepare_file('test_inference_{}.rwa'.format(reference), @@ -231,6 +273,10 @@ def common(self, tmpdir, datadir, cmd, reference): assert status == 0 ref_file = prepare_file(ref_file, datadir, tmpdir) generated, expected = input_file, ref_file + # new procedure (faster checks in case of unstability) + compare_plots(generated, expected, 'map', vars) + return + # former procedure p = subprocess.Popen(('h5diff', expected, generated), stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() @@ -253,19 +299,19 @@ def common(self, tmpdir, datadir, cmd, reference): assert not out def test_d(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'd -j', 'd0') + self.common(tmpdir, datadir, 'd -j', 'd0', ['diffusivity']) def test_df(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'df -j', 'df0') + self.common(tmpdir, datadir, 'df -j', 'df0', ['diffusivity', 'force']) def test_dd(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'dd -j', 'dd0') + self.common(tmpdir, datadir, 'dd -j', 'dd0', ['diffusivity', 'drift']) def test_dv0(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'dv -j --max-iter 10', 'dv0') + self.common(tmpdir, datadir, 'dv -j --max-iter 10', 'dv0', ['diffusivity', 'potential', 'force']) def test_dv1(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'dv -d 1 -v 1 --max-iter 10', 'dv1') + self.common(tmpdir, datadir, 'dv -d 1 -v 1 --max-iter 10', 'dv1', ['diffusivity', 'potential', 'force']) def test_smooth_d(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'd -d 1 -j --max-iter 10', 'd1') + self.common(tmpdir, datadir, 'd -d 1 -j --max-iter 10', 'd1', ['diffusivity']) def test_smooth_df(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'df -d 1 -j --max-iter 10', 'df1') + self.common(tmpdir, datadir, 'df -d 1 -j --max-iter 10', 'df1', ['diffusivity', 'force']) def test_smooth_dd(self, tmpdir, datadir): - self.common(tmpdir, datadir, 'dd -d 1 -j --max-iter 10', 'dd1') + self.common(tmpdir, datadir, 'dd -d 1 -j --max-iter 10', 'dd1', ['diffusivity', 'drift']) diff --git a/tests/test_core.py b/tests/test_core.py index dce103e..e3bd389 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -148,7 +148,7 @@ def test_reindex(self): 796888 193576 30.150600 30.155199 518.280029 842499 204282 30.147100 30.192900 541.239990 907067 219796 30.189301 30.164801 570.400024\ -"""), delim_whitespace=True) +"""), sep='\s+') df = reindex_trajectories(df) assert numpy.all(df['n'].values == numpy.r_[1,1,2,3,3,3,4,5,6,7]) reindex_trajectories(pandas.DataFrame([], columns=list('nxyt'))) diff --git a/tramway/analyzer/artefact/__init__.py b/tramway/analyzer/artefact/__init__.py index 5129699..cca3c30 100644 --- a/tramway/analyzer/artefact/__init__.py +++ b/tramway/analyzer/artefact/__init__.py @@ -19,7 +19,7 @@ # from tramway.core.analyses.base import Analyses from tramway.core.analyses.lazy import Analyses -import pkg_resources +import importlib.metadata import platform import time @@ -28,7 +28,7 @@ def standard_metadata(): return dict( os=platform.system(), python=platform.python_version(), - tramway=pkg_resources.get_distribution("tramway").version, + tramway=importlib.metadata.version("tramway"), datetime=time.strftime("%Y-%m-%d %H:%M:%S UTC%z"), ) diff --git a/tramway/analyzer/env/containers.py b/tramway/analyzer/env/containers.py index fa5eb91..43b9058 100644 --- a/tramway/analyzer/env/containers.py +++ b/tramway/analyzer/env/containers.py @@ -34,22 +34,28 @@ #"tramway-hpc-220307-py310.sif": "http://dl.pasteur.fr/fop/G0Orpfeo/tramway-hpc-220307-py310.sif", #"tramway-hpc-220312-py36.sif": "http://dl.pasteur.fr/fop/u1IwuvG7/tramway-hpc-220312-py36.sif", #"tramway-hpc-220312-py37.sif": "http://dl.pasteur.fr/fop/OBz1IGfd/tramway-hpc-220312-py37.sif", - "tramway-hpc-220312-py38.sif": "http://dl.pasteur.fr/fop/n2SSFwr5/tramway-hpc-220312-py38.sif", - "tramway-hpc-220312-py39.sif": "http://dl.pasteur.fr/fop/W6GcBTqk/tramway-hpc-220312-py39.sif", - "tramway-hpc-220312-py310.sif": "http://dl.pasteur.fr/fop/Cmmobhil/tramway-hpc-220312-py310.sif", + #"tramway-hpc-220312-py38.sif": "http://dl.pasteur.fr/fop/n2SSFwr5/tramway-hpc-220312-py38.sif", + #"tramway-hpc-220312-py39.sif": "http://dl.pasteur.fr/fop/W6GcBTqk/tramway-hpc-220312-py39.sif", + #"tramway-hpc-220312-py310.sif": "http://dl.pasteur.fr/fop/Cmmobhil/tramway-hpc-220312-py310.sif", #"tramway-hpc-230609-py38.sif": "https://dl.pasteur.fr/fop/74lfEWxO/tramway-hpc-230609-py38.sif", #"tramway-hpc-230609-py39.sif": "https://dl.pasteur.fr/fop/XgQ06STg/tramway-hpc-230609-py39.sif", - "tramway-hpc-230609-py310.sif": "https://dl.pasteur.fr/fop/keDoKINW/tramway-hpc-230609-py310.sif", - "tramway-hpc-230609-py311.sif": "https://dl.pasteur.fr/fop/znbYr11f/tramway-hpc-230609-py311.sif", - "tramway-hpc-240423-py38.sif": "https://dl.pasteur.fr/fop/B3LCHpTL/tramway-hpc-240423-py38.sif", - "tramway-hpc-240423-py39.sif": "https://dl.pasteur.fr/fop/crbAgtuu/tramway-hpc-240423-py39.sif", - "tramway-hpc-240423-py310.sif": "https://dl.pasteur.fr/fop/zZ2nrnKP/tramway-hpc-240423-py310.sif", - "tramway-hpc-240423-py311.sif": "https://dl.pasteur.fr/fop/7F8PPD7j/tramway-hpc-240423-py311.sif", - "tramway-hpc-240423-py312.sif": "https://dl.pasteur.fr/fop/lRWEB8oR/tramway-hpc-240423-py312.sif", - "tramway-hpc-250124-py38.sif": "https://dl.pasteur.fr/fop/xVjMGhTa/tramway-hpc-250124-py38.sif", - "tramway-hpc-250124-py39.sif": "https://dl.pasteur.fr/fop/AyK3JTC0/tramway-hpc-250124-py39.sif", - "tramway-hpc-250124-py310.sif": "https://dl.pasteur.fr/fop/jEmrwWhp/tramway-hpc-250124-py310.sif", - "tramway-hpc-250124-py311.sif": "https://dl.pasteur.fr/fop/QmL7TE6w/tramway-hpc-250124-py311.sif", - "tramway-hpc-250124-py312.sif": "https://dl.pasteur.fr/fop/elytCrRB/tramway-hpc-250124-py312.sif", - "tramway-hpc-250124-py313.sif": "https://dl.pasteur.fr/fop/hSeIMslv/tramway-hpc-250124-py313.sif", + #"tramway-hpc-230609-py310.sif": "https://dl.pasteur.fr/fop/keDoKINW/tramway-hpc-230609-py310.sif", + #"tramway-hpc-230609-py311.sif": "https://dl.pasteur.fr/fop/znbYr11f/tramway-hpc-230609-py311.sif", + #"tramway-hpc-240423-py38.sif": "https://dl.pasteur.fr/fop/B3LCHpTL/tramway-hpc-240423-py38.sif", + #"tramway-hpc-240423-py39.sif": "https://dl.pasteur.fr/fop/crbAgtuu/tramway-hpc-240423-py39.sif", + #"tramway-hpc-240423-py310.sif": "https://dl.pasteur.fr/fop/zZ2nrnKP/tramway-hpc-240423-py310.sif", + #"tramway-hpc-240423-py311.sif": "https://dl.pasteur.fr/fop/7F8PPD7j/tramway-hpc-240423-py311.sif", + #"tramway-hpc-240423-py312.sif": "https://dl.pasteur.fr/fop/lRWEB8oR/tramway-hpc-240423-py312.sif", + "tramway-hpc-250124-py38.sif": "https://dl.pasteur.fr/fop/fnQY9aI4/tramway-hpc-250124-py38.sif", + "tramway-hpc-250124-py39.sif": "https://dl.pasteur.fr/fop/bqIvcQOu/tramway-hpc-250124-py39.sif", + "tramway-hpc-250124-py310.sif": "https://dl.pasteur.fr/fop/CGSBWArs/tramway-hpc-250124-py310.sif", + "tramway-hpc-250124-py311.sif": "https://dl.pasteur.fr/fop/AFE8uuyA/tramway-hpc-250124-py311.sif", + "tramway-hpc-250124-py312.sif": "https://dl.pasteur.fr/fop/4KuTx4gX/tramway-hpc-250124-py312.sif", + "tramway-hpc-250124-py313.sif": "https://dl.pasteur.fr/fop/5XftLW2h/tramway-hpc-250124-py313.sif", + "tramway-hpc-260217-py39.sif": "https://dl.pasteur.fr/fop/vk2BlcFV/tramway-hpc-260217-py39.sif", + "tramway-hpc-260217-py310.sif": "https://dl.pasteur.fr/fop/vt7BJqDk/tramway-hpc-260217-py310.sif", + "tramway-hpc-260217-py311.sif": "https://dl.pasteur.fr/fop/yZNQ1Wn4/tramway-hpc-260217-py311.sif", + "tramway-hpc-260217-py312.sif": "https://dl.pasteur.fr/fop/X4Pqd1Bk/tramway-hpc-260217-py312.sif", + "tramway-hpc-260217-py313.sif": "https://dl.pasteur.fr/fop/FBEYEP0E/tramway-hpc-260217-py313.sif", + "tramway-hpc-260217-py314.sif": "https://dl.pasteur.fr/fop/NbJmUfmQ/tramway-hpc-260217-py314.sif", } diff --git a/tramway/analyzer/env/environments.py b/tramway/analyzer/env/environments.py index dc626a1..d2cbf05 100644 --- a/tramway/analyzer/env/environments.py +++ b/tramway/analyzer/env/environments.py @@ -22,7 +22,7 @@ import os import sys import time -import multiprocessing +import multiprocess as mp import subprocess import tempfile import shutil @@ -952,9 +952,9 @@ def worker_count(self): @worker_count.setter def worker_count(self, wc): if wc is None: - wc = max(1, multiprocessing.cpu_count() - 1) + wc = max(1, mp.cpu_count() - 1) elif wc < 0: - wc = max(1, multiprocessing.cpu_count() + wc) + wc = max(1, mp.cpu_count() + wc) self._worker_count = wc def submit_jobs(self): @@ -1741,24 +1741,25 @@ def filter_script_content(self, content): # last line is "[analyzer].run()\n" call = line.find(".run()") if 0 < call: - analyzer_var = line[:call] + start = 0 + while line[start].isspace(): + start += 1 + indent = line[:start] + analyzer_var = line[start:call] # insert mapping for home directories filtered_content = filtered_content[:-1] filtered_content.append( """ -if {1}.env.directory_mapping is None: - {1}.env.directory_mapping = {{}} -if '{0}' not in {1}.env.directory_mapping: - {1}.env.directory_mapping['{0}'] = '~' -""".format( - os.path.expanduser("~").replace("\\", r"\\"), analyzer_var - ) - ) +{2}if {1}.env.directory_mapping is None: +{2} {1}.env.directory_mapping = {{}} +{2}if '{0}' not in {1}.env.directory_mapping: +{2} {1}.env.directory_mapping['{0}'] = '~' +""".format(os.path.expanduser("~").replace("\\", r"\\"), analyzer_var, indent)) filtered_content.append(line) # append output listing filtered_content.append( - "\nprint({0}.env.collectible_prefix()+';'.join({0}.env.format_collectibles()))\n".format( - analyzer_var + "\n{1}print({0}.env.collectible_prefix()+';'.join({0}.env.format_collectibles()))\n".format( + analyzer_var, indent ) ) # @@ -2145,7 +2146,7 @@ def list_uncomplete_task_log_files(self): def select_python_version(major, minor, *args): if major == 3: - minor = max(6, min(13, minor)) + minor = max(6, min(14, minor)) return major, minor @@ -2156,7 +2157,7 @@ class SingularitySlurm(SlurmOverSSH): """ Runs TRamWAy jobs as Slurm jobs in a Singularity container. - The current default Singularity container is *tramway-hpc-250124-py3?.sif*. + The current default Singularity container is *tramway-hpc-260217-py3?.sif*. See also `available_images.md `_. Children classes should define the :meth:`hostname` and :meth:`scratch` methods. @@ -2201,7 +2202,7 @@ def interpreter(self): f.write( r"""#!/bin/bash -for ((minor=12;6<=minor;minor--)); do +for ((minor=14;6<=minor;minor--)); do py=python3.$minor if [ -x "$(command -v $py)" ]; then if [ -z "$($py -m pip show -q tramway 2>&1)" ]; then @@ -2315,7 +2316,7 @@ def get_container_url(self, container=None): @classmethod def default_container(cls, python_version=PYVER): - return f"tramway-hpc-250124-py{python_version}.sif" + return f"tramway-hpc-260217-py{python_version}.sif" def early_setup(self, *argv, connect=False, ensure_container=True, **kwargs): ret = SlurmOverSSH.early_setup(self, *argv, connect=connect, **kwargs) diff --git a/tramway/analyzer/pipeline/__init__.py b/tramway/analyzer/pipeline/__init__.py index 674d6c5..eb8dc3d 100644 --- a/tramway/analyzer/pipeline/__init__.py +++ b/tramway/analyzer/pipeline/__init__.py @@ -392,7 +392,7 @@ def run(self): raise NotImplementedError self.logger.info("\njobs ready") #### IMPORTANT #### - # any change below may be to be also applied to `env.SlurmOverSSH.resume` + # any change below may have to be also applied to `env.SlurmOverSSH.resume` ################### try: self.env.submit_jobs() diff --git a/tramway/analyzer/tracker/__init__.py b/tramway/analyzer/tracker/__init__.py index 1916838..1111325 100644 --- a/tramway/analyzer/tracker/__init__.py +++ b/tramway/analyzer/tracker/__init__.py @@ -213,10 +213,10 @@ def track(self, locations, register=False, source=None): if currently_assigned: source_ = movie_per_frame[frame_index] destination = movie_per_frame[frame_index + 1] - row_to_col = np.full(len(source_), -1, dtype=int) - row_to_col[row] = col if row.size == 1: row, col = _tolist(row), _tolist(col) + row_to_col = np.full(len(source_), -1, dtype=int) + row_to_col[row] = col newly_assigned = set(row) new_assignment = np.zeros(len(destination), dtype=np.uint32) growing_trajectory = currently_assigned & newly_assigned diff --git a/tramway/core/parallel/__init__.py b/tramway/core/parallel/__init__.py index 02e3c02..3545ad5 100644 --- a/tramway/core/parallel/__init__.py +++ b/tramway/core/parallel/__init__.py @@ -16,7 +16,7 @@ import cloudpickle except ImportError: pass -import multiprocessing +import multiprocess as mp try: import queue @@ -53,7 +53,7 @@ class StarQueue(object): __slots__ = ("deck",) - def __init__(self, n, variant=multiprocessing.Queue, **kwargs): + def __init__(self, n, variant=mp.Queue, **kwargs): self.deck = [] queues = [variant(**kwargs) for _ in range(n)] others = [] @@ -78,7 +78,7 @@ def __init__(self, input_queue=None, output_queues=None): @property def joinable(self): - return isinstance(self.input, multiprocessing.JoinableQueue) + return isinstance(self.input, mp.JoinableQueue) @property def variant(self): @@ -157,7 +157,7 @@ def __str__(self): ) -class Worker(multiprocessing.Process): +class Worker(mp.Process): """Worker that runs job steps. The :meth:`target` method may be implemented following the pattern below: @@ -197,7 +197,7 @@ def __init__( __kwargs = {} else: __kwargs = dict(daemon=daemon) - multiprocessing.Process.__init__(self, name=name, **__kwargs) + mp.Process.__init__(self, name=name, **__kwargs) self._id = _id self.workspace = workspace self.tasks = task_queue @@ -354,7 +354,7 @@ def __init__( """ try: # in Python 3.8.10, daemon mode was default - multiprocessing.process.current_process()._config["daemon"] = False + mp.process.current_process()._config["daemon"] = False except Exception: warn("failed to force-disable daemon mode") self.workspace = workspace @@ -367,14 +367,16 @@ def __init__( self.global_timeout = max_runtime self.task_timeout = task_timeout if worker_count is None: - worker_count = multiprocessing.cpu_count() - 1 + worker_count = mp.cpu_count() - 2 elif worker_count < 0: - worker_count = multiprocessing.cpu_count() + worker_count + worker_count = mp.cpu_count() + worker_count + elif worker_count < 1: + worker_count = round(mp.cpu_count() * worker_count) kwargs = dict(kwargs) # copy kwargs.update(_kwargs) if worker_count: - self.task_queue = multiprocessing.Queue() - self.return_queue = multiprocessing.Queue() + self.task_queue = mp.Queue() + self.return_queue = mp.Queue() if worker_count == 1: self.workers = { 0: self.worker( @@ -502,7 +504,11 @@ def get_processed_step(self): while True: # module_logger.debug('get_processed_step: waiting...') # DEBUG # note: `get` does not raise TimeoutError - step, status = self.return_queue.get(timeout=self.timeout) + try: + step, status = self.return_queue.get(timeout=self.timeout) + except queue.Empty: + self.logger.critical('Abnormal termination: empty queue; consider decreasing the number of workers') + return False # module_logger.debug('get_processed_step: received {}'.format(self.active[step.resource_id])) # DEBUG if step is None: if isinstance(status, WorkerNearDeathException): @@ -513,8 +519,6 @@ def get_processed_step(self): print("error: {}".format(status)) if not self.workers: return False - else: - return self.stop(None, None, status) else: break # step.set_workspace(self.workspace) # reload workspace @@ -627,7 +631,7 @@ def __exit__(self, *args): if self.global_timeout and self.global_timeout <= ( time.time() - self.start_time ): - # raise multiprocessing.TimeoutError + # raise mp.TimeoutError break if postponed: for i in list(postponed): @@ -641,7 +645,7 @@ def __exit__(self, *args): if not postponed: k = self.fill_slots(k, postponed) ret = True - except (SystemExit, KeyboardInterrupt, multiprocessing.TimeoutError): + except (SystemExit, KeyboardInterrupt, mp.TimeoutError): ret = False for w in self.workers.values(): try: @@ -696,7 +700,7 @@ def __init__( ): epoch_length = len(tasks) if epoch_length is None else epoch_length if not soft_epochs and not worker_count: - worker_count = min(epoch_length, multiprocessing.cpu_count() - 1) + worker_count = min(epoch_length, mp.cpu_count() - 2) Scheduler.__init__( self, workspace, diff --git a/tramway/helper/base.py b/tramway/helper/base.py index f7dec3a..db119b0 100644 --- a/tramway/helper/base.py +++ b/tramway/helper/base.py @@ -17,7 +17,7 @@ import os.path import six import pandas as pd -import pkg_resources +import importlib.metadata import platform import time @@ -44,11 +44,11 @@ def add_metadata(self, analysis, pkg_version=[]): if "python" not in analysis.metadata: analysis.metadata["python"] = platform.python_version() for pkg in pkg_version: - analysis.metadata["pkg"] = pkg_resources.get_distribution(pkg).version + analysis.metadata["pkg"] = importlib.metadata.version(pkg) if "tramway" not in analysis.metadata: - analysis.metadata["tramway"] = pkg_resources.get_distribution( + analysis.metadata["tramway"] = importlib.metadata.version( "tramway" - ).version + ) if "datetime" not in analysis.metadata: analysis.metadata["datetime"] = time.strftime("%Y-%m-%d %H:%M:%S UTC%z") diff --git a/tramway/inference/base.py b/tramway/inference/base.py index a958961..2180837 100644 --- a/tramway/inference/base.py +++ b/tramway/inference/base.py @@ -28,7 +28,7 @@ import cloudpickle except ImportError: pass -from multiprocessing import Pool, Lock +from multiprocess import Pool, Lock import os # for os.name import six diff --git a/tramway/inference/bayes_factors/calculate_bayes_factors.py b/tramway/inference/bayes_factors/calculate_bayes_factors.py index b8ee9df..20a1b97 100644 --- a/tramway/inference/bayes_factors/calculate_bayes_factors.py +++ b/tramway/inference/bayes_factors/calculate_bayes_factors.py @@ -12,7 +12,8 @@ from numpy.linalg import norm from .calculate_marginalized_integral import (calculate_integral_ratio, - calculate_marginalized_integral) + calculate_marginalized_integral, + asscalar) from .calculate_posteriors import get_lambda_MAP from .convenience_functions import n_pi_func from .convenience_functions import p as pow @@ -131,7 +132,7 @@ def _trange(x): return trange(x, desc='Bayes factor calculation') # Report error if any try: - logging.warn( + logging.warning( "A NaN value was present in the input parameters for the following cells: {nan_cells_list}.\nBayes factor calculations were skipped for them".format(nan_cells_list=nan_cells_list)) except NameError: pass @@ -208,6 +209,7 @@ def calculate_minimal_n(zeta_t, zeta_sp, n0, V, V_pi, loc_error, dim=2, B_thresh Return - 1 if unable to find the min_n """ + n0, V, V_pi = asscalar(n0), asscalar(V), asscalar(V_pi) test = check_for_nan(zeta_t, zeta_sp, n0, V, V_pi, loc_error) if test != 'ok': logging.warning( diff --git a/tramway/inference/bayes_factors/calculate_marginalized_integral.py b/tramway/inference/bayes_factors/calculate_marginalized_integral.py index eef5a60..41ba555 100644 --- a/tramway/inference/bayes_factors/calculate_marginalized_integral.py +++ b/tramway/inference/bayes_factors/calculate_marginalized_integral.py @@ -15,6 +15,13 @@ # max_terms = int(1e5) +def asscalar(x): + if isinstance(x, np.ndarray): + assert len(x) == 1 + x = x[0] + return x + + def calculate_marginalized_integral(zeta_t, zeta_sp, p, v, E, rel_loc_error, zeta_a=[0, 0], factor_za=0, lamb='int'): r""" Calculate the marginalized lambda integral @@ -200,7 +207,7 @@ def ln_integral_with_loc_error(arg_func, pow, x0): """ f, ln_prefactor = get_f_with_loc_error(arg_func, pow, x0) if lamb == 'marg': - ln_res = log(integrate.quad(f, 0, 1, points=break_points, epsrel=rtol)[0]) + ln_res = log(integrate.quad(lambda x: asscalar(f(x)), 0, 1, points=break_points, epsrel=rtol)[0]) else: ln_res = log(f(lamb)) return ln_res + ln_prefactor @@ -213,7 +220,7 @@ def ln_f(l): return -pow * log(arg_func(l) / x0) def f(l): - return exp(ln_f(l)) + return asscalar(exp(ln_f(l))) if lamb == 'marg': ln_res = integrate.quad(f, 0, 1, points=break_points, epsrel=rtol)[0] ln_res = log(ln_res) diff --git a/tramway/inference/bayes_factors/test_calculate_bayes_factor.py b/tramway/inference/bayes_factors/test_calculate_bayes_factor.py index 9f2f022..3691e94 100644 --- a/tramway/inference/bayes_factors/test_calculate_bayes_factor.py +++ b/tramway/inference/bayes_factors/test_calculate_bayes_factor.py @@ -6,7 +6,7 @@ import logging import math import unittest -from multiprocessing import freeze_support +# from multiprocessing import freeze_support import numpy as np from scipy.integrate import dblquad, quad @@ -167,7 +167,7 @@ def test_bayes_factors(self): true_B = 0.2974282533 # Check value - self.assertTrue(np.isclose(10**lg_Bs[0], true_B, rtol=self.rel_tol, atol=self.tol), + self.assertTrue(np.isclose(10**lg_Bs[0, 0], true_B, rtol=self.rel_tol, atol=self.tol), "Bayes factor calculation failed for one bin. The obtained B = %.8g does not match the expected B = %.8g" % (10**lg_Bs[0, 0], true_B)) # Check force presence self.assertTrue((true_B >= self.B_threshold) == @@ -193,8 +193,8 @@ def test_bayes_factors(self): for i in range(N): # Check value - self.assertTrue(np.isclose(10**lg_Bs[i], true_Bs[i], rtol=self.rel_tol, atol=self.tol), - "Bayes factor calculation failed for one bin. For bin no. %i, the obtained B = %.8g does not match the expected B = %.8g" % (i + 1, 10**lg_Bs[i], true_Bs[i])) + self.assertTrue(np.isclose(10**lg_Bs[i, 0], true_Bs[i], rtol=self.rel_tol, atol=self.tol), + "Bayes factor calculation failed for one bin. For bin no. %i, the obtained B = %.8g does not match the expected B = %.8g" % (i + 1, 10**lg_Bs[i, 0], true_Bs[i])) # Check force presence true_forces = (1 * (np.log10(true_Bs[i]) >= np.log10(self.B_threshold)) - 1 * (np.log10(true_Bs[i]) <= -np.log10(self.B_threshold))) diff --git a/tramway/inference/unsmooth_d.py b/tramway/inference/unsmooth_d.py index 2be00fd..2863289 100644 --- a/tramway/inference/unsmooth_d.py +++ b/tramway/inference/unsmooth_d.py @@ -88,7 +88,7 @@ def d_neg_posterior( d_neg_posterior = n * log(pi) + np.sum(np.log(D_dt)) # sum(log(4*pi*Dtot*dt)) d_neg_posterior += np.sum(cell.cache / D_dt) # sum((dx**2+dy**2+..)/(4*Dtot*dt)) if jeffreys_prior: - d_neg_posterior += 2.0 * log(diffusivity * dt_mean + sigma2) + d_neg_posterior += 2.0 * np.log(diffusivity * dt_mean + sigma2) return d_neg_posterior diff --git a/tramway/inference/unsmooth_ddrift.py b/tramway/inference/unsmooth_ddrift.py index 42f6ebc..1e55fda 100644 --- a/tramway/inference/unsmooth_ddrift.py +++ b/tramway/inference/unsmooth_ddrift.py @@ -66,7 +66,7 @@ def dd_neg_posterior(x, dd, cell, sigma2, jeffreys_prior, dt_mean, min_diffusivi n * log(pi) + np.sum(np.log(denominator)) + np.sum(ndsd / denominator) ) if jeffreys_prior: - neg_posterior += 2.0 * log(D * dt_mean + sigma2) + neg_posterior += 2.0 * np.log(D * dt_mean + sigma2) return neg_posterior diff --git a/tramway/inference/unsmooth_df.py b/tramway/inference/unsmooth_df.py index 344358f..d0bcd52 100644 --- a/tramway/inference/unsmooth_df.py +++ b/tramway/inference/unsmooth_df.py @@ -90,7 +90,7 @@ def df_neg_posterior(x, df, cell, sigma2, jeffreys_prior, dt_mean, min_diffusivi ) if jeffreys_prior: try: - neg_posterior += 2.0 * (log(D * dt_mean + sigma2) - log(D)) + neg_posterior += 2.0 * (log(D * dt_mean + sigma2) - np.log(D)) except ValueError as e: # math domain error warn(DiffusivityWarning(e)) return neg_posterior diff --git a/tramway/tessellation/gwr/gas.py b/tramway/tessellation/gwr/gas.py index 1782f60..9c45b6d 100644 --- a/tramway/tessellation/gwr/gas.py +++ b/tramway/tessellation/gwr/gas.py @@ -202,14 +202,14 @@ def habituation_function(self, t, i=0): float or array: habituation. """ return self.habituation_initial - ( - 1 - exp(-self.habituation_alpha[i] * t / self.habituation_tau[i]) + 1 - np.exp(-self.habituation_alpha[i] * t / self.habituation_tau[i]) ) / (self.habituation_alpha[i]) def habituation(self, node, i=0): """Returns the habituation of a node. Set ``i=0`` for the nearest node, ``i=1`` for a neighbor of the nearest node.""" return self.habituation_function( - float(self.get_node_attr(node, "habituation_counter")), i + self.get_node_attr(node, "habituation_counter").astype(float), i ) def plot_habituation(self): @@ -622,7 +622,7 @@ def train( fit = 0 else: fit = 1 / ( - 1 + exp((0.1 - regression.pvalues[1]) / 0.01) + 1 + np.exp((0.1 - regression.pvalues[1]) / 0.01) ) # invert p-value do = tolerance < fit elif stopping_criterion == 2: