Update for recent pandas indices and scipy.spatial types

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Unit tests and code coverage
+name: Unit tests
 
 on:
   push:
@@ -20,7 +20,7 @@ jobs:
           - ubuntu-latest
           - macos-latest
           - windows-latest
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
       fail-fast: true
 
     steps:
@@ -43,22 +43,3 @@ jobs:
     - name: Test with pytest
       run: |
         pytest
-
-  coverage:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Install dependencies and run tests
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install pytest-cov
-        python -m pip install .
-        python -m pytest --cov=./
-    - uses: codecov/codecov-action@v1
-      with:
-        fail_ci_if_error: true

README.md

@@ -0,0 +1,81 @@
+[![](https://img.shields.io/badge/docs-latest-blue.svg)](https://rwa-python.readthedocs.io/en/latest/)
+[![](https://github.com/DecBayComp/RWA-python/actions/workflows/ci.yml/badge.svg)](https://github.com/DecBayComp/RWA-python/actions/workflows/ci.yml)
+[![](https://codecov.io/gh/DecBayComp/RWA-python/branch/master/graph/badge.svg)](https://codecov.io/gh/DecBayComp/RWA-python)
+
+# RWA-python
+
+**RWA-python** serializes Python datatypes and stores them in HDF5 files.
+
+## Code example
+
+In module *A* referenced in *sys.path*:
+
+```python
+class CustomClass(object):
+  def __init__(self, arg=None):
+    self.attr = arg
+```
+
+In module *B*:
+
+```python
+from A import CustomClass
+from rwa import HDF5Store
+
+# make any complex construct
+any_object = CustomClass((CustomClass('a'), dict(b=1)))
+
+# serialize
+hdf5 = HDF5Store('my_file.h5', 'w')
+hdf5.poke('my object', any_object)
+hdf5.close()
+
+# deserialize
+hdf5 = HDF5Store('my_file.h5', 'r')
+reloaded_object = hdf5.peek('my object')
+hdf5.close()
+```
+
+## Introduction
+
+With Python3, **RWA-python** serialization is fully automatic for types with *__slots__* defined or such that the *__init__* constructor does not require any input argument.
+
+The library generates serialization schemes for most custom types.
+When deserializing objects, it also looks for and loads the modules where the corresponding types are defined.
+
+If **RWA-python** complains about a type that cannot be serialized, a partial fix consists of ignoring this datatype:
+
+```python
+hdf5_not_storable(type(unserializable_object))
+```
+
+With Python2, the library requires explicit definitions in most cases.
+In addition, string typing is sometimes problematic. Non-ascii characters should be explicit unicode.
+
+
+## Installation
+
+Python >= 3.5 is required. **RWA-python** may still work with Python 2.7 but support has been dropped.
+
+Windows users should favor Conda for installing **RWA-python**, as Conda will seamlessly install the HDF5 standard library which is a required dependency.
+
+For other users, *pip* should work just fine:
+```
+pip install --user rwa-python
+```
+*pip install* will install some Python dependencies if missing, but you may still need to install the [HDF5 reference library](https://tramway.readthedocs.io/en/latest/libhdf5.html).
+Note that most package managers include this library.
+
+The **RWA-python** package can also be installed using Conda:
+```
+conda install rwa-python -c conda-forge
+```
+or poetry:
+```
+poetry add rwa-python
+```
+
+## See also
+
+**RWA-python** is on [readthedocs](https://rwa-python.readthedocs.io/en/latest/).
+

containers/rwa-focal

@@ -0,0 +1 @@
+rwa-openmpi-dev

containers/rwa-jammy

@@ -0,0 +1,144 @@
+Bootstrap: docker
+From: ubuntu:22.04
+
+%help
+RWA-python is available in multiple Python environments:
+    python2.7 (alias python2)
+    python3.5
+    python3.6
+    python3.7
+    python3.8
+    python3.9
+    python3.10 (alias python3)
+    python3.11
+The container OS is Ubuntu Jammy.
+
+%setup
+
+    echo "fr_FR.UTF-8 UTF-8" > ${SINGULARITY_ROOTFS}/etc/locale.gen
+    for OLD in 2.7 3.5 3.6; do
+        if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py ]; then
+            wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py
+            mv ${SINGULARITY_ROOTFS}/root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py
+        fi
+    done
+    if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip.py ]; then
+        wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/get-pip.py
+    fi
+
+    # test local changes that have not been committed yet;
+    # to be run from any subdirectory in the RWA-python directory, e.g. containers, tests...
+    LOCAL="$(pwd)/.."
+    CONTAINED=${SINGULARITY_ROOTFS}/root/RWA-python
+    mkdir  -p ${CONTAINED}
+    cp -u  -t ${CONTAINED}/ \
+              ${LOCAL}/setup.py ${LOCAL}/requirements.txt \
+              ${LOCAL}/README.rst ${LOCAL}/LICENSE
+    cp -ru -t ${CONTAINED}/ ${LOCAL}/rwa ${LOCAL}/.git
+    cd ${CONTAINED} && git clean -xfd
+
+%post
+
+    ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime
+    apt-get update  -y
+    apt-get install -y --no-install-recommends locales
+    locale-gen
+
+    apt-get install -y --no-install-recommends libhdf5-openmpi-dev \
+        build-essential git software-properties-common pkg-config
+
+    add-apt-repository -y ppa:deadsnakes/ppa
+    apt-get update  -y
+    apt-get install -y --no-install-recommends \
+        python2.7 python2.7-dev \
+        python3.5 python3.5-dev \
+        python3.6 python3.6-dev python3.6-distutils \
+        python3.7 python3.7-dev python3.7-distutils \
+        python3.8 python3.8-dev python3.8-venv \
+        python3.9 python3.9-dev \
+        python3.10 python3.10-dev python3.10-venv \
+        python3.11 python3.11-dev python3.11-venv
+
+    # python 3.8 and 3.10 won't work with get-pip.py without package venv
+
+    for OLD in 2.7 3.5 3.6; do
+        python$OLD /root/get-pip$OLD.py
+    done
+    for VER in 3.7 3.8 3.9 3.10 3.11; do
+        python$VER /root/get-pip.py
+    done
+
+    cd /root
+    if [ -d RWA-python ]; then
+        cd RWA-python
+        git pull || true
+    else
+        git clone git://github.com/DecBayComp/RWA-python -b dev
+        cd RWA-python
+    fi
+
+    export CC=mpicc
+    export HDF5_MPI="ON"
+    export HDF5_LIB=/usr/lib/x86_64-linux-gnu
+    cur=`pwd`; cd $HDF5_LIB
+    ln -s libhdf5_openmpi.so libhdf5.so
+    cd $cur
+
+    export LC_ALL=C
+    for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11; do
+        python="python${version}"
+        pip="$python -m pip"
+        pip_install="$pip install -U"
+        if [ "$version" = "3.6" -o "$version" = "3.7" -o "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then
+            $pip_install --force-reinstall setuptools
+            $pip_install pytest
+        fi
+        if [ "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then
+            $pip_install mpi4py
+            #$pip_install --no-binary=h5py h5py
+        fi
+
+        #$pip uninstall -qy rwa-python || true
+        $pip_install . -r requirements.txt
+
+        for pkg in scipy pandas; do
+            if [ -z "$($pip show $pkg)" ]; then
+                $pip_install $pkg
+            fi
+        done
+    done
+
+    mkdir -p /pasteur
+
+%runscript
+
+    python="python2.7"
+    if [ -n "$1" ]; then
+        if [ "$1" = "-2" -o "$1" = "-27" ]; then
+            # nothing to do
+            shift
+        elif [ "$1" = "-3" -o "$1" = "-35" ]; then
+            python="python3.5"
+            shift
+        elif [ "$1" = "-36" ]; then
+            python="python3.6"
+            shift
+        elif [ "$1" = "-37" ]; then
+            python="python3.7"
+            shift
+        elif [ "$1" = "-38" ]; then
+            python="python3.8"
+            shift
+        elif [ "$1" = "-39" ]; then
+            python="python3.9"
+            shift
+        elif [ "$1" = "-310" ]; then
+            python="python3.10"
+            shift
+        elif [ "$1" = "-311" ]; then
+            python="python3.11"
+            shift
+        fi
+    fi
+    exec $python $@
+

containers/rwa-openmpi-dev

@@ -10,6 +10,7 @@ RWA-python is available in multiple Python environments:
     python3.8 (alias python3)
     python3.9
     python3.10
+    python3.11
 The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6.
 
 %setup
@@ -18,7 +19,7 @@ The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6.
     for OLD in 2.7 3.5 3.6; do
         if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py ]; then
             wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py
-	    mv ${SINGULARITY_ROOTFS}/root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py
+            mv ${SINGULARITY_ROOTFS}/root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py
         fi
     done
     if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip.py ]; then
@@ -51,18 +52,19 @@ The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6.
     apt-get install -y --no-install-recommends \
         python2.7 python2.7-dev \
         python3.5 python3.5-dev \
-        python3.6 python3.6-dev \
-        python3.7 python3.7-dev \
+        python3.6 python3.6-dev python3.6-distutils \
+        python3.7 python3.7-dev python3.7-distutils \
         python3.8 python3.8-dev python3.8-venv \
         python3.9 python3.9-dev \
-        python3.10 python3.10-dev python3.10-venv
+        python3.10 python3.10-dev python3.10-venv \
+        python3.11 python3.11-dev python3.11-venv
 
     # python 3.8 and 3.10 won't work with get-pip.py without package venv
 
     for OLD in 2.7 3.5 3.6; do
         python$OLD /root/get-pip$OLD.py
     done
-    for VER in 3.7 3.8 3.9 3.10; do
+    for VER in 3.7 3.8 3.9 3.10 3.11; do
         python$VER /root/get-pip.py
     done
 
@@ -71,7 +73,7 @@ The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6.
         cd RWA-python
         git pull || true
     else
-        git clone git://github.com/DecBayComp/RWA-python -b master
+        git clone git://github.com/DecBayComp/RWA-python -b dev
         cd RWA-python
     fi
 
@@ -83,16 +85,15 @@ The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6.
     cd $cur
 
     export LC_ALL=C
-    for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10; do
+    for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11; do
         python="python${version}"
-        pip_install="$python -m pip install -U"
-        if [ "$version" = "3.6" -o "$version" = "3.7" -o "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" ]; then
+        pip="$python -m pip"
+        pip_install="$pip install -U"
+        if [ "$version" = "3.6" -o "$version" = "3.7" -o "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then
             $pip_install --force-reinstall setuptools
             $pip_install pytest
-        else
-            : #$pip install --upgrade pip
         fi
-        if [ "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" ]; then
+        if [ "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then
             $pip_install mpi4py
             #$pip_install --no-binary=h5py h5py
         fi
@@ -101,7 +102,7 @@ The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6.
         $pip_install . -r requirements.txt
 
         for pkg in scipy pandas; do
-            if [ -z "$($pip show $pkg)" ]; then
+            if [ -z "$($pip show -q $pkg)" ]; then
                 $pip_install $pkg
             fi
         done
@@ -134,6 +135,9 @@ The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6.
         elif [ "$1" = "-310" ]; then
             python="python3.10"
             shift
+        elif [ "$1" = "-311" ]; then
+            python="python3.11"
+            shift
         fi
     fi
     exec $python $@

pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "rwa"
+version = "0.9.2"
+description = "HDF5-based serialization library for Python datatypes"
+authors = ["François Laurent <francois.laurent@pasteur.fr>"]
+license = "Apache 2.0"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8,<3.12"
+six = "^1.16.0"
+numpy = "^1.24.3"
+scipy = "^1.10.1"
+pandas = "^2.0.2"
+h5py = "^3.8.0"
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.3.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"