From 7c8420b4079cd95029c816f3eeb767b8c903e227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Wed, 11 Feb 2026 20:09:49 +0100 Subject: [PATCH 1/3] python3.14 and podman instead of apptainer --- .gitignore | 2 + containers/Containerfile | 126 +++++++++++++++++++++++++++++++++++++++ tests/multi-py_compat.sh | 90 +++++++++++++++++----------- tests/test_pandas.py | 2 +- 4 files changed, 184 insertions(+), 36 deletions(-) create mode 100644 containers/Containerfile diff --git a/.gitignore b/.gitignore index e64249a..f78e2be 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ __pycache__/ dist/ build/ _build/ +poetry.lock +uv.lock # anything hidden should be manually added .* diff --git a/containers/Containerfile b/containers/Containerfile new file mode 100644 index 0000000..3149ee9 --- /dev/null +++ b/containers/Containerfile @@ -0,0 +1,126 @@ +FROM ubuntu:22.04 + +COPY . /root/RWA-python + +ENV PYENV_ROOT=/root/.pyenv \ + CC=mpicc \ + HDF5_MPI="ON" \ + HDF5_LIB=/usr/lib/x86_64-linux-gnu \ + LC_ALL=C + +RUN cd /root/RWA-python \ + && rm -rf .git .github .pytest_cache .venv containers dist doc tests .gitignore .readthedocs.yml meta.yaml poetry.lock pyproject.toml uv.lock rwa/__pycache__ \ + && ls -a + +COPY <<"EOT" /bin/entrypoint +#!/bin/bash +python="python3" +if [ -n "$1" ]; then + if [ "$1" = "-2" -o "$1" = "-27" ]; then + python="python2.7" + shift + elif [ "$1" = "-35" ]; then + python="/root/python3.5" + shift + elif [ "$1" = "-36" ]; then + python="/root/python3.6" + shift + elif [ "$1" = "-37" ]; then + python="python3.7" + shift + elif [ "$1" = "-38" ]; then + python="python3.8" + shift + elif [ "$1" = "-39" ]; then + python="python3.9" + shift + elif [ "$1" = "-3" -o "$1" = "-310" ]; then + python="python3.10" + shift + elif [ "$1" = "-311" ]; then + python="python3.11" + shift + elif [ "$1" = "-312" ]; then + python="python3.12" + shift + elif [ "$1" = "-313" ]; then + python="python3.13" + shift + elif [ "$1" = "-314" ]; then + python="python3.14" + shift + fi +fi +# echo "running: $python $@" >&2 +exec $python "$@" +EOT + +RUN chmod +x /bin/entrypoint \ + && ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime \ + && apt-get update -y \ + && apt-get install -y --no-install-recommends locales \ + && locale-gen \ + && apt-get install -y --no-install-recommends libhdf5-openmpi-dev \ + build-essential git software-properties-common pkg-config \ + gpg-agent wget curl libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils \ + tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ + && curl -fsSL https://pyenv.run | bash \ + && for ver in 3.5 3.6; do \ + /root/.pyenv/bin/pyenv install $ver \ + && echo -e "#!/bin/bash\nPYTHON=\$(ls /root/.pyenv/versions/$ver*/bin/python)\neval \"\$PYTHON \$@\"" >/root/python$ver \ + && chmod +x /root/python$ver; \ + done; \ + add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get update -y \ + && apt-get install -y --no-install-recommends \ + python2.7 python2.7-dev \ + python3.7 python3.7-dev python3.7-distutils \ + python3.8 python3.8-dev python3.8-venv \ + python3.9 python3.9-dev python3.9-distutils \ + python3.10-dev python3.10-venv \ + python3.11 python3.11-dev python3.11-venv \ + python3.12 python3.12-dev python3.12-venv \ + python3.13 python3.13-dev python3.13-venv \ + python3.14 python3.14-dev python3.14-venv \ + && for old in 2.7 3.5 3.6 3.7 3.8; do \ + ([ -f /root/get-pip$old.py ] \ + || wget -O /root/get-pip$old.py "https://bootstrap.pypa.io/pip/$old/get-pip.py") \ + && (command -v python$old >/dev/null \ + || python$old /root/get-pip$old.py --root-user-action=ignore); \ + done; \ + ([ -f /root/get-pip.py ] \ + || wget -P /root/ -- 'https://bootstrap.pypa.io/get-pip.py') \ + && for VER in 3.9 3.10 3.11 3.12 3.13 3.14; do \ + python$VER /root/get-pip.py --root-user-action=ignore; \ + done; \ + (curr=`pwd`; cd "$HDF5_LIB"; ln -s libhdf5_openmpi.so libhdf5.so; cd "$curr") \ + && cd /root/RWA-python \ + && for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do \ + python="python$version"; \ + if [ "$version" = "2.7" ] || [ "$version" = "3.5" ] || [ "$version" = "3.6" ] || [ "$version" = "3.7" ] || \ + [ "$version" = "3.8" ]; then \ + python="/root/$python"; \ + if [ ! -f "$python" ]; then continue; fi \ + elif ! command -v $python >/dev/null; then continue; fi; \ + pip="$python -m pip"; \ + pip_install="$pip install --root-user-action=ignore -U"; \ + if [ "$version" = "3.6" ] || [ "$version" = "3.7" ] || [ "$version" = "3.8" ] || [ "$version" = "3.9" ] || \ + [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || [ "$version" = "3.12" ] || [ "$version" = "3.13" ] || \ + [ "$version" = "3.14" ]; then \ + $pip_install --force-reinstall setuptools; \ + $pip_install pytest; \ + fi; \ + if [ "$version" = "3.8" ] || [ "$version" = "3.9" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || \ + [ "$version" = "3.12" ] || [ "$version" = "3.13" ] || [ "$version" = "3.14" ]; then \ + $pip_install mpi4py; \ + fi; \ + $pip_install . -r requirements.txt; \ + for pkg in scipy pandas; do \ + if [ -z "$($pip show $pkg)" ]; then \ + $pip_install $pkg; \ + fi; \ + done; \ + done + +ENTRYPOINT ["/bin/entrypoint"] diff --git a/tests/multi-py_compat.sh b/tests/multi-py_compat.sh index 9416205..a122a25 100755 --- a/tests/multi-py_compat.sh +++ b/tests/multi-py_compat.sh @@ -1,32 +1,39 @@ #!/bin/sh -versions="3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13" - -if [ "$(pwd | rev | cut -d/ -f1 | rev)" = "tests" ]; then - container="$(pwd)/../containers/rwa-openmpi-dev.sif" -elif [ -d "tests" ]; then - container="$(pwd)/containers/rwa-openmpi-dev.sif" - cd tests -else - echo "Please run $0 in the tests directory" - exit 1 -fi +set -e +versions="2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14" -if ! [ -f "$container" -o -h "$container" ]; then - cd ../containers # if this crashes, $0 is not run from the tests directory as it should be - echo "No container found; building one..." - if command -v apptainer &>/dev/null; then - echo "apptainer build rwa-openmpi-dev.sif rwa-jammy" - apptainer build rwa-openmpi-dev.sif rwa-jammy || exit - else - echo "singularity build --fakeroot rwa-openmpi-dev.sif rwa-focal" - singularity build --fakeroot rwa-openmpi-dev.sif rwa-focal || exit - fi - echo "======================================" - echo "Container ready; starting the tests..." - echo "======================================" - cd ../tests +if [ -f "tests/multi-py_compat.sh" ]; then + cd tests +elif ! [ -f "multi-py_compat.sh" ]; then + echo "Please run $0 in the tests directory" >&2 + exit 1 +fi +# clean up after failed test runs +rm -rf tmp.* + +if command -v podman >/dev/null; then + if [ -z "`podman images | grep localhost/rwa-python`" ]; then + podman build -t rwa-python -f ../containers/Containerfile .. + fi + run() { + local ver=$1 path=$2 + shift 2 + local file=`basename "$path"` dir=`dirname "$path"` + podman run -v "$dir":/data --security-opt label=disable --rm rwa-python "$ver" "/data/$file" "$@" + } +elif command -v apptainer >/dev/null; then + container=`realpath ../containers/rwa-jammy.sif` + if [ ! -f "$container" ] && [ ! -h "$container" ]; then + (cd ../containers; apptainer build rwa-jammy.sif rwa-jammy) + fi + run() { + apptainer run $container "$@" + } +else + echo "No container engines found" >&2 + exit 1 fi tmpdir=$(mktemp -d -p .) || exit @@ -37,11 +44,14 @@ peek_script=$(mktemp -p $tmpdir) || exit trap "rm -f -- '$hdf_file' '$poke_script' '$peek_script'" EXIT cat < $poke_script +import os +file = os.path.join(os.path.dirname(os.path.abspath(__file__)), '`basename "$hdf5_file"`') + from rwa import * from numpy.random import rand from scipy.spatial import * from pandas import Series -store = HDF5Store('$hdf5_file', 'w')#, verbose=True) +store = HDF5Store(file, 'w')#, verbose=True) store.poke('range', range(4)) store.poke('delaunay', Delaunay(rand(5,2))) @@ -53,8 +63,11 @@ store.close() EOT cat < $peek_script +import os +file = os.path.join(os.path.dirname(os.path.abspath(__file__)), '`basename "$hdf5_file"`') + from rwa import * -store = HDF5Store('$hdf5_file', 'r', verbose=True) +store = HDF5Store(file, 'r', verbose=True) store.peek('range') store.peek('delaunay') @@ -66,25 +79,32 @@ store.close() EOT +i=0 for poke_version in $versions; do -poke_python="singularity run $container -$(echo $poke_version | cut -c1,3-)" +i=$(( i + 1 )) +poke_python="-$(echo $poke_version | cut -c1,3-)" + +j=0 for peek_version in $versions; do +j=$(( j + 1 )) +[ $i -le $j ] || continue + if [ "$poke_version" = "$peek_version" ]; then - continue + continue fi -peek_python="singularity run $container -$(echo $peek_version | cut -c1,3-)" +peek_python="-$(echo $peek_version | cut -c1,3-)" -echo "----\n${poke_version}->${peek_version}\n----" +echo -e "----\n${poke_version}->${peek_version}\n----" -echo -n "poking with Python${poke_version}...\t" -$poke_python $poke_script && echo "[ok]" +echo -ne "poking with Python${poke_version}...\t" +run $poke_python $poke_script && echo "[ok]" -echo -n "peeking with Python${peek_version}...\t" -$peek_python $peek_script && echo "[ok]" +echo -ne "peeking with Python${peek_version}...\t" +run $peek_python $peek_script && echo "[ok]" done done diff --git a/tests/test_pandas.py b/tests/test_pandas.py index 9982404..ce5d51b 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -129,7 +129,7 @@ def test_categorical(self, tmpdir): categories=['c', 'b', 'a'])} if _test_categoricaldtype: data['series_with_categories'] = Series(['a', 'b', 'a', 'c'], dtype=\ - CategoricalDtype(categories=['b', 'a'], ordered=True)) + CategoricalDtype(categories=['c', 'b', 'a'], ordered=True)) # write store = HDF5Store(test_file, 'w') try: From 6bfbc3023ee1aaf62da4b1ba9f0fe8d857240337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 13 Feb 2026 18:04:53 +0100 Subject: [PATCH 2/3] fix cross-version pandas.Series --- .github/workflows/ci.yml | 4 +- README.md | 2 +- README.rst | 2 +- containers/Containerfile | 19 +++-- containers/Containerfile-legacy | 121 ++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- rwa/hdf5.py | 4 ++ rwa/pandas.py | 7 +- rwa/storable.py | 9 +++ tests/multi-py_compat.sh | 10 ++- 10 files changed, 164 insertions(+), 16 deletions(-) create mode 100644 containers/Containerfile-legacy diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf61fc3..4366dda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ name: Unit tests on: push: - branches: [ master ] + branches: [ master, dev ] pull_request: branches: [ master ] @@ -20,7 +20,7 @@ jobs: - ubuntu-latest - macos-latest - windows-latest - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] fail-fast: true steps: diff --git a/README.md b/README.md index c709f3d..6a4fa5e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ In addition, string typing is sometimes problematic. Non-ascii characters should ## Installation -Python >= 3.5 is required. **RWA-python** may still work with Python 2.7 but support has been dropped. +Python >= 3.6 is required. **RWA-python** may still work with Python 2.7 and 3.5 but support has been dropped. Windows users should favor Conda for installing **RWA-python**, as Conda will seamlessly install the HDF5 standard library which is a required dependency. diff --git a/README.rst b/README.rst index 8ee4d75..ffe0e3c 100644 --- a/README.rst +++ b/README.rst @@ -64,7 +64,7 @@ In addition, string typing is sometimes problematic. Non-ascii characters should Installation ------------ -Python >= 3.5 is required. **RWA-python** may still work with Python 2.7 but support has been dropped. +Python >= 3.6 is required. **RWA-python** may still work with Python 2.7 and 3.5 but support has been dropped. Windows users should favor Conda for installing **RWA-python**, as Conda will seamlessly install the HDF5 standard library which is a required dependency. diff --git a/containers/Containerfile b/containers/Containerfile index 3149ee9..735e31d 100644 --- a/containers/Containerfile +++ b/containers/Containerfile @@ -83,11 +83,17 @@ RUN chmod +x /bin/entrypoint \ python3.12 python3.12-dev python3.12-venv \ python3.13 python3.13-dev python3.13-venv \ python3.14 python3.14-dev python3.14-venv \ - && for old in 2.7 3.5 3.6 3.7 3.8; do \ - ([ -f /root/get-pip$old.py ] \ - || wget -O /root/get-pip$old.py "https://bootstrap.pypa.io/pip/$old/get-pip.py") \ - && (command -v python$old >/dev/null \ - || python$old /root/get-pip$old.py --root-user-action=ignore); \ + && for very_old in 2.7 3.5 3.6; do \ + (command -v python$very_old >/dev/null \ + && ([ -f /root/get-pip$very_old.py ] \ + || wget -O /root/get-pip$very_old.py "https://bootstrap.pypa.io/pip/$very_old/get-pip.py") \ + && python$very_old /root/get-pip$very_old.py); \ + done; \ + for old in 3.7 3.8; do \ + (command -v python$old >/dev/null \ + && ([ -f /root/get-pip$old.py ] \ + || wget -O /root/get-pip$old.py "https://bootstrap.pypa.io/pip/$old/get-pip.py") \ + && python$old /root/get-pip$old.py --root-user-action=ignore); \ done; \ ([ -f /root/get-pip.py ] \ || wget -P /root/ -- 'https://bootstrap.pypa.io/get-pip.py') \ @@ -98,8 +104,7 @@ RUN chmod +x /bin/entrypoint \ && cd /root/RWA-python \ && for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do \ python="python$version"; \ - if [ "$version" = "2.7" ] || [ "$version" = "3.5" ] || [ "$version" = "3.6" ] || [ "$version" = "3.7" ] || \ - [ "$version" = "3.8" ]; then \ + if [ "$version" = "2.7" ] || [ "$version" = "3.5" ] || [ "$version" = "3.6" ]; then \ python="/root/$python"; \ if [ ! -f "$python" ]; then continue; fi \ elif ! command -v $python >/dev/null; then continue; fi; \ diff --git a/containers/Containerfile-legacy b/containers/Containerfile-legacy new file mode 100644 index 0000000..ac2c9e8 --- /dev/null +++ b/containers/Containerfile-legacy @@ -0,0 +1,121 @@ +FROM ubuntu:18.04 + +COPY . /root/RWA-python + +ENV PYENV_ROOT=/root/.pyenv \ + CC=mpicc \ + HDF5_MPI="ON" \ + HDF5_LIB=/usr/lib/x86_64-linux-gnu \ + LC_ALL=C + +RUN cd /root/RWA-python \ + && rm -rf .git .github .pytest_cache .venv containers dist doc tests .gitignore .readthedocs.yml meta.yaml poetry.lock pyproject.toml uv.lock rwa/__pycache__ \ + && ls -a + +COPY <<"EOT" /bin/entrypoint +#!/bin/bash +python="python3" +if [ -n "$1" ]; then + if [ "$1" = "-2" -o "$1" = "-27" ]; then + python="python2.7" + shift + elif [ "$1" = "-35" ]; then + python="python3.5" + shift + elif [ "$1" = "-36" ]; then + python="python3.6" + shift + elif [ "$1" = "-37" ]; then + python="python3.7" + shift + elif [ "$1" = "-3" -o "$1" = "-38" ]; then + python="python3.8" + shift + elif [ "$1" = "-39" ]; then + python="python3.9" + shift + elif [ "$1" = "-310" ]; then + python="python3.10" + shift + elif [ "$1" = "-311" ]; then + python="python3.11" + shift + elif [ "$1" = "-312" ]; then + python="python3.12" + shift + elif [ "$1" = "-313" ]; then + python="python3.13" + shift + elif [ "$1" = "-314" ]; then + python="python3.14" + shift + fi +fi +# echo "running: $python $@" >&2 +exec $python "$@" +EOT + +RUN chmod +x /bin/entrypoint \ + && ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime \ + && apt-get update -y \ + && apt-get install -y --no-install-recommends locales \ + && locale-gen \ + && apt-get install -y --no-install-recommends libhdf5-openmpi-dev \ + build-essential git software-properties-common pkg-config \ + gpg-agent wget curl libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils \ + tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get update -y \ + && apt-get install -y --no-install-recommends \ + python2.7 python2.7-dev \ + python3.6 python3.6-dev python3.6-distutils \ + python3.7 python3.7-dev python3.7-distutils \ + python3.8 python3.8-dev python3.8-venv \ + && for very_old in 2.7 3.5 3.6; do \ + (command -v python$very_old >/dev/null \ + && ([ -f /root/get-pip$very_old.py ] \ + || wget -O /root/get-pip$very_old.py "https://bootstrap.pypa.io/pip/$very_old/get-pip.py") \ + && python$very_old /root/get-pip$very_old.py); \ + done; \ + for old in 3.7 3.8; do \ + (command -v python$old >/dev/null \ + && ([ -f /root/get-pip$old.py ] \ + || wget -O /root/get-pip$old.py "https://bootstrap.pypa.io/pip/$old/get-pip.py") \ + && python$old /root/get-pip$old.py --root-user-action=ignore); \ + done; \ + ([ -f /root/get-pip.py ] \ + || wget -P /root/ -- 'https://bootstrap.pypa.io/get-pip.py') \ + && for VER in 3.9 3.10 3.11 3.12 3.13 3.14; do \ + (command -v python$VER >/dev/null && python$VER /root/get-pip.py --root-user-action=ignore); \ + done; \ + (curr=`pwd`; cd "$HDF5_LIB"; ln -s libhdf5_openmpi.so libhdf5.so; cd "$curr") \ + && cd /root/RWA-python \ + && for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do \ + python="python$version"; \ + if ! command -v $python >/dev/null; then continue; fi; \ + pip="$python -m pip"; \ + if [ "$version" = "2.7" ] || [ "$version" = "3.6" ]; then \ + pip_install="$pip install -U"; \ + else \ + pip_install="$pip install --root-user-action=ignore -U"; \ + fi; \ + if [ "$version" = "3.6" ] || [ "$version" = "3.7" ] || [ "$version" = "3.8" ] || [ "$version" = "3.9" ] || \ + [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || [ "$version" = "3.12" ] || [ "$version" = "3.13" ] || \ + [ "$version" = "3.14" ]; then \ + $pip_install --force-reinstall setuptools; \ + $pip_install pytest; \ + fi; \ + if [ "$version" = "3.8" ] || [ "$version" = "3.9" ] || [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || \ + [ "$version" = "3.12" ] || [ "$version" = "3.13" ] || [ "$version" = "3.14" ]; then \ + $pip_install mpi4py; \ + fi; \ + $pip_install . -r requirements.txt; \ + for pkg in scipy pandas; do \ + if [ -z "$($pip show $pkg)" ]; then \ + $pip_install $pkg; \ + fi; \ + done; \ + done + +ENTRYPOINT ["/bin/entrypoint"] diff --git a/pyproject.toml b/pyproject.toml index a82b1aa..416145b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "rwa-python" -version = "0.9.5" +version = "0.10" description = "HDF5-based serialization library for Python datatypes" authors = ["François Laurent "] license = "Apache 2.0" diff --git a/rwa/hdf5.py b/rwa/hdf5.py index 7a1a735..0d0fb52 100644 --- a/rwa/hdf5.py +++ b/rwa/hdf5.py @@ -266,6 +266,7 @@ def peek_array(self, store, elemtype, container, _stack): spatial_storables, \ pandas_storables)) +hdf5_aliases = pandas_aliases # global variable @@ -273,6 +274,9 @@ def peek_array(self, store, elemtype, container, _stack): for s in hdf5_storables: hdf5_service.registerStorable(s) +for s in hdf5_aliases: + hdf5_service.registerAlias(*s) + def hdf5_storable(type_or_storable, *args, **kwargs): '''Registers a `Storable` instance in the global service.''' if not isinstance(type_or_storable, Storable): diff --git a/rwa/pandas.py b/rwa/pandas.py index 6ba0d03..b6ddabc 100644 --- a/rwa/pandas.py +++ b/rwa/pandas.py @@ -16,6 +16,7 @@ class Python35Warning(DeprecationWarning): import pandas except ImportError: pandas_storables = [] + pandas_aliases = [] else: rwa_params['pandas.index.force_unicode'] = True rwa_params['pandas.columns.force_unicode'] = True @@ -336,8 +337,12 @@ def peek_dataframe(service, container, _stack=None, force_unicode=None): return df pandas_storables += [ \ - Storable(pandas.Series, handlers=StorableHandler(poke=poke_series, peek=peek_series)), \ + Storable(pandas.Series, \ + key='Python.pandas.core.series.Series', \ + handlers=StorableHandler(poke=poke_series, peek=peek_series)), \ Storable(pandas.DataFrame, \ handlers=StorableHandler(poke=poke_dataframe, peek=peek_dataframe, \ peek_option='pandas.columns.force_unicode'))] + pandas_aliases = [('Python.pandas.core.series.Series', 'Python.pandas.Series')] + diff --git a/rwa/storable.py b/rwa/storable.py index d7fe792..f9a3809 100644 --- a/rwa/storable.py +++ b/rwa/storable.py @@ -306,6 +306,15 @@ def registerStorable(self, storable, replace=False, agnostic=False, deprivatize= self.by_python_type[storable.python_type] = existing self.by_storable_type[storable.storable_type] = existing + def registerAlias(self, existing_key, alias): + if existing_key in self.by_storable_type.keys(): + if alias in self.by_storable_type.keys(): + raise KeyError('storable type already registered', alias) + else: + self.by_storable_type[alias] = self.by_storable_type[existing_key] + else: + raise KeyError('storable type not found', existing_key) + def byPythonType(self, t, istype=False): if istype:#isinstance(t, type): try: diff --git a/tests/multi-py_compat.sh b/tests/multi-py_compat.sh index a122a25..2bfd125 100755 --- a/tests/multi-py_compat.sh +++ b/tests/multi-py_compat.sh @@ -2,7 +2,7 @@ set -e -versions="2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14" +versions="3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14" if [ -f "tests/multi-py_compat.sh" ]; then cd tests @@ -16,12 +16,16 @@ rm -rf tmp.* if command -v podman >/dev/null; then if [ -z "`podman images | grep localhost/rwa-python`" ]; then podman build -t rwa-python -f ../containers/Containerfile .. + podman build -t rwa-python:legacy -f ../containers/Containerfile-legacy .. fi run() { - local ver=$1 path=$2 + local ver=$1 path=$2 tag= shift 2 local file=`basename "$path"` dir=`dirname "$path"` - podman run -v "$dir":/data --security-opt label=disable --rm rwa-python "$ver" "/data/$file" "$@" + if [ "$ver" = "-27" ] || [ "$ver" = "-35" ] || [ "$ver" = "-36" ]; then + tag=":legacy" + fi + podman run -v "$dir":/data --security-opt label=disable --rm rwa-python$tag "$ver" "/data/$file" "$@" } elif command -v apptainer >/dev/null; then container=`realpath ../containers/rwa-jammy.sif` From 3768e02f526e01c04bd005dd0f36c5a224f454f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Laurent?= Date: Fri, 13 Feb 2026 18:26:30 +0100 Subject: [PATCH 3/3] clean up --- containers/Containerfile | 13 +-- containers/rwa-focal | 1 - containers/rwa-jammy | 174 ------------------------------------- containers/rwa-openmpi-dev | 144 ------------------------------ 4 files changed, 3 insertions(+), 329 deletions(-) delete mode 120000 containers/rwa-focal delete mode 100644 containers/rwa-jammy delete mode 100644 containers/rwa-openmpi-dev diff --git a/containers/Containerfile b/containers/Containerfile index 735e31d..66a48aa 100644 --- a/containers/Containerfile +++ b/containers/Containerfile @@ -74,7 +74,6 @@ RUN chmod +x /bin/entrypoint \ add-apt-repository -y ppa:deadsnakes/ppa \ && apt-get update -y \ && apt-get install -y --no-install-recommends \ - python2.7 python2.7-dev \ python3.7 python3.7-dev python3.7-distutils \ python3.8 python3.8-dev python3.8-venv \ python3.9 python3.9-dev python3.9-distutils \ @@ -83,13 +82,7 @@ RUN chmod +x /bin/entrypoint \ python3.12 python3.12-dev python3.12-venv \ python3.13 python3.13-dev python3.13-venv \ python3.14 python3.14-dev python3.14-venv \ - && for very_old in 2.7 3.5 3.6; do \ - (command -v python$very_old >/dev/null \ - && ([ -f /root/get-pip$very_old.py ] \ - || wget -O /root/get-pip$very_old.py "https://bootstrap.pypa.io/pip/$very_old/get-pip.py") \ - && python$very_old /root/get-pip$very_old.py); \ - done; \ - for old in 3.7 3.8; do \ + && for old in 3.7 3.8; do \ (command -v python$old >/dev/null \ && ([ -f /root/get-pip$old.py ] \ || wget -O /root/get-pip$old.py "https://bootstrap.pypa.io/pip/$old/get-pip.py") \ @@ -106,13 +99,13 @@ RUN chmod +x /bin/entrypoint \ python="python$version"; \ if [ "$version" = "2.7" ] || [ "$version" = "3.5" ] || [ "$version" = "3.6" ]; then \ python="/root/$python"; \ - if [ ! -f "$python" ]; then continue; fi \ + if [ ! -f "$python" ]; then continue; fi \ elif ! command -v $python >/dev/null; then continue; fi; \ pip="$python -m pip"; \ pip_install="$pip install --root-user-action=ignore -U"; \ if [ "$version" = "3.6" ] || [ "$version" = "3.7" ] || [ "$version" = "3.8" ] || [ "$version" = "3.9" ] || \ [ "$version" = "3.10" ] || [ "$version" = "3.11" ] || [ "$version" = "3.12" ] || [ "$version" = "3.13" ] || \ - [ "$version" = "3.14" ]; then \ + [ "$version" = "3.14" ]; then \ $pip_install --force-reinstall setuptools; \ $pip_install pytest; \ fi; \ diff --git a/containers/rwa-focal b/containers/rwa-focal deleted file mode 120000 index acfd8f7..0000000 --- a/containers/rwa-focal +++ /dev/null @@ -1 +0,0 @@ -rwa-openmpi-dev \ No newline at end of file diff --git a/containers/rwa-jammy b/containers/rwa-jammy deleted file mode 100644 index caf7e5c..0000000 --- a/containers/rwa-jammy +++ /dev/null @@ -1,174 +0,0 @@ -Bootstrap: docker -From: ubuntu:22.04 - -%help -RWA-python is available in multiple Python environments: - python2.7 (alias python2) - python3.5 - python3.6 - python3.7 - python3.8 - python3.9 - python3.10 (alias python3) - python3.11 - python3.12 - python3.13 -The container OS is Ubuntu Jammy. - -%setup - - echo "fr_FR.UTF-8 UTF-8" > ${SINGULARITY_ROOTFS}/etc/locale.gen - - # test local changes that have not been committed yet; - # to be run from any subdirectory in the RWA-python directory, e.g. containers, tests... - LOCAL="$(pwd)/.." - CONTAINED=${SINGULARITY_ROOTFS}/root/RWA-python - mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ \ - ${LOCAL}/setup.py ${LOCAL}/requirements.txt \ - ${LOCAL}/README.rst ${LOCAL}/LICENSE - cp -ru -t ${CONTAINED}/ ${LOCAL}/rwa ${LOCAL}/.git - cd ${CONTAINED} && git clean -xfd - -%post - - ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime - apt-get update -y - apt-get install -y --no-install-recommends locales - locale-gen - - apt-get install -y --no-install-recommends libhdf5-openmpi-dev \ - build-essential git software-properties-common pkg-config \ - gpg-agent wget curl libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils \ - tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev - - curl -fsSL https://pyenv.run | bash #&>/dev/null - export PYENV_ROOT=/root/.pyenv - for VER in 3.5 3.6; do - /root/.pyenv/bin/pyenv install $VER - cat </root/python$VER -#!/bin/bash -PYTHON=\$(ls /root/.pyenv/versions/$VER*/bin/python) -eval "\$PYTHON \$@" -EOF - chmod a+x /root/python$VER - done - - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update -y - apt-get install -y --no-install-recommends \ - python2.7 python2.7-dev \ - python3.7 python3.7-dev python3.7-distutils \ - python3.8 python3.8-dev python3.8-venv \ - python3.9 python3.9-dev python3.9-distutils \ - python3.10-dev python3.10-venv \ - python3.11 python3.11-dev python3.11-venv \ - python3.12 python3.12-dev python3.12-venv \ - python3.13 python3.13-dev python3.13-venv - #python3.5 python3.5-dev \ - #python3.6 python3.6-dev python3.6-distutils \ - - # python 3.8 and 3.10 won't work with get-pip.py without package venv - - for OLD in 2.7 3.7; do - if ! [ -f /root/get-pip$OLD.py ]; then - wget -P /root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py - mv /root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py - fi - if command -v python$OLD &>/dev/null; then - python$OLD /root/get-pip$OLD.py - fi - done - if ! [ -f /root/get-pip.py ]; then - wget -P /root/ -- https://bootstrap.pypa.io/get-pip.py - fi - for VER in 3.8 3.9 3.10 3.11 3.12 3.13; do - python$VER /root/get-pip.py - done - - cd /root - if [ -d RWA-python ]; then - cd RWA-python - git pull || true - else - git clone git://github.com/DecBayComp/RWA-python -b dev - cd RWA-python - fi - - export CC=mpicc - export HDF5_MPI="ON" - export HDF5_LIB=/usr/lib/x86_64-linux-gnu - cur=`pwd`; cd $HDF5_LIB - ln -s libhdf5_openmpi.so libhdf5.so - cd $cur - - export LC_ALL=C - for version in 2.7 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13; do - python="python${version}" - if [ "$version" = "3.5" -o "$version" = "3.6" ]; then - python="/root/$python" - elif ! command -v $python $>/dev/null; then - continue - fi - pip="$python -m pip" - pip_install="$pip install -U" - if [ "$version" = "3.6" -o "$version" = "3.7" -o "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" -o "$version" = "3.12" -o "$version" = "3.13" ]; then - $pip_install --force-reinstall setuptools - $pip_install pytest - fi - if [ "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" -o "$version" = "3.12" -o "$version" = "3.13" ]; then - $pip_install mpi4py - #$pip_install --no-binary=h5py h5py - fi - - #$pip uninstall -qy rwa-python || true - $pip_install . -r requirements.txt - - for pkg in scipy pandas; do - if [ -z "$($pip show $pkg)" ]; then - $pip_install $pkg - fi - done - done - - mkdir -p /pasteur - -%runscript - - python="python3" - if [ -n "$1" ]; then - if [ "$1" = "-2" -o "$1" = "-27" ]; then - python="python2.7" - shift - elif [ "$1" = "-35" ]; then - python="/root/python3.5" - shift - elif [ "$1" = "-36" ]; then - python="/root/python3.6" - shift - elif [ "$1" = "-37" ]; then - python="python3.7" - shift - elif [ "$1" = "-38" ]; then - python="python3.8" - shift - elif [ "$1" = "-39" ]; then - python="python3.9" - shift - elif [ "$1" = "-3" -o "$1" = "-310" ]; then - python="python3.10" - shift - elif [ "$1" = "-311" ]; then - python="python3.11" - shift - elif [ "$1" = "-312" ]; then - python="python3.12" - shift - elif [ "$1" = "-313" ]; then - python="python3.13" - shift - fi - fi - exec $python $@ - diff --git a/containers/rwa-openmpi-dev b/containers/rwa-openmpi-dev deleted file mode 100644 index 7b0e652..0000000 --- a/containers/rwa-openmpi-dev +++ /dev/null @@ -1,144 +0,0 @@ -Bootstrap: docker -From: ubuntu:focal - -%help -RWA-python is available in multiple Python environments: - python2.7 (alias python2) - python3.5 - python3.6 - python3.7 - python3.8 (alias python3) - python3.9 - python3.10 - python3.11 -The container OS is Ubuntu Focal and can run on top of old OSes like CentOS6. - -%setup - - echo "fr_FR.UTF-8 UTF-8" > ${SINGULARITY_ROOTFS}/etc/locale.gen - for OLD in 2.7 3.5 3.6; do - if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py ]; then - wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/pip/$OLD/get-pip.py - mv ${SINGULARITY_ROOTFS}/root/get-pip.py ${SINGULARITY_ROOTFS}/root/get-pip$OLD.py - fi - done - if ! [ -f ${SINGULARITY_ROOTFS}/root/get-pip.py ]; then - wget -P ${SINGULARITY_ROOTFS}/root/ -- https://bootstrap.pypa.io/get-pip.py - fi - - # test local changes that have not been committed yet; - # to be run from any subdirectory in the RWA-python directory, e.g. containers, tests... - LOCAL="$(pwd)/.." - CONTAINED=${SINGULARITY_ROOTFS}/root/RWA-python - mkdir -p ${CONTAINED} - cp -u -t ${CONTAINED}/ \ - ${LOCAL}/setup.py ${LOCAL}/requirements.txt \ - ${LOCAL}/README.rst ${LOCAL}/LICENSE - cp -ru -t ${CONTAINED}/ ${LOCAL}/rwa ${LOCAL}/.git - cd ${CONTAINED} && git clean -xfd - -%post - - ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime - apt-get update -y - apt-get install -y --no-install-recommends locales - locale-gen - - apt-get install -y --no-install-recommends libhdf5-openmpi-dev \ - build-essential git software-properties-common pkg-config - - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update -y - apt-get install -y --no-install-recommends \ - python2.7 python2.7-dev \ - python3.5 python3.5-dev \ - python3.6 python3.6-dev python3.6-distutils \ - python3.7 python3.7-dev python3.7-distutils \ - python3.8 python3.8-dev python3.8-venv \ - python3.9 python3.9-dev \ - python3.10 python3.10-dev python3.10-venv \ - python3.11 python3.11-dev python3.11-venv - - # python 3.8 and 3.10 won't work with get-pip.py without package venv - - for OLD in 2.7 3.5 3.6; do - python$OLD /root/get-pip$OLD.py - done - for VER in 3.7 3.8 3.9 3.10 3.11; do - python$VER /root/get-pip.py - done - - cd /root - if [ -d RWA-python ]; then - cd RWA-python - git pull || true - else - git clone git://github.com/DecBayComp/RWA-python -b dev - cd RWA-python - fi - - export CC=mpicc - export HDF5_MPI="ON" - export HDF5_LIB=/usr/lib/x86_64-linux-gnu - cur=`pwd`; cd $HDF5_LIB - ln -s libhdf5_openmpi.so libhdf5.so - cd $cur - - export LC_ALL=C - for version in 2.7 3.5 3.6 3.7 3.8 3.9 3.10 3.11; do - python="python${version}" - pip="$python -m pip" - pip_install="$pip install -U" - if [ "$version" = "3.6" -o "$version" = "3.7" -o "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then - $pip_install --force-reinstall setuptools - $pip_install pytest - fi - if [ "$version" = "3.8" -o "$version" = "3.9" -o "$version" = "3.10" -o "$version" = "3.11" ]; then - $pip_install mpi4py - #$pip_install --no-binary=h5py h5py - fi - - #$pip uninstall -qy rwa-python || true - $pip_install . -r requirements.txt - - for pkg in scipy pandas; do - if [ -z "$($pip show -q $pkg)" ]; then - $pip_install $pkg - fi - done - done - - mkdir -p /pasteur - -%runscript - - python="python2.7" - if [ -n "$1" ]; then - if [ "$1" = "-2" -o "$1" = "-27" ]; then - # nothing to do - shift - elif [ "$1" = "-3" -o "$1" = "-35" ]; then - python="python3.5" - shift - elif [ "$1" = "-36" ]; then - python="python3.6" - shift - elif [ "$1" = "-37" ]; then - python="python3.7" - shift - elif [ "$1" = "-38" ]; then - python="python3.8" - shift - elif [ "$1" = "-39" ]; then - python="python3.9" - shift - elif [ "$1" = "-310" ]; then - python="python3.10" - shift - elif [ "$1" = "-311" ]; then - python="python3.11" - shift - fi - fi - exec $python $@ -