openPMD · ax3l · Aug 19, 2023 · Apr 24, 2023 · Apr 24, 2023 · May 3, 2023
diff --git a/.github/workflows/dependencies/install_icc b/.github/workflows/dependencies/install_icc
@@ -7,9 +7,14 @@ sudo apt-get -qqq update
 sudo apt-get install g++
 #                    libopenmpi-dev
 sudo apt-get install -y wget build-essential pkg-config cmake ca-certificates gnupg
-sudo wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB
-echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
+# download the key to system keyring
+wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+
+# add signed entry to apt sources and configure the APT client to use Intel repository
+echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
 sudo apt-get update
 sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
 #                       intel-oneapi-python
diff --git a/.github/workflows/dependencies/install_icx b/.github/workflows/dependencies/install_icx
@@ -5,17 +5,20 @@ set -eu -o pipefail
 
 # Ref.: https://github.com/rscohn2/oneapi-ci
 # intel-basekit intel-hpckit are too large in size
-wget -q -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \
-  | sudo apt-key add -
-echo "deb https://apt.repos.intel.com/oneapi all main" \
-  | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
+# download the key to system keyring
+wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+
+# add signed entry to apt sources and configure the APT client to use Intel repository
+echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
 
 sudo apt-get update
 
 sudo apt-get install -y --no-install-recommends \
     build-essential \
     cmake           \
-    intel-oneapi-dpcpp-cpp-compiler intel-oneapi-mkl-devel \
+    intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \
     g++ gfortran
 #    libopenmpi-dev
 #    openmpi-bin
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -28,10 +28,11 @@ jobs:
         python3 -m pip install -U mpi4py numpy pandas
         set -e
     - name: Build
-      env: {CXXFLAGS: -Werror, MACOSX_DEPLOYMENT_TARGET: 10.15}
+      env: {CXXFLAGS: -Werror, MACOSX_DEPLOYMENT_TARGET: 11.0}
       # 10.14+ due to std::visit
       # 10.15+ due to std::filesystem in toml11
       #   https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions
+      # 11.0+ for arm64/aarch64 (M1/M2) builds
       run: |
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
@@ -43,14 +44,13 @@ jobs:
         cmake --build build --parallel 2
         ctest --test-dir build --verbose
 
-  appleclang12_py_ad1:
-    runs-on: macos-10.15
-    # next: macOS-11
+  appleclang13_py:
+    runs-on: macos-11
     if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v3
     - name: Install
-      env: {MACOSX_DEPLOYMENT_TARGET: 10.14}
+      env: {MACOSX_DEPLOYMENT_TARGET: 11.0}
       run: |
         set +e
         python3 -m pip install -U numpy pandas
@@ -63,10 +63,11 @@ jobs:
         make install
         set -e
     - name: Build
-      env: {CXXFLAGS: -Werror -DTOML11_DISABLE_STD_FILESYSTEM, MACOSX_DEPLOYMENT_TARGET: 10.14}
+      env: {CXXFLAGS: -Werror, MACOSX_DEPLOYMENT_TARGET: 11.0}
       # 10.14+ due to std::visit
-      # std::filesystem in toml11 needs macOS 10.15
+      # 10.15+ due to std::filesystem in toml11
       #   https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions
+      # 11.0+ for arm64/aarch64 (M1/M2) builds
       run: |
         share/openPMD/download_samples.sh build
         cmake -S . -B build \
@@ -75,8 +76,9 @@ jobs:
           -DopenPMD_USE_HDF5=OFF   \
           -DopenPMD_USE_ADIOS1=ON \
           -DopenPMD_USE_ADIOS2=OFF \
-          -DopenPMD_USE_INVASIVE_TESTS=ON
-        cmake --build build --parallel 2
+          -DopenPMD_USE_INVASIVE_TESTS=ON \
+          -DPython_EXECUTABLE=$(which python3)
+        cmake --build build --parallel 3
         ctest --test-dir build --verbose
 
 # TODO: apple_conda_ompi_all (similar to conda_ompi_all on Linux)

diff --git a/.github/workflows/source.yml b/.github/workflows/source.yml
@@ -41,7 +41,7 @@ jobs:
         update-conda: true
         conda-channels: conda-forge
     - name: Install
-      run: conda install -c conda-forge doxygen
+      run: conda install -c conda-forge doxygen=1.9.6
     - name: Doxygen
       run: .github/workflows/source/buildDoxygen
 

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -42,7 +42,7 @@ jobs:
 
         python3.exe -m pip wheel .
         if(!$?) { Exit $LASTEXITCODE }
-        python3.exe -m pip install openPMD_api-0.15.1-cp39-cp39-win_amd64.whl
+        python3.exe -m pip install openPMD_api-0.15.2-cp39-cp39-win_amd64.whl
         if(!$?) { Exit $LASTEXITCODE }
 
         python3.exe -c "import openpmd_api as api; print(api.variants)"

diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -10,5 +10,8 @@ formats:
   - epub
 
 build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
   apt_packages:
     - librsvg2-bin
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -3,6 +3,75 @@
 Changelog
 =========
 
+0.15.2
+------
+**Date:** 2023-08-18
+
+Python, ADIOS2 and HDF5 Fixes
+
+This release fixed regressions in the Python frontend as well as the ADIOS2 and HDF5 backends.
+Supported macOS versions are now 11.0+ and Python versions are 3.8+.
+
+Changes to "0.15.1"
+^^^^^^^^^^^^^^^^^^^
+
+Bug Fixes
+"""""""""
+
+- Don't require unitSI when reading a patch record component #1470
+- Examples:
+
+  - Streaming examples: Set WAN as default transport #1511
+  - Fix types of particle constant records #1316 #1510
+- Python:
+
+  - DataFrame to ASCII: Header of First Column in CSV bug documentation third party #1480 #1501
+  - Update ``__repr__`` method of major objects in openPMD hierarchy #1476
+  - openpmd-pipe: set correct install permissions #1459
+  - Better error message when loading to a buffer with mismatched type #1452
+  - Use lazy imports for dask and pandas #1442
+- ADIOS2:
+
+  - Fixes for variable-based encoding in backends without step support #1484 #1481
+  - Warn on BP5+Blosc in ADIOS2 v2.9 up to patch level 1 #1497
+  - Ensure that a step is always active at write time #1492
+  - Fix gcc9 warning #1429
+- HDF5:
+
+  - Handle unknown datatypes in datasets #1469
+  - Support for float128 on ARM64/PPC64 #1364
+  - Fix Char Type Matching #1433 #1431
+  - Install: Warn and Continue on Empty ``HDF5_VERSION`` in CMake #1512
+- CI:
+
+  - type comparison in openpmd-pipe #1490
+
+Other
+"""""
+
+- Better handling for file extensions #1473 #1471
+- Optional debugging output for ``AbstractIOHandlerImpl::flush()`` #1495
+- Python: 3.8+ #1502
+- CI:
+
+  - macOS 11.0+ #1486 #1446
+  - oneAPI 2023.2.0 #1478
+  - Doxygen 1.9.7 Broken #1464
+- Docs:
+
+  - Analysis with third party data science frameworks #1444
+  - Sphinx Copybutton and Design #1461
+  - Fix small documentation issues after 0.15 release #1440
+  - ``HDF5_DO_MPI_FILE_SYNC`` #1427
+  - OpenMPI-ROMIO/HDF5/Chunking issue #1441
+  - Remove ADIOS2 schema 2021 #1451
+  - Linking to C++ Projects #1445
+  - Fix deprecated APIs in first read/write examples #1435
+  - Update ``.readthedocs.yml`` #1438
+  - Fix Bib Authors #1434
+  - More careful documentation of streaming API #1430
+
+
 0.15.1
 ------
 **Date:** 2023-04-02

diff --git a/CITATION.cff b/CITATION.cff
@@ -25,7 +25,7 @@ contact:
   orcid: https://orcid.org/0000-0003-1943-7141
   email: axelhuebl@lbl.gov
 title: "openPMD-api: C++ & Python API for Scientific I/O with openPMD"
-version: 0.15.1
+version: 0.15.2
 repository-code: https://github.com/openPMD/openPMD-api
 doi: 10.14278/rodare.27
 license: LGPL-3.0-or-later

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,7 +2,7 @@
 #
 cmake_minimum_required(VERSION 3.15.0)
 
-project(openPMD VERSION 0.15.1) # LANGUAGES CXX
+project(openPMD VERSION 0.15.2) # LANGUAGES CXX
 
 # the openPMD "markup"/"schema" standard version
 set(openPMD_STANDARD_VERSION 1.1.0)
@@ -334,10 +334,16 @@ endif()
 # HDF5 checks
 string(CONCAT openPMD_HDF5_STATUS "")
 # version: lower limit
-if(openPMD_HAVE_HDF5 AND HDF5_VERSION VERSION_LESS 1.8.13)
-    string(CONCAT openPMD_HDF5_STATUS
-        "Found HDF5 version ${HDF5_VERSION} is too old. At least "
-        "version 1.8.13 is required.\n")
+if(openPMD_HAVE_HDF5)
+    if(HDF5_VERSION STREQUAL "")
+        message(WARNING "HDF5_VERSION is empty. Now assuming it is 1.8.13 or newer.")
+    else()
+        if(HDF5_VERSION VERSION_LESS 1.8.13)
+            string(CONCAT openPMD_HDF5_STATUS
+                "Found HDF5 version ${HDF5_VERSION} is too old. At least "
+                "version 1.8.13 is required.\n")
+        endif()
+    endif()
 endif()
 # we imply support for parallel I/O if MPI variant is ON
 if(openPMD_HAVE_MPI AND openPMD_HAVE_HDF5
@@ -458,7 +464,7 @@ if(CMAKE_VERSION VERSION_LESS 3.18.0)
     set(_PY_DEV_MODULE Development)
 endif()
 if(openPMD_USE_PYTHON STREQUAL AUTO)
-    find_package(Python 3.7.0 COMPONENTS Interpreter ${_PY_DEV_MODULE})
+    find_package(Python 3.8.0 COMPONENTS Interpreter ${_PY_DEV_MODULE})
     if(Python_FOUND)
         if(openPMD_USE_INTERNAL_PYBIND11)
             add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")
@@ -546,6 +552,7 @@ set(IO_ADIOS1_SEQUENTIAL_SOURCE
         src/auxiliary/JSON.cpp
         src/IO/AbstractIOHandlerImpl.cpp
         src/ChunkInfo.cpp
+        src/Datatype.cpp
         src/Error.cpp
         src/IO/IOTask.cpp
         src/IO/ADIOS/CommonADIOS1IOHandler.cpp
@@ -556,6 +563,7 @@ set(IO_ADIOS1_SOURCE
         src/auxiliary/JSON.cpp
         src/IO/AbstractIOHandlerImpl.cpp
         src/ChunkInfo.cpp
+        src/Datatype.cpp
         src/Error.cpp
         src/IO/IOTask.cpp
         src/IO/ADIOS/CommonADIOS1IOHandler.cpp
@@ -1323,10 +1331,9 @@ if(openPMD_INSTALL)
         if(openPMD_BUILD_CLI_TOOLS)
             foreach(toolname ${openPMD_PYTHON_CLI_TOOL_NAMES})
                 install(
-                    FILES ${openPMD_SOURCE_DIR}/src/cli/${toolname}.py
+                    PROGRAMS ${openPMD_SOURCE_DIR}/src/cli/${toolname}.py
                     DESTINATION ${openPMD_INSTALL_BINDIR}
                     RENAME openpmd-${toolname}
-                    PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
                 )
             endforeach()
         endif()

diff --git a/Dockerfile b/Dockerfile
@@ -5,8 +5,8 @@ FROM       quay.io/pypa/manylinux2010_x86_64 as build-env
 # FROM       quay.io/pypa/manylinux1_x86_64 as build-env
 ENV        DEBIAN_FRONTEND noninteractive
 
-# Python 3.7-3.11 via "37m 38 39 311"
-ARG        PY_VERSIONS="37m 38 39 310 311"
+# Python 3.8-3.11 via "38 39 311"
+ARG        PY_VERSIONS="38 39 310 311"
 
 # static libs need relocatable symbols for linking to shared python lib
 ENV        CFLAGS="-fPIC ${CFLAGS}"
@@ -122,30 +122,6 @@ RUN         for whl in /opt/src/dist/*.whl; do \
             && du -hs /opt/src/dist/* \
             && du -hs /wheelhouse/*
 
-# test in fresh env: Debian:Buster + Python 3.7
-FROM       debian:buster
-ENV        DEBIAN_FRONTEND noninteractive
-COPY --from=build-env /wheelhouse/openPMD_api-*-cp37-cp37m-manylinux2010_x86_64.whl .
-RUN        apt-get update \
-           && apt-get install -y --no-install-recommends python3 python3-pip \
-           && rm -rf /var/lib/apt/lists/*
-           # binutils
-RUN        python3 --version \
-           && python3 -m pip install -U pip \
-           && python3 -m pip install openPMD_api-*-cp37-cp37m-manylinux2010_x86_64.whl
-RUN        find / -name "openpmd*"
-RUN        ls -hal /usr/local/lib/python3.7/dist-packages/
-RUN        ls -hal /usr/local/lib/python3.7/dist-packages/openpmd_api/
-# RUN        ls -hal /usr/local/lib/python3.7/dist-packages/.libsopenpmd_api
-# RUN        objdump -x /usr/local/lib/python3.7/dist-packages/openpmd_api.cpython-37m-x86_64-linux-gnu.so | grep RPATH
-RUN        ldd /usr/local/lib/python3.7/dist-packages/openpmd_api/openpmd_api_cxx.cpython-37m-x86_64-linux-gnu.so
-RUN        python3 -c "import openpmd_api as io; print(io.__version__); print(io.variants)"
-RUN        python3 -m openpmd_api.ls --help
-RUN        openpmd-ls --help
-#RUN        echo "* soft core 100000" >> /etc/security/limits.conf && \
-#           python3 -c "import openpmd_api as io"; \
-#           gdb -ex bt -c core
-
 # test in fresh env: Debian:Sid + Python 3.8
 FROM       debian:sid
 ENV        DEBIAN_FRONTEND noninteractive

diff --git a/NEWS.rst b/NEWS.rst
@@ -3,6 +3,13 @@
 Upgrade Guide
 =============
 
+0.15.2
+------
+
+Python 3.7 is removed, please use 3.8 or newer.
+macOS 10.15 is removed, please use 11.0 or newer.
+
+
 0.15.0
 ------
 

diff --git a/README.md b/README.md
@@ -116,7 +116,7 @@ while those can be built either with or without:
 
 Optional language bindings:
 * Python:
-  * Python 3.7 - 3.11
+  * Python 3.8 - 3.11
   * pybind11 2.10.1+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)
@@ -296,7 +296,7 @@ The install will contain header files and libraries in the path set with `-DCMAK
 
 ### CMake
 
-If your project is using CMake for its build, one can conveniently use our provided `openPMDConfig.cmake` package which is installed alongside the library.
+If your project is using CMake for its build, one can conveniently use our provided `openPMDConfig.cmake` package, which is installed alongside the library.
 
 First set the following environment hint if openPMD-api was *not* installed in a system path:
 
@@ -308,7 +308,7 @@ export CMAKE_PREFIX_PATH=$HOME/somepath:$CMAKE_PREFIX_PATH
 Use the following lines in your project's `CMakeLists.txt`:
 ```cmake
 # supports:                       COMPONENTS MPI NOMPI HDF5 ADIOS1 ADIOS2
-find_package(openPMD 0.9.0 CONFIG)
+find_package(openPMD 0.15.0 CONFIG)
 
 if(openPMD_FOUND)
     target_link_libraries(YourTarget PRIVATE openPMD::openPMD)
@@ -336,13 +336,13 @@ set(openPMD_INSTALL OFF)            # or instead use:
 set(openPMD_USE_PYTHON OFF)
 FetchContent_Declare(openPMD
   GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git"
-  GIT_TAG        "dev")
+  GIT_TAG        "0.15.0")
 FetchContent_MakeAvailable(openPMD)
 ```
 
 ### Manually
 
-If your (Linux/OSX) project is build by calling the compiler directly or uses a manually written `Makefile`, consider using our `openPMD.pc` helper file for `pkg-config` which are installed alongside the library.
+If your (Linux/OSX) project is build by calling the compiler directly or uses a manually written `Makefile`, consider using our `openPMD.pc` helper file for `pkg-config`, which are installed alongside the library.
 
 First set the following environment hint if openPMD-api was *not* installed in a system path:
 

diff --git a/conda.yml b/conda.yml
@@ -36,7 +36,7 @@ dependencies:
   - pre-commit
   - pyarrow  # for dask
 #  - pybind11  # shipped internally
-  - python>=3.7
+  - python>=3.8
 
 # just a note for later hackery, we could install pip packages inside the env, too:
 #  - pip:

diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -7,6 +7,8 @@ pygments
 recommonmark
 scipy
 sphinx>=5.3
+sphinx-copybutton
+sphinx-design
 sphinx_rtd_theme>=1.1.1
 sphinxcontrib-svg2pdfconverter
 sphinxcontrib.programoutput