From a43b565ff19f10bc9f51ff0d863a5e7b6907d38d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 6 Apr 2023 10:21:43 -0700 Subject: [PATCH 1/3] Remove ADIOS1 - Long Live ADIOS2 Finally removing the deprecated and end-of-life backend ADIOS1. Long live ADIOS2! :rocket: --- .github/ISSUE_TEMPLATE/bug_report.md | 1 - .github/ci/sanitizer/clang/Leak.supp | 3 - .../spack.yaml | 3 - .../spack.yaml | 3 - .../spack.yaml | 3 - .../spack.yaml | 3 - .../spack.yaml | 3 - .../spack.yaml | 3 - .github/workflows/codeql.yml | 2 +- .github/workflows/linux.yml | 53 +- .github/workflows/macos.yml | 12 +- .github/workflows/tooling.yml | 13 +- CMakeLists.txt | 230 +- Dockerfile | 14 +- NEWS.rst | 11 + README.md | 12 +- docs/source/backends/adios1.rst | 78 +- docs/source/backends/adios2.rst | 4 - docs/source/backends/overview.rst | 2 +- docs/source/details/adios1.json | 7 - docs/source/details/adios1.toml | 2 - docs/source/details/backendconfig.rst | 19 +- docs/source/details/config_layout.json | 1 - docs/source/details/config_layout.toml | 3 - docs/source/details/mpi.rst | 2 +- docs/source/dev/buildoptions.rst | 2 - docs/source/index.rst | 2 +- docs/source/install/install.rst | 6 +- docs/source/usage/firstread.rst | 2 +- docs/source/usage/firstwrite.rst | 2 +- docs/source/usage/workflow.rst | 3 +- examples/8_benchmark_parallel.cpp | 4 +- include/openPMD/Error.hpp | 2 +- include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp | 279 --- .../openPMD/IO/ADIOS/ADIOS1FilePosition.hpp | 36 - include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp | 81 - .../openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp | 64 - .../IO/ADIOS/CommonADIOS1IOHandler.hpp | 131 -- .../IO/ADIOS/ParallelADIOS1IOHandler.hpp | 69 - .../IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp | 71 - include/openPMD/IO/Format.hpp | 1 - include/openPMD/RecordComponent.hpp | 1 - include/openPMD/ThrowError.hpp | 9 - include/openPMD/backend/Writable.hpp | 3 - include/openPMD/config.hpp.in | 2 +- src/Format.cpp | 24 +- src/IO/ADIOS/ADIOS1IOHandler.cpp | 467 ---- src/IO/ADIOS/CommonADIOS1IOHandler.cpp | 2054 ----------------- src/IO/ADIOS/ParallelADIOS1IOHandler.cpp | 511 ---- src/IO/AbstractIOHandlerHelper.cpp | 37 - src/Iteration.cpp | 14 - src/Series.cpp | 1 - src/auxiliary/JSON.cpp | 2 +- src/config.cpp | 4 +- test/CoreTest.cpp | 33 +- test/ParallelIOTest.cpp | 111 +- test/SerialIOTest.cpp | 41 +- test/python/unittest/API/APITest.py | 96 +- 58 files changed, 126 insertions(+), 4526 deletions(-) rename .github/ci/spack-envs/{clang14_py311_nompi_h5_ad1_ad2 => clang14_py311_nompi_h5_ad2}/spack.yaml (95%) rename .github/ci/spack-envs/{clang7_nopy_ompi_h5_ad1_ad2 => clang7_nopy_ompi_h5_ad2}/spack.yaml (95%) rename .github/ci/spack-envs/{clang7_nopy_ompi_h5_ad1_ad2_bp3_libcpp => clang7_nopy_ompi_h5_ad2_libcpp}/spack.yaml (95%) rename .github/ci/spack-envs/{clang8_py38_mpich_h5_ad1_ad2 => clang8_py38_mpich_h5_ad2}/spack.yaml (95%) rename .github/ci/spack-envs/{clangtidy_nopy_ompi_h5_ad1_ad2 => clangtidy_nopy_ompi_h5_ad2}/spack.yaml (95%) rename .github/ci/spack-envs/{gcc7_py36_ompi_h5_ad1_ad2 => gcc7_py36_ompi_h5_ad2}/spack.yaml (95%) delete mode 100644 docs/source/details/adios1.json delete mode 100644 docs/source/details/adios1.toml delete mode 100644 include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp delete mode 100644 include/openPMD/IO/ADIOS/ADIOS1FilePosition.hpp delete mode 100644 include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp delete mode 100644 include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp delete mode 100644 include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp delete mode 100644 include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp delete mode 100644 include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp delete mode 100644 src/IO/ADIOS/ADIOS1IOHandler.cpp delete mode 100644 src/IO/ADIOS/CommonADIOS1IOHandler.cpp delete mode 100644 src/IO/ADIOS/ParallelADIOS1IOHandler.cpp diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 86f073b184..eaef13f4ff 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -43,7 +43,6 @@ A clear and concise description of what you expected to happen. - machine: [Are you running on a public cluster? It's likely we compute on it as well!] - name and version of Python implementation: [e.g. CPython 3.9] - version of HDF5: [e.g. 1.12.0] - - version of ADIOS1: [e.g. 1.13.1] - version of ADIOS2: [e.g. 2.7.1] - name and version of MPI: [e.g. OpenMPI 4.1.1] diff --git a/.github/ci/sanitizer/clang/Leak.supp b/.github/ci/sanitizer/clang/Leak.supp index 81e8ee7f75..a093615cab 100644 --- a/.github/ci/sanitizer/clang/Leak.supp +++ b/.github/ci/sanitizer/clang/Leak.supp @@ -9,9 +9,6 @@ leak:libevent* leak:*py* # 10 byte memleak in main of "make" 4.1 leak:/usr/bin/make* -# ADIOS 1.13.1 -leak:adios_read_bp_open_file -leak:adios_inq_var # ADIOS2 leak:adios2::core::engine::SstReader::* leak:adios2::core::engine::SstWriter::* diff --git a/.github/ci/spack-envs/clang14_py311_nompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml similarity index 95% rename from .github/ci/spack-envs/clang14_py311_nompi_h5_ad1_ad2/spack.yaml rename to .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml index 2abc61177a..06e9d1ac40 100644 --- a/.github/ci/spack-envs/clang14_py311_nompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang14_py311_nompi_h5_ad2/spack.yaml @@ -6,15 +6,12 @@ # spack: specs: - - adios - adios2 - hdf5 packages: hdf5: variants: ~mpi - adios: - variants: ~mpi ~zfp ~sz ~lz4 ~blosc adios2: variants: ~mpi ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: diff --git a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml similarity index 95% rename from .github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/spack.yaml rename to .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml index e5a71811e5..fa2e588608 100644 --- a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/spack.yaml @@ -6,14 +6,11 @@ # spack: specs: - - adios - adios2 - hdf5 - openmpi packages: - adios: - variants: ~zfp ~sz ~lz4 ~blosc adios2: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: diff --git a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml similarity index 95% rename from .github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml rename to .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml index 5308c57ce5..8a8dd00024 100644 --- a/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2_bp3_libcpp/spack.yaml +++ b/.github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/spack.yaml @@ -6,14 +6,11 @@ # spack: specs: - - adios - adios2 - hdf5 - openmpi packages: - adios: - variants: ~zfp ~sz ~lz4 ~blosc adios2: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: diff --git a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml similarity index 95% rename from .github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml rename to .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml index 4a6ab7f74e..a6d568611f 100644 --- a/.github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clang8_py38_mpich_h5_ad2/spack.yaml @@ -6,14 +6,11 @@ # spack: specs: - - adios - adios2@2.7.1 - hdf5 - mpich packages: - adios: - variants: ~zfp ~sz ~lz4 ~blosc adios2: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: diff --git a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml similarity index 95% rename from .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml rename to .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml index 98acde6e62..1543ec794c 100644 --- a/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/spack.yaml @@ -6,14 +6,11 @@ # spack: specs: - - adios - adios2 - hdf5 - openmpi packages: - adios: - variants: ~zfp ~sz ~lz4 ~blosc adios2: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: diff --git a/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/spack.yaml b/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml similarity index 95% rename from .github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/spack.yaml rename to .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml index f0b051678e..8785650cf2 100644 --- a/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/spack.yaml +++ b/.github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/spack.yaml @@ -6,14 +6,11 @@ # spack: specs: - - adios - adios2 - hdf5 - openmpi packages: - adios: - variants: ~zfp ~sz ~lz4 ~blosc adios2: variants: ~zfp ~sz ~png ~dataman ~python ~fortran ~ssc ~shared ~bzip2 cmake: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1925663362..02d72d06e9 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -33,7 +33,7 @@ jobs: - name: Install Packages run: | sudo apt-get update - sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev + sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev python -m pip install --upgrade pip python -m pip install --upgrade wheel diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 17226a99d6..91f089e03a 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -39,14 +39,14 @@ jobs: cmake --build build --parallel 2 ctest --test-dir build --output-on-failure - clang7_nopy_ompi_h5_ad1_ad2_bp3_libcpp: + clang7_nopy_ompi_h5_ad2_libcpp: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v3 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad1_ad2_bp3_libcpp_v2} + with: {path: /opt/spack, key: clang7_nopy_ompi_h5_ad2_libcpp_v2} - name: Install run: | sudo apt-get update @@ -62,7 +62,7 @@ jobs: mpiexec --version perl --version python --version - eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2_bp3_libcpp/) + eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2_libcpp/) spack install share/openPMD/download_samples.sh build @@ -70,7 +70,6 @@ jobs: -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_ADIOS1=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON @@ -79,7 +78,6 @@ jobs: find . -name *.bp | xargs -n1 -P1 -I {} rm -rf {} find . -name *.bp.dir | xargs -n1 -P1 -I {} rm -rf {} - export OPENPMD_BP_BACKEND=ADIOS1 ctest --test-dir build --output-on-failure clang7_nopy_ompi_h5_ad2_newLayout: @@ -98,7 +96,7 @@ jobs: - name: Build env: {CC: clang-7, CXX: clang++-7, CXXFLAGS: -Werror, OPENPMD2_ADIOS2_SCHEMA: 20210209} run: | - eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad1_ad2/) + eval $(spack env activate --sh .github/ci/spack-envs/clang7_nopy_ompi_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -106,7 +104,6 @@ jobs: -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_ADIOS1=OFF \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON @@ -114,16 +111,16 @@ jobs: ctest --test-dir build --output-on-failure # TODO -# clang7_py36_nompi_h5_ad1_ad2_libstdc++ +# clang7_py36_nompi_h5_ad2_libstdc++ - clang14_py311_nompi_h5_ad1_ad2: + clang14_py311_nompi_h5_ad2: runs-on: ubuntu-22.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v3 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang14_py311_nompi_h5_ad1_ad2_v2 } + with: {path: /opt/spack, key: clang14_py311_nompi_h5_ad2_v2 } - name: Install run: | sudo apt update @@ -135,7 +132,7 @@ jobs: - name: Build env: {CC: clang-14, CXX: clang++-14, CXXFLAGS: -Werror} run: | - eval $(spack env activate --sh .github/ci/spack-envs/clang14_py311_nompi_h5_ad1_ad2/) + eval $(spack env activate --sh .github/ci/spack-envs/clang14_py311_nompi_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -143,7 +140,6 @@ jobs: -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=OFF \ -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_ADIOS1=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -152,14 +148,14 @@ jobs: ctest --test-dir build --output-on-failure # ADIOS2 v2.7.1 - clang8_py38_mpich_h5_ad1_ad2_newLayout: + clang8_py38_mpich_h5_ad2_newLayout: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v3 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clang8_py38_mpich_h5_ad1_ad2_newLayout_v2 } + with: {path: /opt/spack, key: clang8_py38_mpich_h5_ad2_newLayout_v2 } - name: Install run: | sudo apt-get update @@ -172,7 +168,7 @@ jobs: mpiexec --version perl --version python --version - eval $(spack env activate --sh .github/ci/spack-envs/clang8_py38_mpich_h5_ad1_ad2/) + eval $(spack env activate --sh .github/ci/spack-envs/clang8_py38_mpich_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -180,35 +176,34 @@ jobs: -DopenPMD_USE_PYTHON=OFF \ -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_ADIOS1=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 ctest --test-dir build --output-on-failure # TODO: (old Travis-CI coverage) -# clang10_py38_ompi_h5_1-10-6_ad1_ad2_release +# clang10_py38_ompi_h5_1-10-6_ad2_release # ..._h5coll with OPENPMD_HDF5_INDEPENDENT: OFF # TODO: (old Travis-CI coverage) -# gcc-4.9.4_nopy_nompi_h5_ad1_ad2 -# gcc-4.9.4_nopy_ompi_h5_ad1_ad2 -# gcc-9.3.0_nopy_nompi_h5_ad1_ad2-2.6.0 -# gcc-7.4.0_nopy_ompi_h5_ad1_h5coll -# gcc-6.5.0_py35_nompi_h5_ad1-1.13.1 -# gcc-8.1.0_py37_nompi_h5_ad1_static +# gcc-4.9.4_nopy_nompi_h5_ad2 +# gcc-4.9.4_nopy_ompi_h5_ad2 +# gcc-9.3.0_nopy_nompi_h5_ad2-2.6.0 +# gcc-7.4.0_nopy_ompi_h5_h5coll +# gcc-6.5.0_py35_nompi_h5-1.13.1 +# gcc-8.1.0_py37_nompi_h5_static # gcc-6.5.0_py36_nompi_h5-1.8.13 # gcc-4.8.5_py35_nompi_h5 -# gcc-7.4.0_py_ompi_h5_ad1_ad2_coveralls +# gcc-7.4.0_py_ompi_h5_ad2_coveralls - gcc7_py36_pd_dd_ompi_h5_ad1_ad2: + gcc7_py36_pd_dd_ompi_h5_ad2: runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v3 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: gcc7_py36_ompi_h5_ad1_ad2_v2 } + with: {path: /opt/spack, key: gcc7_py36_ompi_h5_ad2_v2 } - name: Install run: | sudo apt-get update @@ -223,7 +218,7 @@ jobs: - name: Build env: {CC: gcc-7, CXX: g++-7, CXXFLAGS: -Werror} run: | - eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad1_ad2/) + eval $(spack env activate --sh .github/ci/spack-envs/gcc7_py36_ompi_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -231,7 +226,6 @@ jobs: -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_ADIOS1=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 @@ -245,7 +239,7 @@ jobs: - name: Install run: | sudo apt-get update - sudo apt-get install g++ libopenmpi-dev libhdf5-openmpi-dev libadios-dev python3 python3-numpy python3-mpi4py python3-pandas + sudo apt-get install g++ libopenmpi-dev libhdf5-openmpi-dev python3 python3-numpy python3-mpi4py python3-pandas # TODO ADIOS2 - name: Build env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig} @@ -313,7 +307,6 @@ jobs: -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_ADIOS1=ON \ -DopenPMD_USE_ADIOS2=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 04fbbbe30e..2295770ce2 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -8,7 +8,7 @@ concurrency: jobs: # TODO: (old Travis-CI coverage) -# appleclang9_py37_nompi_h5_ad1 +# appleclang9_py37_nompi_h5 # appleclang10_py37_h5_ad2_libcpp # appleclang11_nopy_nompi_h5_ad2 @@ -43,7 +43,7 @@ jobs: cmake --build build --parallel 3 ctest --test-dir build --verbose - appleclang12_py_ad1: + appleclang12_py: runs-on: macos-10.15 # next: macOS-11 if: github.event.pull_request.draft == false @@ -54,13 +54,6 @@ jobs: run: | set +e python3 -m pip install -U numpy pandas - - curl -Lo adios-1.13.1.tar.gz http://users.nccs.gov/~pnorbert/adios-1.13.1.tar.gz - tar -xzf adios-1.13.1.tar.gz - cd adios-1.13.1/ - CFLAGS="-fPIC" ./configure --enable-static --enable-shared --prefix=/usr/local --without-mpi --disable-fortran - make -j 3 - make install set -e - name: Build env: {CXXFLAGS: -Werror -DTOML11_DISABLE_STD_FILESYSTEM, MACOSX_DEPLOYMENT_TARGET: 10.14} @@ -73,7 +66,6 @@ jobs: -DopenPMD_USE_PYTHON=ON \ -DopenPMD_USE_MPI=OFF \ -DopenPMD_USE_HDF5=OFF \ - -DopenPMD_USE_ADIOS1=ON \ -DopenPMD_USE_ADIOS2=OFF \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 3 diff --git a/.github/workflows/tooling.yml b/.github/workflows/tooling.yml index 9b46734578..96a1e5f8fd 100644 --- a/.github/workflows/tooling.yml +++ b/.github/workflows/tooling.yml @@ -7,7 +7,7 @@ concurrency: cancel-in-progress: true jobs: - clangtidy10_nopy_ompi_h5_ad1_ad2: + clangtidy10_nopy_ompi_h5_ad2: name: clang-tidy w/o py runs-on: ubuntu-20.04 if: github.event.pull_request.draft == false @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 - name: Spack Cache uses: actions/cache@v3 - with: {path: /opt/spack, key: clangtidy10_nopy_ompi_h5_ad1_ad2 } + with: {path: /opt/spack, key: clangtidy10_nopy_ompi_h5_ad2 } - name: Install run: | sudo apt-get update @@ -24,7 +24,7 @@ jobs: - name: Build env: {CC: clang, CXX: clang++} run: | - eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad1_ad2/) + eval $(spack env activate --sh .github/ci/spack-envs/clangtidy_nopy_ompi_h5_ad2/) spack install share/openPMD/download_samples.sh build @@ -35,7 +35,7 @@ jobs: cat build/clang-tidy.log if [[ $(wc -m - $) - target_include_directories(openPMD.ADIOS1.Parallel SYSTEM PRIVATE - $ - $) - - set_target_properties(openPMD.ADIOS1.Serial PROPERTIES - COMPILE_PDB_NAME openPMD.ADIOS1.Serial - ARCHIVE_OUTPUT_DIRECTORY ${openPMD_ARCHIVE_OUTPUT_DIRECTORY} - LIBRARY_OUTPUT_DIRECTORY ${openPMD_LIBRARY_OUTPUT_DIRECTORY} - RUNTIME_OUTPUT_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY} - PDB_OUTPUT_DIRECTORY ${openPMD_PDB_OUTPUT_DIRECTORY} - COMPILE_PDB_OUTPUT_DIRECTORY ${openPMD_COMPILE_PDB_OUTPUT_DIRECTORY} - - POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN ON - ) - # note: same as above, but for Multi-Config generators - if(isMultiConfig) - foreach(CFG IN LISTS CMAKE_CONFIGURATION_TYPES) - string(TOUPPER "${CFG}" CFG_UPPER) - set_target_properties(openPMD.ADIOS1.Serial PROPERTIES - COMPILE_PDB_NAME_${CFG_UPPER} openPMD.ADIOS1.Serial - ARCHIVE_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_ARCHIVE_OUTPUT_DIRECTORY}/${CFG} - LIBRARY_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_LIBRARY_OUTPUT_DIRECTORY}/${CFG} - RUNTIME_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/${CFG} - PDB_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_PDB_OUTPUT_DIRECTORY}/${CFG} - COMPILE_PDB_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_COMPILE_PDB_OUTPUT_DIRECTORY}/${CFG} - ) - endforeach() - endif() - if("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") - set_target_properties(openPMD.ADIOS1.Serial PROPERTIES - LINK_FLAGS "-Wl,--exclude-libs,ALL") - elseif("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin") - set_target_properties(openPMD.ADIOS1.Serial PROPERTIES - XCODE_ATTRIBUTE_STRIP_STYLE "non-global" - XCODE_ATTRIBUTE_DEPLOYMENT_POSTPROCESSING "YES" - XCODE_ATTRIBUTE_SEPARATE_STRIP "YES" - ) - endif() - foreach(adlib ${ADIOS_LIBRARIES_SEQUENTIAL}) - target_link_libraries(openPMD.ADIOS1.Serial PRIVATE ${adlib}) - endforeach() - target_include_directories(openPMD.ADIOS1.Serial SYSTEM PRIVATE ${ADIOS_INCLUDE_DIRS_SEQUENTIAL}) - target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE "${ADIOS_DEFINITIONS_SEQUENTIAL}") - target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE openPMD_HAVE_ADIOS1=1) - target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE openPMD_HAVE_MPI=0) - target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE _NOMPI) # ADIOS header - - if(openPMD_HAVE_MPI) - set_target_properties(openPMD.ADIOS1.Parallel PROPERTIES - COMPILE_PDB_NAME openPMD.ADIOS1.Parallel - ARCHIVE_OUTPUT_DIRECTORY ${openPMD_ARCHIVE_OUTPUT_DIRECTORY} - LIBRARY_OUTPUT_DIRECTORY ${openPMD_LIBRARY_OUTPUT_DIRECTORY} - RUNTIME_OUTPUT_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY} - PDB_OUTPUT_DIRECTORY ${openPMD_PDB_OUTPUT_DIRECTORY} - COMPILE_PDB_OUTPUT_DIRECTORY ${openPMD_COMPILE_PDB_OUTPUT_DIRECTORY} - - POSITION_INDEPENDENT_CODE ON - CXX_VISIBILITY_PRESET hidden - VISIBILITY_INLINES_HIDDEN 1 - ) - # note: same as above, but for Multi-Config generators - if(isMultiConfig) - foreach(CFG IN LISTS CMAKE_CONFIGURATION_TYPES) - string(TOUPPER "${CFG}" CFG_UPPER) - set_target_properties(openPMD.ADIOS1.Parallel PROPERTIES - COMPILE_PDB_NAME_${CFG_UPPER} opemPMD.ADIOS1.Parallel - ARCHIVE_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_ARCHIVE_OUTPUT_DIRECTORY}/${CFG} - LIBRARY_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_LIBRARY_OUTPUT_DIRECTORY}/${CFG} - RUNTIME_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/${CFG} - PDB_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_PDB_OUTPUT_DIRECTORY}/${CFG} - COMPILE_PDB_OUTPUT_DIRECTORY_${CFG_UPPER} ${openPMD_COMPILE_PDB_OUTPUT_DIRECTORY}/${CFG} - ) - endforeach() - endif() - if("${CMAKE_SYSTEM_NAME}" MATCHES "Linux") - set_target_properties(openPMD.ADIOS1.Parallel PROPERTIES - LINK_FLAGS "-Wl,--exclude-libs,ALL") - elseif("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin") - set_target_properties(openPMD.ADIOS1.Parallel PROPERTIES - XCODE_ATTRIBUTE_STRIP_STYLE "non-global" - XCODE_ATTRIBUTE_DEPLOYMENT_POSTPROCESSING "YES" - XCODE_ATTRIBUTE_SEPARATE_STRIP "YES" - ) - endif() - foreach(adlib ${ADIOS_LIBRARIES}) - target_link_libraries(openPMD.ADIOS1.Parallel PRIVATE ${adlib}) - endforeach() - target_link_libraries(openPMD.ADIOS1.Parallel PUBLIC ${openPMD_MPI_TARGETS}) - - target_include_directories(openPMD.ADIOS1.Parallel SYSTEM PRIVATE ${ADIOS_INCLUDE_DIRS}) - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE "${ADIOS_DEFINITIONS}") - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_HAVE_ADIOS1=1) - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_HAVE_MPI=1) - else() - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_HAVE_ADIOS1=0) - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_HAVE_MPI=0) - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE _NOMPI) # ADIOS header - endif() - # This ensures that the ADIOS1 targets don't ever include Error.hpp - # To avoid incompatible error types in weird compile configurations - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE OPENPMD_ADIOS1_IMPLEMENTATION) - - # Runtime parameter and API status checks ("asserts") - if(openPMD_USE_VERIFY) - target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE openPMD_USE_VERIFY=1) - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_USE_VERIFY=1) - else() - target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE openPMD_USE_VERIFY=0) - target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_USE_VERIFY=0) - endif() - - target_link_libraries(openPMD PUBLIC openPMD.ADIOS1.Serial) - target_link_libraries(openPMD PUBLIC openPMD.ADIOS1.Parallel) -else() - # add stubs to prevent missing symbols in Clang ASAN/UBSAN - target_sources(openPMD PRIVATE ${IO_ADIOS1_SEQUENTIAL_SOURCE} ${IO_ADIOS1_SOURCE}) -endif() - # ADIOS2 Backend if(openPMD_HAVE_ADIOS2) if(openPMD_HAVE_MPI) @@ -1265,11 +1047,6 @@ write_basic_package_version_file("openPMDConfigVersion.cmake" if(openPMD_INSTALL) set(openPMD_INSTALL_TARGET_NAMES openPMD) - if(openPMD_HAVE_ADIOS1) - list(APPEND openPMD_INSTALL_TARGET_NAMES - openPMD.ADIOS1.Serial openPMD.ADIOS1.Parallel) - endif() - if(openPMD_BUILD_CLI_TOOLS) foreach(toolname ${openPMD_CLI_TOOL_NAMES}) list(APPEND openPMD_INSTALL_TARGET_NAMES openpmd-${toolname}) @@ -1354,11 +1131,6 @@ if(openPMD_INSTALL) ${openPMD_BINARY_DIR}/openPMDConfigVersion.cmake DESTINATION ${openPMD_INSTALL_CMAKEDIR} ) - install( - FILES - ${openPMD_SOURCE_DIR}/share/openPMD/cmake/FindADIOS.cmake - DESTINATION ${openPMD_INSTALL_CMAKEDIR}/Modules - ) # pkg-config .pc file for depending legacy projects # This is for projects that do not use a build file generator, e.g. # because they compile manually on the command line or write their @@ -1565,7 +1337,7 @@ if(openPMD_BUILD_TESTING) # openpmd-pipe (python) test if( NOT WIN32 AND openPMD_HAVE_HDF5 - AND (openPMD_HAVE_ADIOS2 OR openPMD_HAVE_ADIOS1) + AND openPMD_HAVE_ADIOS2 AND EXAMPLE_DATA_FOUND ) if( openPMD_HAVE_MPI ) diff --git a/Dockerfile b/Dockerfile index a7830ecbb7..54e9050987 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ENV CFLAGS="-fPIC ${CFLAGS}" ENV CXXFLAGS="-fPIC ${CXXFLAGS}" # install dependencies -# CMake, zlib?, HDF5, c-blosc, ADIOS1, ADIOS2 +# CMake, zlib?, HDF5, c-blosc, ADIOS2 RUN yum check-update -y \ ; yum -y install \ glibc-static \ @@ -39,7 +39,7 @@ RUN curl -sLo hdf5-1.10.5.tar.gz https://support.hdfgroup.org/ftp/HDF5/re && make \ && make install -# avoid picking up a static libpthread in adios (also: those libs lack -fPIC) +# avoid picking up a static libpthread (also: those libs lack -fPIC) RUN rm -f /usr/lib64/libpthread.a /usr/lib64/libm.a /usr/lib64/librt.a RUN rm -f /usr/lib/libpthread.a /usr/lib/libm.a /usr/lib/librt.a @@ -56,15 +56,6 @@ RUN curl -sLo c-blosc-1.15.0.tar.gz https://github.com/Blosc/c-blosc/arch && make \ && make install -RUN curl -sLo adios-1.13.1.tar.gz http://users.nccs.gov/~pnorbert/adios-1.13.1.tar.gz \ - && file adios*.tar.gz \ - && tar -xzf adios*.tar.gz \ - && rm adios*.tar.gz \ - && cd adios-* \ - && ./configure --enable-static --disable-shared --disable-fortran --without-mpi --prefix=/usr --with-blosc=/usr \ - && make \ - && make install - RUN curl -sLo adios2-2.7.1.tar.gz https://github.com/ornladios/ADIOS2/archive/v2.7.1.tar.gz \ && file adios2*.tar.gz \ && tar -xzf adios2*.tar.gz \ @@ -84,7 +75,6 @@ RUN ls /opt/python/ ENV HDF5_USE_STATIC_LIBRARIES=ON \ ZLIB_USE_STATIC_LIBS=ON \ - ADIOS_USE_STATIC_LIBS=ON \ openPMD_BUILD_TESTING=OFF \ openPMD_BUILD_EXAMPLES=OFF diff --git a/NEWS.rst b/NEWS.rst index 40246be635..86ea34fff4 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -3,6 +3,17 @@ Upgrade Guide ============= +0.16.0 +------ + +The ADIOS1 library is no longer developed in favor of ADIOS2. +Consequently, ADIOS1 support was removed in openPMD-api 0.16.0 and newer. +Please transition to ADIOS2. + +For reading legacy ADIOS1 BP3 files, either use an older version of openPMD-api or the BP3 backend in ADIOS2. +Note that ADIOS2 does not support compression in BP3 files. + + 0.15.0 ------ diff --git a/README.md b/README.md index b422379d44..7d596ad94e 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ openPMD is an open meta-data schema that provides meaning and self-description f See [the openPMD standard](https://github.com/openPMD/openPMD-standard) for details of this schema. This library provides a reference API for openPMD data handling. -Since openPMD is a schema (or markup) on top of portable, hierarchical file formats, this library implements various backends such as HDF5, ADIOS1, ADIOS2 and JSON. +Since openPMD is a schema (or markup) on top of portable, hierarchical file formats, this library implements various backends such as HDF5, ADIOS2 and JSON. Writing & reading through those backends and their associated files are supported for serial and [MPI-parallel](https://www.mpi-forum.org/docs/) workflows. ## Usage @@ -108,7 +108,6 @@ Shipped internally in `share/openPMD/thirdParty/`: I/O backends: * [JSON](https://en.wikipedia.org/wiki/JSON) * [HDF5](https://support.hdfgroup.org/HDF5) 1.8.13+ (optional) -* [ADIOS1](https://www.olcf.ornl.gov/center-projects/adios) 1.13.1+ (optional, deprecated) * [ADIOS2](https://github.com/ornladios/ADIOS2) 2.7.0+ (optional) while those can be built either with or without: @@ -144,7 +143,7 @@ Choose *one* of the install methods below to get started: [![Spack Use Case](https://img.shields.io/badge/use_case-desktop_%28C%2B%2B,_py%29,_development,_HPC-brightgreen)](https://spack.readthedocs.io/en/latest/package_list.html#openpmd-api) ```bash -# optional: +python +adios1 -adios2 -hdf5 -mpi +# optional: +python -adios2 -hdf5 -mpi spack install openpmd-api spack load openpmd-api ``` @@ -182,7 +181,7 @@ brew install openpmd-api [![PyPI Format](https://img.shields.io/pypi/format/openPMD-api)](https://pypi.org/project/openPMD-api) [![PyPI Downloads](https://img.shields.io/pypi/dm/openPMD-api)](https://pypi.org/project/openPMD-api) -On very old macOS versions (<10.9) or on exotic processor architectures, this install method *compiles from source* against the found installations of HDF5, ADIOS1, ADIOS2, and/or MPI (in system paths, from other package managers, or loaded via a module system, ...). +On very old macOS versions (<10.9) or on exotic processor architectures, this install method *compiles from source* against the found installations of HDF5, ADIOS2, and/or MPI (in system paths, from other package managers, or loaded via a module system, ...). ```bash # we need pip 19 or newer @@ -251,7 +250,6 @@ CMake controls options with prefixed `-D`, e.g. `-DopenPMD_USE_MPI=OFF`: |------------------------------|------------------|------------------------------------------------------------------------------| | `openPMD_USE_MPI` | **AUTO**/ON/OFF | Parallel, Multi-Node I/O for clusters | | `openPMD_USE_HDF5` | **AUTO**/ON/OFF | HDF5 backend (`.h5` files) | -| `openPMD_USE_ADIOS1` | AUTO/ON/**OFF** | ADIOS1 backend (`.bp` files up to version BP3) - deprecated | | `openPMD_USE_ADIOS2` | **AUTO**/ON/OFF | ADIOS2 backend (`.bp` files in BP3, BP4 or higher) | | `openPMD_USE_PYTHON` | **AUTO**/ON/OFF | Enable Python bindings | | `openPMD_USE_INVASIVE_TESTS` | ON/**OFF** | Enable unit tests that modify source code 1 | @@ -307,7 +305,7 @@ export CMAKE_PREFIX_PATH=$HOME/somepath:$CMAKE_PREFIX_PATH Use the following lines in your project's `CMakeLists.txt`: ```cmake -# supports: COMPONENTS MPI NOMPI HDF5 ADIOS1 ADIOS2 +# supports: COMPONENTS MPI NOMPI HDF5 ADIOS2 find_package(openPMD 0.9.0 CONFIG) if(openPMD_FOUND) @@ -435,7 +433,7 @@ This work was partially funded by the Center of Advanced Systems Understanding ( openPMD-api stands on the shoulders of giants and we are grateful for the following projects included as direct dependencies: -* [ADIOS1](https://github.com/ornladios/ADIOS) and [ADIOS2](https://github.com/ornladios/ADIOS2) by [S. Klasky (ORNL), team, collaborators](https://csmd.ornl.gov/adios) and [contributors](https://github.com/ornladios/ADIOS2/graphs/contributors) +* [ADIOS2](https://github.com/ornladios/ADIOS2) by [S. Klasky, N. Podhorszki, W.F. Godoy (ORNL), team, collaborators](https://csmd.ornl.gov/adios) and [contributors](https://github.com/ornladios/ADIOS2/graphs/contributors) * [Catch2](https://github.com/catchorg/Catch2) by [Phil Nash](https://github.com/philsquared), [Martin Hořeňovský](https://github.com/horenmar) and [contributors](https://github.com/catchorg/Catch2/graphs/contributors) * HDF5 by [the HDF group](https://www.hdfgroup.org) and community * [json](https://github.com/nlohmann/json) by [Niels Lohmann](https://github.com/nlohmann) and [contributors](https://github.com/nlohmann/json/graphs/contributors) diff --git a/docs/source/backends/adios1.rst b/docs/source/backends/adios1.rst index 11680d7e49..4be380b4ff 100644 --- a/docs/source/backends/adios1.rst +++ b/docs/source/backends/adios1.rst @@ -3,80 +3,12 @@ ADIOS1 ====== -openPMD supports writing to and reading from ADIOS1 ``.bp`` files. -For this, the installed copy of openPMD must have been built with support for the ADIOS1 backend. -To build openPMD with support for ADIOS, use the CMake option ``-DopenPMD_USE_ADIOS1=ON``. -For further information, check out the :ref:`installation guide `, -:ref:`build dependencies ` and the :ref:`build options `. +The ADIOS1 library is no longer developed in favor of ADIOS2. +Consequently, ADIOS1 support was removed in openPMD-api 0.16.0 and newer. +Please transition to ADIOS2. -.. note:: - - This backend is deprecated, please use ADIOS2 instead. - - -I/O Method ----------- - -ADIOS1 has several staging methods for alternative file formats, yet natively writes to ``.bp`` files. -We currently implement the ``MPI_AGGREGATE`` transport method for MPI-parallel write (``POSIX`` for serial write) and ``ADIOS_READ_METHOD_BP`` for read. - - -Backend-Specific Controls -------------------------- - -The following environment variables control ADIOS1 I/O behavior at runtime. -Fine-tuning these is especially useful when running at large scale. - -============================================== ========== ================================================================================ -environment variable default description -============================================== ========== ================================================================================ -``OPENPMD_ADIOS_NUM_AGGREGATORS`` ``1`` Number of I/O aggregator nodes for ADIOS1 ``MPI_AGGREGATE`` transport method. -``OPENPMD_ADIOS_NUM_OST`` ``0`` Number of I/O OSTs for ADIOS1 ``MPI_AGGREGATE`` transport method. -``OPENPMD_ADIOS_HAVE_METADATA_FILE`` ``1`` Online creation of the adios journal file (``1``: yes, ``0``: no). -``OPENPMD_BP_BACKEND`` ``ADIOS2`` Chose preferred ``.bp`` file backend if ``ADIOS1`` and ``ADIOS2`` are available. -``OPENPMD_ADIOS_SUPPRESS_DEPRECATED_WARNING`` ``0`` Set to ``1`` to suppress ADIOS1 deprecation warnings. -============================================== ========== ================================================================================ - -Please refer to the `ADIOS1 manual, section 6.1.5 `_ for details on I/O tuning. - -In case both the ADIOS1 backend and the :ref:`ADIOS2 backend ` are enabled, set ``OPENPMD_BP_BACKEND`` to ``ADIOS1`` to enforce using ADIOS1. -If only the ADIOS1 backend was compiled but not the :ref:`ADIOS2 backend `, the default of ``OPENPMD_BP_BACKEND`` is automatically switched to ``ADIOS1``. -Be advised that ADIOS1 only supports ``.bp`` files up to the internal version BP3, while ADIOS2 supports BP3, BP4 and later formats. - - -Best Practice at Large Scale ----------------------------- - -A good practice at scale is to disable the online creation of the metadata file. -After writing the data, run ``bpmeta`` on the (to-be-created) filename to generate the metadata file offline (repeat per iteration for file-based encoding). -This metadata file is needed for reading, while the actual heavy data resides in ``.dir/`` directories. - -Further options depend heavily on filesystem type, specific file striping, network infrastructure and available RAM on the aggregator nodes. -If your filesystem exposes explicit object-storage-targets (OSTs), such as Lustre, try to set the number of OSTs to the maximum number available and allowed per job (e.g. non-full), assuming the number of writing MPI ranks is larger. -A good number for aggregators is usually the number of contributing nodes divided by four. - -For fine-tuning at extreme scale or for exotic systems, please refer to the ADIOS1 manual and talk to your filesystem admins and the ADIOS1 authors. -Be aware that extreme-scale I/O is a research topic after all. - - -Limitations ------------ - -.. note:: - - You cannot initialize and use more than one ``openPMD::Series`` with ADIOS1 backend at the same time in a process, even if both Series operate on totally independent data. - This is an upstream bug in ADIOS1 that we cannot control: ADIOS1 cannot be initialized more than once, probably because it shares some internal state. - -.. note:: - - The way we currently implement ADIOS1 in openPMD-api is sub-ideal and we close/re-open file handles way too often. - Consequently, this can lead to severe performance degradation unless fixed. - Mea culpa, we did better in the past (in PIConGPU). - Please consider using our ADIOS2 backend instead, on which we focus our developments these days. - -.. note:: - - ADIOS1 does not support attributes that are `arrays of complex types `_. +For reading legacy ADIOS1 BP3 files, either use an older version of openPMD-api or the BP3 backend in ADIOS2. +Note that ADIOS2 does not support compression in BP3 files. Selected References diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 807538c2f6..ac5e50930f 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -84,7 +84,6 @@ environment variable default description ``OPENPMD_ADIOS2_ENGINE`` ``File`` `ADIOS2 engine `_ ``OPENPMD2_ADIOS2_SCHEMA`` ``0`` ADIOS2 schema version (see below) ``OPENPMD_ADIOS2_STATS_LEVEL`` ``0`` whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no). -``OPENPMD_BP_BACKEND`` ``ADIOS2`` Chose preferred ``.bp`` file backend if ``ADIOS1`` and ``ADIOS2`` are available. ``OPENPMD_ADIOS2_BP5_BufferChunkMB`` ``0`` ADIOS2 BP5 engine: applies when using either EveryoneWrites or EveryoneWritesSerial aggregation ``OPENPMD_ADIOS2_BP5_MaxShmMB`` ``0`` ADIOS2 BP5 engine: applies when using TwoLevelShm aggregation ``OPENPMD_ADIOS2_BP5_NumSubFiles`` ``0`` ADIOS2 BP5 engine: num of subfiles @@ -94,9 +93,6 @@ environment variable default description Please refer to the `ADIOS2 documentation `_ for details on I/O tuning. -In case the ADIOS2 backend was not compiled but only the deprecated :ref:`ADIOS1 backend `, the default of ``OPENPMD_BP_BACKEND`` will fall back to ``ADIOS1``. -Be advised that ADIOS1 only supports ``.bp`` files up to the internal version BP3, while ADIOS2 supports BP3, BP4 and later formats. - Notice that the ADIOS2 backend is alternatively configurable via :ref:`JSON parameters `. Due to performance considerations, the ADIOS2 backend configures ADIOS2 not to compute any dataset statistics (Min/Max) by default. diff --git a/docs/source/backends/overview.rst b/docs/source/backends/overview.rst index cb426575d0..28c20df1c7 100644 --- a/docs/source/backends/overview.rst +++ b/docs/source/backends/overview.rst @@ -23,7 +23,7 @@ Performance A- A B C Native File Format ``.bp`` (BP3) ``.bp`` (BP3-5) ``.h5`` ``.json`` ================== ============= =============== ========= ========== -:ref:`ADIOS1 is deprecated and will be removed in a future release `. +:ref:`ADIOS1 was removed in version 0.16.0 `. Please use ADIOS2 instead. * supported/yes: implemented and accessible for users of openPMD-api diff --git a/docs/source/details/adios1.json b/docs/source/details/adios1.json deleted file mode 100644 index 95c934c76b..0000000000 --- a/docs/source/details/adios1.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "adios1": { - "dataset": { - "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" - } - } -} diff --git a/docs/source/details/adios1.toml b/docs/source/details/adios1.toml deleted file mode 100644 index 6d5dca07c3..0000000000 --- a/docs/source/details/adios1.toml +++ /dev/null @@ -1,2 +0,0 @@ -[adios1.dataset] -transform = "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" diff --git a/docs/source/details/backendconfig.rst b/docs/source/details/backendconfig.rst index cef1e4d6f9..f1c47c78bd 100644 --- a/docs/source/details/backendconfig.rst +++ b/docs/source/details/backendconfig.rst @@ -29,7 +29,7 @@ The following list specifies the priority of these means, beginning with the low 5. Explicit API calls such as ``setIterationEncoding()`` The configuration is read in a case-insensitive manner, keys as well as values. -An exception to this are string values which are forwarded to other libraries such as ADIOS1 and ADIOS2. +An exception to this are string values which are forwarded to other libraries such as ADIOS2. Those are read "as-is" and interpreted by the backend library. Parameters that are directly passed through to an external library and not interpreted within openPMD API (e.g. ``adios2.engine.parameters``) are unaffected by this and follow the respective library's conventions. @@ -77,7 +77,7 @@ For a consistent user interface, backends shall follow the following rules: Backend-independent JSON configuration -------------------------------------- -The openPMD backend can be chosen via the JSON/TOML key ``backend`` which recognizes the alternatives ``["hdf5", "adios1", "adios2", "json"]``. +The openPMD backend can be chosen via the JSON/TOML key ``backend`` which recognizes the alternatives ``["hdf5", "adios2", "json"]``. The iteration encoding can be chosen via the JSON/TOML key ``iteration_encoding`` which recognizes the alternatives ``["file_based", "group_based", "variable_based"]``. Note that for file-based iteration encoding, specification of the expansion pattern in the file name (e.g. ``data_%T.json``) remains mandatory. @@ -187,20 +187,7 @@ Explanation of the single keys: ``"none"`` can be used to disable chunking. Chunking generally improves performance and only needs to be disabled in corner-cases, e.g. when heavily relying on independent, parallel I/O that non-collectively declares data records. -ADIOS1 -^^^^^^ - -ADIOS1 allows configuring custom dataset transforms via JSON/TOML: - -.. literalinclude:: adios1.json - :language: json - -.. literalinclude:: adios1.toml - :language: toml - -This configuration can be passed globally (i.e. for the ``Series`` object) to apply for all datasets. -Alternatively, it can also be passed for single ``Dataset`` objects to only apply for single datasets. - +.. _backendconfig-other: Other backends ^^^^^^^^^^^^^^ diff --git a/docs/source/details/config_layout.json b/docs/source/details/config_layout.json index 382060f364..84b45e938b 100644 --- a/docs/source/details/config_layout.json +++ b/docs/source/details/config_layout.json @@ -1,5 +1,4 @@ { - "adios1": "put ADIOS config here", "adios2": "put ADIOS2 config here", "hdf5": "put HDF5 config here", "json": "put JSON config here" diff --git a/docs/source/details/config_layout.toml b/docs/source/details/config_layout.toml index d44522c3bb..2dafa05ade 100644 --- a/docs/source/details/config_layout.toml +++ b/docs/source/details/config_layout.toml @@ -1,6 +1,3 @@ -[adios1] -# put ADIOS config here - [adios2] # put ADIOS2 config here diff --git a/docs/source/details/mpi.rst b/docs/source/details/mpi.rst index 5c0f1674fa..b94e886fac 100644 --- a/docs/source/details/mpi.rst +++ b/docs/source/details/mpi.rst @@ -38,7 +38,7 @@ Functionality Behavior Description .. [2] Dataset declarations in :ref:`parallel HDF5 ` are only non-collective if :ref:`chunking ` is set to ``none`` (``auto`` by default). Otherwise these operations are collective. -.. [3] :ref:`HDF5 ` only supports collective attribute definitions/writes; :ref:`ADIOS1 ` and :ref:`ADIOS2 ` attributes can be written independently. +.. [3] :ref:`HDF5 ` only supports collective attribute definitions/writes; :ref:`ADIOS2 ` attributes can be written independently. If you want to support all backends equally, treat as a collective operation. Note that openPMD represents constant record components with attributes, thus inheriting this for ``::makeConstant``. diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index 018857fc9e..ab03a1cfcb 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -14,7 +14,6 @@ CMake Option Values Description ============================== =============== ======================================================================== ``openPMD_USE_MPI`` **AUTO**/ON/OFF Parallel, Multi-Node I/O for clusters ``openPMD_USE_HDF5`` **AUTO**/ON/OFF HDF5 backend (``.h5`` files) -``openPMD_USE_ADIOS1`` AUTO/ON/**OFF** ADIOS1 backend (``.bp`` files up to version BP3) - deprecated ``openPMD_USE_ADIOS2`` **AUTO**/ON/OFF ADIOS2 backend (``.bp`` files in BP3, BP4 or higher) ``openPMD_USE_PYTHON`` **AUTO**/ON/OFF Enable Python bindings ``openPMD_USE_INVASIVE_TESTS`` ON/**OFF** Enable unit tests that modify source code :sup:`1` @@ -42,7 +41,6 @@ CMake Option Values Description ============================== =============== ================================================== ``openPMD_BUILD_SHARED_LIBS`` **ON**/OFF Build the C++ API as shared library ``HDF5_USE_STATIC_LIBRARIES`` ON/**OFF** Require static HDF5 library -``ADIOS_USE_STATIC_LIBS`` ON/**OFF** Require static ADIOS1 library ============================== =============== ================================================== Note that python modules (``openpmd_api.cpython.[...].so`` or ``openpmd_api.pyd``) are always dynamic libraries. diff --git a/docs/source/index.rst b/docs/source/index.rst index 37b1e4c630..1d6ad14703 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,7 @@ openPMD is an open meta-data schema that provides meaning and self-description f See `the openPMD standard `_ for details of this schema. This library provides a reference API for openPMD data handling. -Since openPMD is a schema (or markup) on top of portable, hierarchical file formats, this library implements various backends such as HDF5, ADIOS1, ADIOS2 and JSON. +Since openPMD is a schema (or markup) on top of portable, hierarchical file formats, this library implements various backends such as HDF5, ADIOS2 and JSON. Writing & reading through those backends and their associated files is supported for serial and `MPI-parallel `_ workflows. .. raw:: html diff --git a/docs/source/install/install.rst b/docs/source/install/install.rst index 176924e976..48b9f6c3e9 100644 --- a/docs/source/install/install.rst +++ b/docs/source/install/install.rst @@ -34,7 +34,7 @@ A package for openPMD-api is available via the `Spack `_ packa .. code-block:: bash - # optional: +python +adios1 -adios2 -hdf5 -mpi + # optional: +python -adios2 -hdf5 -mpi spack install openpmd-api spack load openpmd-api @@ -72,7 +72,7 @@ A package for openPMD-api is available via the `Homebrew `_/`L brew tap openpmd/openpmd brew install openpmd-api -Brew ship only the latest release, includes (Open)MPI support and lacks the ADIOS1 backend. +Brew ship only the latest release and includes (Open)MPI support. .. _install-pypi: @@ -85,7 +85,7 @@ Using the PyPI Package A package for openPMD-api is available via the Python Package Index (`PyPI `_). -On very old macOS versions (<10.9) or on exotic processor architectures, this install method *compiles from source* against the found installations of HDF5, ADIOS1, ADIOS2, and/or MPI (in system paths, from other package managers, or loaded via a module system, ...). +On very old macOS versions (<10.9) or on exotic processor architectures, this install method *compiles from source* against the found installations of HDF5, ADIOS2, and/or MPI (in system paths, from other package managers, or loaded via a module system, ...). .. code-block:: bash diff --git a/docs/source/usage/firstread.rst b/docs/source/usage/firstread.rst index f3edf391f3..f27e3c595b 100644 --- a/docs/source/usage/firstread.rst +++ b/docs/source/usage/firstread.rst @@ -78,7 +78,7 @@ Open Open an existing openPMD series in ``data.h5``. Further file formats than ``.h5`` (`HDF5 `_) are supported: -``.bp`` (`ADIOS1 `_/`ADIOS2 `_) or ``.json`` (`JSON `_). +``.bp`` (`ADIOS2 `_) or ``.json`` (`JSON `_). C++17 ^^^^^ diff --git a/docs/source/usage/firstwrite.rst b/docs/source/usage/firstwrite.rst index 193530f09f..d339d00ad1 100644 --- a/docs/source/usage/firstwrite.rst +++ b/docs/source/usage/firstwrite.rst @@ -76,7 +76,7 @@ Open Write into a new openPMD series in ``myOutput/data_<00...N>.h5``. Further file formats than ``.h5`` (`HDF5 `_) are supported: -``.bp`` (`ADIOS1 `_/`ADIOS2 `_) or ``.json`` (`JSON `_). +``.bp`` (`ADIOS2 `_) or ``.json`` (`JSON `_). C++17 ^^^^^ diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index 64194629ba..bf5027b53b 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -39,11 +39,10 @@ The openPMD-api distinguishes between a number of different access modes: New datasets and iterations will be inserted as needed. Not fully supported by all backends: - * ADIOS1: Automatically coerced to *Create* mode if the file does not exist yet and to *Read-only* mode if it exists. * ADIOS2: Automatically coerced to *Create* mode if the file does not exist yet and to *Read-only* mode if it exists. Since this happens on a per-file level, this mode allows to read from existing iterations and write to new iterations at the same time in file-based iteration encoding. -* **Append mode**: Restricted mode for appending new iterations to an existing Series that is supported by all backends at least in file-based iteration encoding, and by all but ADIOS1 in other encodings. +* **Append mode**: Restricted mode for appending new iterations to an existing Series that is supported by all backends in all encodings. The API is equivalent to that of the *Create* mode, meaning that no reading is supported whatsoever. If the Series does not exist yet, this behaves equivalently to the *Create* mode. Existing iterations will not be deleted, newly-written iterations will be inserted. diff --git a/examples/8_benchmark_parallel.cpp b/examples/8_benchmark_parallel.cpp index 7cad63f7d7..5adf2512ff 100644 --- a/examples/8_benchmark_parallel.cpp +++ b/examples/8_benchmark_parallel.cpp @@ -80,7 +80,7 @@ int main(int argc, char *argv[]) // given that you provide it with an appropriate DatasetFillerProvider // (template parameter of the Benchmark class). using type = uint64_t; -#if openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2 || openPMD_HAVE_HDF5 +#if openPMD_HAVE_ADIOS2 || openPMD_HAVE_HDF5 openPMD::Datatype dt = openPMD::determineDatatype(); #endif @@ -141,7 +141,7 @@ int main(int argc, char *argv[]) // * The number of iterations. Effectively, the benchmark will be repeated // for this many // times. -#if openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2 +#if openPMD_HAVE_ADIOS2 benchmark.addConfiguration( R"({"adios2": {"dataset":{"operators":[{"type": "blosc"}]}}})", "bp", diff --git a/include/openPMD/Error.hpp b/include/openPMD/Error.hpp index f503dd2294..c50e2918b6 100644 --- a/include/openPMD/Error.hpp +++ b/include/openPMD/Error.hpp @@ -44,7 +44,7 @@ namespace error * @brief An operation was requested that is not supported in a specific * backend. * - * Example: Append mode is not available in ADIOS1. + * Example: Append mode is not available in JSON. */ class OperationUnsupportedInBackend : public Error { diff --git a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp deleted file mode 100644 index 62e9f00493..0000000000 --- a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp +++ /dev/null @@ -1,279 +0,0 @@ -/* Copyright 2018-2021 Fabian Koller - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/IO/ADIOS/ADIOS1FilePosition.hpp" -#include "openPMD/ThrowError.hpp" -#include "openPMD/auxiliary/StringManip.hpp" -#include "openPMD/backend/Attribute.hpp" -#include "openPMD/backend/Writable.hpp" - -#include - -#include -#include -#include -#include -#include -#include - -namespace openPMD -{ -inline std::string -getBP1Extent(Extent const &e, std::string const &delimiter = ",") -{ - switch (e.size()) - { - case 0: - return ""; - case 1: - return std::to_string(e[0]); - default: - std::ostringstream os; - std::for_each( - e.begin(), e.end() - 1, [&os, &delimiter](std::uint64_t const ext) { - os << std::to_string(ext) << delimiter; - }); - os << std::to_string(*e.rbegin()); - return os.str(); - } -} - -inline std::string -getZerosLikeBP1Extent(Extent const &e, std::string const &delimiter = ",") -{ - switch (e.size()) - { - case 0: - return ""; - case 1: - return "0"; - default: - std::ostringstream os; - std::for_each( - e.begin(), e.end() - 1, [&os, &delimiter](std::uint64_t const) { - os << "0" << delimiter; - }); - os << "0"; - return os.str(); - } -} - -inline ADIOS_DATATYPES getBP1DataType(Datatype dtype) -{ - using DT = Datatype; - - // note the ill-named fixed-byte adios_... types - // https://github.com/ornladios/ADIOS/issues/187 - switch (dtype) - { - case DT::CHAR: - case DT::VEC_CHAR: - case DT::SCHAR: - case DT::VEC_SCHAR: - return adios_byte; - case DT::UCHAR: - case DT::VEC_UCHAR: - case DT::BOOL: - return adios_unsigned_byte; - case DT::SHORT: - case DT::VEC_SHORT: - if (sizeof(short) == 2u) - return adios_short; - else if (sizeof(short) == 4u) - return adios_integer; - else if (sizeof(long) == 8u) - return adios_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::SHORT found."); - case DT::INT: - case DT::VEC_INT: - if (sizeof(int) == 2u) - return adios_short; - else if (sizeof(int) == 4u) - return adios_integer; - else if (sizeof(int) == 8u) - return adios_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::INT found."); - case DT::LONG: - case DT::VEC_LONG: - if (sizeof(long) == 2u) - return adios_short; - else if (sizeof(long) == 4u) - return adios_integer; - else if (sizeof(long) == 8u) - return adios_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::LONG found."); - case DT::LONGLONG: - case DT::VEC_LONGLONG: - if (sizeof(long long) == 2u) - return adios_short; - else if (sizeof(long long) == 4u) - return adios_integer; - else if (sizeof(long long) == 8u) - return adios_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::LONGLONG found."); - case DT::USHORT: - case DT::VEC_USHORT: - if (sizeof(unsigned short) == 2u) - return adios_unsigned_short; - else if (sizeof(unsigned short) == 4u) - return adios_unsigned_integer; - else if (sizeof(unsigned long) == 8u) - return adios_unsigned_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::USHORT found."); - case DT::UINT: - case DT::VEC_UINT: - if (sizeof(unsigned int) == 2u) - return adios_unsigned_short; - else if (sizeof(unsigned int) == 4u) - return adios_unsigned_integer; - else if (sizeof(unsigned int) == 8u) - return adios_unsigned_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::UINT found."); - case DT::ULONG: - case DT::VEC_ULONG: - if (sizeof(unsigned long) == 2u) - return adios_unsigned_short; - else if (sizeof(unsigned long) == 4u) - return adios_unsigned_integer; - else if (sizeof(unsigned long) == 8u) - return adios_unsigned_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::ULONG found."); - case DT::ULONGLONG: - case DT::VEC_ULONGLONG: - if (sizeof(unsigned long long) == 2u) - return adios_unsigned_short; - else if (sizeof(unsigned long long) == 4u) - return adios_unsigned_integer; - else if (sizeof(unsigned long long) == 8u) - return adios_unsigned_long; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "No native equivalent for Datatype::ULONGLONG found."); - case DT::FLOAT: - case DT::VEC_FLOAT: - return adios_real; - case DT::DOUBLE: - case DT::ARR_DBL_7: - case DT::VEC_DOUBLE: - return adios_double; - case DT::LONG_DOUBLE: - case DT::VEC_LONG_DOUBLE: - return adios_long_double; - case DT::CFLOAT: - case DT::VEC_CFLOAT: - return adios_complex; - case DT::CDOUBLE: - case DT::VEC_CDOUBLE: - return adios_double_complex; - case DT::CLONG_DOUBLE: - case DT::VEC_CLONG_DOUBLE: - error::throwOperationUnsupportedInBackend( - "ADIOS1", "No native equivalent for Datatype::CLONG_DOUBLE found."); - case DT::STRING: - return adios_string; - case DT::VEC_STRING: - return adios_string_array; - case DT::UNDEFINED: - throw std::runtime_error("Unknown Attribute datatype (ADIOS datatype)"); - default: - throw std::runtime_error("Datatype not implemented in ADIOS IO"); - } -} - -inline std::string concrete_bp1_file_position(Writable *w) -{ - std::stack hierarchy; - if (!w->abstractFilePosition) - w = w->parent; - while (w) - { - hierarchy.push(w); - w = w->parent; - } - - std::string pos; - while (!hierarchy.empty()) - { - auto const tmp_ptr = std::dynamic_pointer_cast( - hierarchy.top()->abstractFilePosition); - if (tmp_ptr == nullptr) - throw std::runtime_error( - "Dynamic pointer cast returned a nullptr!"); - pos += tmp_ptr->location; - hierarchy.pop(); - } - - return auxiliary::replace_all(pos, "//", "/"); -} - -inline std::string -getEnvNum(std::string const &key, std::string const &defaultValue) -{ - char const *env = std::getenv(key.c_str()); - if (env != nullptr) - { - char const *tmp = env; - while (tmp) - { - if (isdigit(*tmp)) - ++tmp; - else - { - std::cerr << key << " is invalid" << std::endl; - break; - } - } - if (!tmp) - return std::string(env, std::strlen(env)); - else - return defaultValue; - } - else - return defaultValue; -} - -template -inline Attribute readVectorAttributeInternal(void *data, int size) -{ - auto d = reinterpret_cast(data); - std::vector v; - v.resize(size); - for (int i = 0; i < size; ++i) - v[i] = d[i]; - return Attribute(v); -} -} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS1FilePosition.hpp b/include/openPMD/IO/ADIOS/ADIOS1FilePosition.hpp deleted file mode 100644 index 0d1d7ab619..0000000000 --- a/include/openPMD/IO/ADIOS/ADIOS1FilePosition.hpp +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/IO/AbstractFilePosition.hpp" - -#include - -namespace openPMD -{ -struct ADIOS1FilePosition : public AbstractFilePosition -{ - ADIOS1FilePosition(std::string const &s) : location{s} - {} - - std::string location; -}; // ADIOS1FilePosition -} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp deleted file mode 100644 index bef874c607..0000000000 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/IO/AbstractIOHandler.hpp" -#include "openPMD/auxiliary/Export.hpp" -#include "openPMD/auxiliary/JSON_internal.hpp" -#include "openPMD/config.hpp" - -#include -#include -#include -#if openPMD_HAVE_ADIOS1 -#include -#endif - -namespace openPMD -{ -class OPENPMDAPI_EXPORT ADIOS1IOHandlerImpl; - -#if openPMD_HAVE_ADIOS1 -class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler -{ - friend class ADIOS1IOHandlerImpl; - -public: - ADIOS1IOHandler(std::string path, Access, json::TracingJSON); - ~ADIOS1IOHandler() override; - - std::string backendName() const override - { - return "ADIOS1"; - } - - std::future flush(internal::ParsedFlushParams &) override; - - void enqueue(IOTask const &) override; - -private: - std::queue m_setup; - std::unique_ptr m_impl; -}; // ADIOS1IOHandler -#else -class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler -{ - friend class ADIOS1IOHandlerImpl; - -public: - ADIOS1IOHandler(std::string path, Access, json::TracingJSON); - ~ADIOS1IOHandler() override; - - std::string backendName() const override - { - return "DUMMY_ADIOS1"; - } - - std::future flush(internal::ParsedFlushParams &) override; - -private: - std::unique_ptr m_impl; -}; // ADIOS1IOHandler -#endif -} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp deleted file mode 100644 index 42305bc78c..0000000000 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/IO/AbstractIOHandler.hpp" -#include "openPMD/auxiliary/Export.hpp" -#include "openPMD/config.hpp" - -#if openPMD_HAVE_ADIOS1 -#include "openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp" -#endif - -#include -#include -#include -#if openPMD_HAVE_ADIOS1 -#include -#include -#endif - -namespace openPMD -{ -#if openPMD_HAVE_ADIOS1 -class OPENPMDAPI_EXPORT ADIOS1IOHandlerImpl - : public CommonADIOS1IOHandlerImpl -{ -private: - using Base_t = CommonADIOS1IOHandlerImpl; - -public: - ADIOS1IOHandlerImpl(AbstractIOHandler *, json::TracingJSON); - virtual ~ADIOS1IOHandlerImpl(); - - virtual void init(); - - std::future flush(); - - virtual int64_t open_write(Writable *); - virtual ADIOS_FILE *open_read(std::string const &name); - int64_t initialize_group(std::string const &name); -}; // ADIOS1IOHandlerImpl -#else -class OPENPMDAPI_EXPORT ADIOS1IOHandlerImpl -{}; // ADIOS1IOHandlerImpl -#endif -} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp deleted file mode 100644 index d1a079b17f..0000000000 --- a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp +++ /dev/null @@ -1,131 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller and Franz Poeschel - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/config.hpp" - -#if openPMD_HAVE_ADIOS1 - -#include "openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp" -#include "openPMD/IO/ADIOS/ADIOS1FilePosition.hpp" -#include "openPMD/IO/AbstractIOHandler.hpp" -#include "openPMD/IO/AbstractIOHandlerImpl.hpp" -#include "openPMD/auxiliary/DerefDynamicCast.hpp" -#include "openPMD/auxiliary/Filesystem.hpp" -#include "openPMD/auxiliary/JSON_internal.hpp" -#include "openPMD/auxiliary/Memory.hpp" -#include "openPMD/auxiliary/StringManip.hpp" - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace openPMD -{ -template // CRT pattern -class CommonADIOS1IOHandlerImpl : public AbstractIOHandlerImpl -{ -public: - void - createFile(Writable *, Parameter const &) override; - void checkFile(Writable *, Parameter &) override; - void - createPath(Writable *, Parameter const &) override; - void createDataset( - Writable *, Parameter const &) override; - void extendDataset( - Writable *, Parameter const &) override; - void openFile(Writable *, Parameter &) override; - void - closeFile(Writable *, Parameter const &) override; - void availableChunks( - Writable *, Parameter &) override; - void openPath(Writable *, Parameter const &) override; - void openDataset(Writable *, Parameter &) override; - void - deleteFile(Writable *, Parameter const &) override; - void - deletePath(Writable *, Parameter const &) override; - void deleteDataset( - Writable *, Parameter const &) override; - void deleteAttribute( - Writable *, Parameter const &) override; - void - writeDataset(Writable *, Parameter &) override; - void writeAttribute( - Writable *, Parameter const &) override; - void readDataset(Writable *, Parameter &) override; - void readAttribute(Writable *, Parameter &) override; - void listPaths(Writable *, Parameter &) override; - void - listDatasets(Writable *, Parameter &) override; - void listAttributes(Writable *, Parameter &) override; - void - deregister(Writable *, Parameter const &) override; - - void close(int64_t); - void close(ADIOS_FILE *); - void - flush_attribute(int64_t group, std::string const &name, Attribute const &); - -protected: - template - CommonADIOS1IOHandlerImpl(Args &&...args) - : AbstractIOHandlerImpl{std::forward(args)...} - {} - - ADIOS_READ_METHOD m_readMethod; - std::unordered_map > m_filePaths; - std::unordered_map, int64_t> m_groups; - std::unordered_map, bool> m_existsOnDisk; - std::unordered_map, int64_t> - m_openWriteFileHandles; - std::unordered_map, ADIOS_FILE *> - m_openReadFileHandles; - struct ScheduledRead - { - ADIOS_SELECTION *selection; - std::shared_ptr data; // needed to avoid early freeing - }; - std::unordered_map > - m_scheduledReads; - std::unordered_map > - m_attributeWrites; - // config options - std::string m_defaultTransform; - /** - * Call this function to get adios file id for a Writable. Will create one - * if does not exist - * @return returns an adios file id. - */ - int64_t GetFileHandle(Writable *); - - void initJson(json::TracingJSON); -}; // ParallelADIOS1IOHandlerImpl -} // namespace openPMD - -#endif diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp deleted file mode 100644 index e28122582b..0000000000 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/IO/AbstractIOHandler.hpp" -#include "openPMD/auxiliary/Export.hpp" -#include "openPMD/auxiliary/JSON_internal.hpp" -#include "openPMD/config.hpp" - -#include -#include -#include -#if openPMD_HAVE_ADIOS1 -#include -#endif - -namespace openPMD -{ -class OPENPMDAPI_EXPORT ParallelADIOS1IOHandlerImpl; - -class OPENPMDAPI_EXPORT ParallelADIOS1IOHandler : public AbstractIOHandler -{ - friend class ParallelADIOS1IOHandlerImpl; - -public: -#if openPMD_HAVE_MPI - ParallelADIOS1IOHandler( - std::string path, Access, json::TracingJSON, MPI_Comm); -#else - ParallelADIOS1IOHandler(std::string path, Access, json::TracingJSON); -#endif - ~ParallelADIOS1IOHandler() override; - - std::string backendName() const override - { - return "MPI_ADIOS1"; - } - - std::future flush(internal::ParsedFlushParams &) override; -#if openPMD_HAVE_ADIOS1 - void enqueue(IOTask const &) override; -#endif - -private: -#if openPMD_HAVE_ADIOS1 - std::queue m_setup; -#endif - std::unique_ptr m_impl; -}; // ParallelADIOS1IOHandler - -} // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp deleted file mode 100644 index 0b1bb2ca34..0000000000 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#pragma once - -#include "openPMD/IO/AbstractIOHandler.hpp" -#include "openPMD/auxiliary/Export.hpp" -#include "openPMD/auxiliary/JSON_internal.hpp" -#include "openPMD/config.hpp" - -#if openPMD_HAVE_ADIOS1 && openPMD_HAVE_MPI -#include "openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp" -#endif - -#include -#include -#include -#if openPMD_HAVE_ADIOS1 -#include -#include -#endif - -namespace openPMD -{ -#if openPMD_HAVE_ADIOS1 && openPMD_HAVE_MPI -class OPENPMDAPI_EXPORT ParallelADIOS1IOHandlerImpl - : public CommonADIOS1IOHandlerImpl -{ -private: - using Base_t = CommonADIOS1IOHandlerImpl; - -public: - ParallelADIOS1IOHandlerImpl( - AbstractIOHandler *, json::TracingJSON, MPI_Comm); - virtual ~ParallelADIOS1IOHandlerImpl(); - - virtual void init(); - - std::future flush(); - - virtual int64_t open_write(Writable *); - virtual ADIOS_FILE *open_read(std::string const &name); - int64_t initialize_group(std::string const &name); - -protected: - MPI_Comm m_mpiComm; - MPI_Info m_mpiInfo; -}; // ParallelADIOS1IOHandlerImpl -#else -class OPENPMDAPI_EXPORT ParallelADIOS1IOHandlerImpl -{}; // ParallelADIOS1IOHandlerImpl -#endif - -} // namespace openPMD diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 697e73f788..43ec4d04a1 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -29,7 +29,6 @@ namespace openPMD enum class Format { HDF5, - ADIOS1, ADIOS2_BP, ADIOS2_BP4, ADIOS2_BP5, diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index a4bc694e16..3443b27d7a 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -136,7 +136,6 @@ class RecordComponent : public BaseRecordComponent * * Backend support for resizing datasets: * * JSON: Supported - * * ADIOS1: Unsupported * * ADIOS2: Supported as of ADIOS2 2.7.0 * * HDF5: (Currently) unsupported. * Will be probably supported as soon as chunking is supported in HDF5. diff --git a/include/openPMD/ThrowError.hpp b/include/openPMD/ThrowError.hpp index 4e48e9bfdc..f2695f7ae0 100644 --- a/include/openPMD/ThrowError.hpp +++ b/include/openPMD/ThrowError.hpp @@ -19,15 +19,6 @@ * If not, see . */ -/* - * For objects that must not include Error.hpp but still need to throw errors. - * In some exotic compiler configurations (clang-6 with libc++), - * including Error.hpp into the ADIOS1 backend leads to incompatible error type - * symbols. - * So, use only the functions defined in here in the ADIOS1 backend. - * Definitions are in Error.cpp. - */ - #pragma once #include "openPMD/auxiliary/Export.hpp" diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index bc0e35bc31..7ee1eaa341 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -79,9 +79,6 @@ class Writable final friend class Record; friend class AbstractIOHandlerImpl; template - friend class CommonADIOS1IOHandlerImpl; - friend class ADIOS1IOHandlerImpl; - friend class ParallelADIOS1IOHandlerImpl; friend class ADIOS2IOHandlerImpl; friend class HDF5IOHandlerImpl; friend class ParallelHDF5IOHandlerImpl; diff --git a/include/openPMD/config.hpp.in b/include/openPMD/config.hpp.in index 8da1b5e6fc..8df5dae9de 100644 --- a/include/openPMD/config.hpp.in +++ b/include/openPMD/config.hpp.in @@ -35,7 +35,7 @@ #endif #ifndef openPMD_HAVE_ADIOS1 -#cmakedefine01 openPMD_HAVE_ADIOS1 +#define openPMD_HAVE_ADIOS1 0 #endif #ifndef openPMD_HAVE_ADIOS2 diff --git a/src/Format.cpp b/src/Format.cpp index 7f30e1a793..d5a8acf5f3 100644 --- a/src/Format.cpp +++ b/src/Format.cpp @@ -32,28 +32,7 @@ Format determineFormat(std::string const &filename) if (auxiliary::ends_with(filename, ".h5")) return Format::HDF5; if (auxiliary::ends_with(filename, ".bp")) - { - auto const bp_backend = auxiliary::getEnvString( - "OPENPMD_BP_BACKEND", -#if openPMD_HAVE_ADIOS2 - "ADIOS2" -#elif openPMD_HAVE_ADIOS1 - "ADIOS1" -#else - "ADIOS2" -#endif - ); - - if (bp_backend == "ADIOS2") - return Format::ADIOS2_BP; - else if (bp_backend == "ADIOS1") - return Format::ADIOS1; - else - throw std::runtime_error( - "Environment variable OPENPMD_BP_BACKEND for .bp backend is " - "neither ADIOS1 nor ADIOS2: " + - bp_backend); - } + return Format::ADIOS2_BP; if (auxiliary::ends_with(filename, ".bp4")) return Format::ADIOS2_BP4; if (auxiliary::ends_with(filename, ".bp5")) @@ -75,7 +54,6 @@ std::string suffix(Format f) { case Format::HDF5: return ".h5"; - case Format::ADIOS1: case Format::ADIOS2_BP: return ".bp"; case Format::ADIOS2_BP4: diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp deleted file mode 100644 index 1e13fc1c82..0000000000 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ /dev/null @@ -1,467 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller, Axel Huebl - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ - -#include "openPMD/IO/ADIOS/ADIOS1IOHandler.hpp" -#include "openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp" - -#if openPMD_HAVE_ADIOS1 -#include "openPMD/IO/AbstractIOHandlerImpl.hpp" - -#include "openPMD/IO/IOTask.hpp" -#include -#include -#include -#include -#include -#include -#endif - -namespace openPMD -{ -#if openPMD_HAVE_ADIOS1 -#if openPMD_USE_VERIFY -#define VERIFY(CONDITION, TEXT) \ - { \ - if (!(CONDITION)) \ - throw std::runtime_error((TEXT)); \ - } -#else -#define VERIFY(CONDITION, TEXT) \ - do \ - { \ - (void)sizeof(CONDITION); \ - } while (0) -#endif - -ADIOS1IOHandlerImpl::ADIOS1IOHandlerImpl( - AbstractIOHandler *handler, json::TracingJSON json) - : Base_t(handler) -{ - initJson(std::move(json)); -} - -ADIOS1IOHandlerImpl::~ADIOS1IOHandlerImpl() -{ - for (auto &f : m_openReadFileHandles) - close(f.second); - m_openReadFileHandles.clear(); - - if (access::write(m_handler->m_backendAccess)) - { - for (auto &group : m_attributeWrites) - for (auto &att : group.second) - flush_attribute(group.first, att.first, att.second); - - for (auto &f : m_openWriteFileHandles) - close(f.second); - m_openWriteFileHandles.clear(); - } - - int status; - status = adios_read_finalize_method(m_readMethod); - if (status != err_no_error) - std::cerr << "Internal error: Failed to finalize ADIOS reading method " - "(serial)\n"; - - status = adios_finalize(0); - if (status != err_no_error) - std::cerr << "Internal error: Failed to finalize ADIOS (serial)\n"; -} - -std::future ADIOS1IOHandlerImpl::flush() -{ - using namespace auxiliary; - - auto handler = dynamic_cast(m_handler); - while (!handler->m_setup.empty()) - { - IOTask &i = handler->m_setup.front(); - try - { - switch (i.operation) - { - using O = Operation; - case O::CREATE_FILE: - createFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CHECK_FILE: - checkFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CREATE_PATH: - createPath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::OPEN_PATH: - openPath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CREATE_DATASET: - createDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::WRITE_ATT: - writeAttribute( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::OPEN_FILE: - openFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::KEEP_SYNCHRONOUS: - keepSynchronous( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DEREGISTER: - deregister( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - default: - VERIFY( - false, - "[ADIOS1] Internal error: Wrong operation in ADIOS setup " - "queue"); - } - } - catch (...) - { - std::cerr << "[AbstractIOHandlerImpl] IO Task " - << internal::operationAsString(i.operation) - << " failed with exception. Clearing IO queue and " - "passing on the exception." - << std::endl; - while (!m_handler->m_work.empty()) - { - m_handler->m_work.pop(); - } - throw; - } - handler->m_setup.pop(); - } - - while (!handler->m_work.empty()) - { - using namespace auxiliary; - - IOTask &i = handler->m_work.front(); - try - { - switch (i.operation) - { - using O = Operation; - case O::EXTEND_DATASET: - extendDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CLOSE_PATH: - closePath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::OPEN_DATASET: - openDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CLOSE_FILE: - closeFile( - i.writable, - *dynamic_cast *>( - i.parameter.get())); - break; - case O::DELETE_FILE: - deleteFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DELETE_PATH: - deletePath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DELETE_DATASET: - deleteDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DELETE_ATT: - deleteAttribute( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::WRITE_DATASET: - writeDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::READ_DATASET: - readDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::GET_BUFFER_VIEW: - getBufferView( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::READ_ATT: - readAttribute( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::LIST_PATHS: - listPaths( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::LIST_DATASETS: - listDatasets( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::LIST_ATTS: - listAttributes( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::ADVANCE: - advance( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::AVAILABLE_CHUNKS: - availableChunks( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - default: - VERIFY( - false, - "[ADIOS1] Internal error: Wrong operation in ADIOS work " - "queue"); - } - } - catch (...) - { - std::cerr << "[AbstractIOHandlerImpl] IO Task " - << internal::operationAsString(i.operation) - << " failed with exception. Clearing IO queue and " - "passing on the exception." - << std::endl; - while (!m_handler->m_work.empty()) - { - m_handler->m_work.pop(); - } - throw; - } - handler->m_work.pop(); - } - - int status; - for (auto &file : m_scheduledReads) - { - status = adios_perform_reads(file.first, 1); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to perform ADIOS reads during " - "dataset reading"); - - for (auto &sel : file.second) - adios_selection_delete(sel.selection); - } - m_scheduledReads.clear(); - - return std::future(); -} - -void ADIOS1IOHandlerImpl::init() -{ - int status; - status = adios_init_noxml(MPI_COMM_NULL); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to initialize ADIOS"); - - m_readMethod = ADIOS_READ_METHOD_BP; - status = adios_read_init_method(m_readMethod, MPI_COMM_NULL, ""); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to initialize ADIOS reading method"); -} -#endif - -#if openPMD_HAVE_ADIOS1 -ADIOS1IOHandler::ADIOS1IOHandler( - std::string path, Access at, json::TracingJSON json) - : AbstractIOHandler(std::move(path), at) - , m_impl{new ADIOS1IOHandlerImpl(this, std::move(json))} -{ - m_impl->init(); -} - -ADIOS1IOHandler::~ADIOS1IOHandler() = default; - -std::future ADIOS1IOHandler::flush(internal::ParsedFlushParams &) -{ - return m_impl->flush(); -} - -void ADIOS1IOHandler::enqueue(IOTask const &i) -{ - switch (i.operation) - { - case Operation::CREATE_FILE: - case Operation::CHECK_FILE: - case Operation::CREATE_PATH: - case Operation::OPEN_PATH: - case Operation::CREATE_DATASET: - case Operation::OPEN_FILE: - case Operation::WRITE_ATT: - case Operation::KEEP_SYNCHRONOUS: - case Operation::DEREGISTER: - m_setup.push(i); - return; - default: - m_work.push(i); - return; - } -} - -int64_t ADIOS1IOHandlerImpl::open_write(Writable *writable) -{ - auto res = m_filePaths.find(writable); - if (res == m_filePaths.end()) - res = m_filePaths.find(writable->parent); - - std::string mode; - if (m_existsOnDisk[res->second]) - { - mode = "u"; - /* close the handle that corresponds to the file we want to append to */ - if (m_openReadFileHandles.find(res->second) != - m_openReadFileHandles.end()) - { - close(m_openReadFileHandles[res->second]); - m_openReadFileHandles.erase(res->second); - } - } - else - { - mode = "w"; - m_existsOnDisk[res->second] = true; - } - - int64_t fd = -1; - int status; - status = adios_open( - &fd, - res->second->c_str(), - res->second->c_str(), - mode.c_str(), - MPI_COMM_NULL); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to open_write ADIOS file"); - - return fd; -} - -ADIOS_FILE *ADIOS1IOHandlerImpl::open_read(std::string const &name) -{ - ADIOS_FILE *f = nullptr; - f = adios_read_open_file(name.c_str(), m_readMethod, MPI_COMM_NULL); - VERIFY( - adios_errno != err_file_not_found, - "[ADIOS1] Internal error: ADIOS file not found"); - VERIFY( - f != nullptr, - "[ADIOS1] Internal error: Failed to open_read ADIOS file"); - - return f; -} - -int64_t ADIOS1IOHandlerImpl::initialize_group(std::string const &name) -{ - int status; - int64_t group; - ADIOS_STATISTICS_FLAG noStatistics = adios_stat_no; - status = adios_declare_group(&group, name.c_str(), "", noStatistics); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to declare ADIOS group"); - status = adios_select_method(group, "POSIX", "", ""); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to select ADIOS method"); - return group; -} - -#else -ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at, json::TracingJSON) - : AbstractIOHandler(std::move(path), at) -{ - throw std::runtime_error("openPMD-api built without ADIOS1 support"); -} - -ADIOS1IOHandler::~ADIOS1IOHandler() = default; - -std::future ADIOS1IOHandler::flush(internal::ParsedFlushParams &) -{ - return std::future(); -} -#endif -} // namespace openPMD diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp deleted file mode 100644 index e5e32d2dca..0000000000 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ /dev/null @@ -1,2054 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller, Axel Huebl - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ - -#include "openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp" -#include "openPMD/Error.hpp" - -#if openPMD_HAVE_ADIOS1 - -#include "openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp" -#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp" -#include "openPMD/auxiliary/JSON_internal.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace openPMD -{ - -#if openPMD_USE_VERIFY -#define VERIFY(CONDITION, TEXT) \ - { \ - if (!(CONDITION)) \ - throw std::runtime_error((TEXT)); \ - } -#else -#define VERIFY(CONDITION, TEXT) \ - do \ - { \ - (void)sizeof(CONDITION); \ - } while (0) -#endif - -template -void CommonADIOS1IOHandlerImpl::close(int64_t fd) -{ - int status; - status = adios_close(fd); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to close ADIOS file (open_write)"); -} - -template -void CommonADIOS1IOHandlerImpl::close(ADIOS_FILE *f) -{ - int status; - status = adios_read_close(f); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to close ADIOS file (open_read)"); -} - -template -void CommonADIOS1IOHandlerImpl::flush_attribute( - int64_t group, std::string const &name, Attribute const &att) -{ - auto dtype = att.dtype; - // https://github.com/ComputationalRadiationPhysics/picongpu/pull/1756 - if (dtype == Datatype::BOOL) - dtype = Datatype::UCHAR; - - int nelems = 0; - switch (dtype) - { - using DT = Datatype; - case DT::VEC_CHAR: - nelems = att.get >().size(); - break; - case DT::VEC_SHORT: - nelems = att.get >().size(); - break; - case DT::VEC_INT: - nelems = att.get >().size(); - break; - case DT::VEC_LONG: - nelems = att.get >().size(); - break; - case DT::VEC_LONGLONG: - nelems = att.get >().size(); - break; - case DT::VEC_UCHAR: - nelems = att.get >().size(); - break; - case DT::VEC_SCHAR: - nelems = att.get >().size(); - break; - case DT::VEC_USHORT: - nelems = att.get >().size(); - break; - case DT::VEC_UINT: - nelems = att.get >().size(); - break; - case DT::VEC_ULONG: - nelems = att.get >().size(); - break; - case DT::VEC_ULONGLONG: - nelems = att.get >().size(); - break; - case DT::VEC_FLOAT: - nelems = att.get >().size(); - break; - case DT::VEC_DOUBLE: - nelems = att.get >().size(); - break; - case DT::VEC_LONG_DOUBLE: - nelems = att.get >().size(); - break; - case DT::VEC_STRING: - nelems = att.get >().size(); - break; - case DT::ARR_DBL_7: - nelems = 7; - break; - case DT::UNDEFINED: - throw std::runtime_error( - "[ADIOS1] Unknown Attribute datatype (ADIOS1 Attribute flush)"); - default: - nelems = 1; - } - - auto values = auxiliary::allocatePtr(dtype, nelems); - switch (att.dtype) - { - using DT = Datatype; - case DT::CHAR: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::UCHAR: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::SCHAR: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::SHORT: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::INT: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::LONG: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::LONGLONG: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::USHORT: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::UINT: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::ULONG: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::ULONGLONG: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::FLOAT: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::DOUBLE: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::LONG_DOUBLE: { - auto ptr = reinterpret_cast(values.get()); - *ptr = att.get(); - break; - } - case DT::CFLOAT: { - auto ptr = reinterpret_cast *>(values.get()); - *ptr = att.get >(); - break; - } - case DT::CDOUBLE: { - auto ptr = reinterpret_cast *>(values.get()); - *ptr = att.get >(); - break; - } - case DT::CLONG_DOUBLE: { - throw std::runtime_error( - "[ADIOS1] Unknown Attribute datatype (CLONG_DOUBLE)"); - break; - } - case DT::STRING: { - auto const &v = att.get(); - if (v.empty()) - { - error::throwOperationUnsupportedInBackend( - "ADIOS1", "Empty string attributes not supported."); - } - values = auxiliary::allocatePtr(Datatype::CHAR, v.length() + 1u); - strcpy((char *)values.get(), v.c_str()); - break; - } - case DT::VEC_CHAR: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_SHORT: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_INT: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_LONG: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_LONGLONG: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_UCHAR: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_SCHAR: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_USHORT: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_UINT: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_ULONG: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_ULONGLONG: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_FLOAT: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_DOUBLE: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - case DT::VEC_LONG_DOUBLE: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - ptr[i] = vec[i]; - break; - } - /* not supported by ADIOS 1.13.1: - * https://github.com/ornladios/ADIOS/issues/212 - */ - case DT::VEC_CFLOAT: - case DT::VEC_CDOUBLE: - case DT::VEC_CLONG_DOUBLE: { - throw std::runtime_error( - "[ADIOS1] Arrays of complex attributes are not supported"); - break; - } - case DT::VEC_STRING: { - auto ptr = reinterpret_cast(values.get()); - auto const &vec = att.get >(); - for (size_t i = 0; i < vec.size(); ++i) - { - if (vec[i].empty()) - { - error::throwOperationUnsupportedInBackend( - "ADIOS1", "Empty string attributes not supported."); - } - size_t size = vec[i].size() + 1; - ptr[i] = new char[size]; - strncpy(ptr[i], vec[i].c_str(), size); - } - break; - } - case DT::ARR_DBL_7: { - auto ptr = reinterpret_cast(values.get()); - auto const &arr = att.get >(); - for (size_t i = 0; i < 7; ++i) - ptr[i] = arr[i]; - break; - } - case DT::BOOL: { - auto ptr = reinterpret_cast(values.get()); - *ptr = static_cast(att.get()); - break; - } - case DT::UNDEFINED: - throw std::runtime_error( - "[ADIOS1] Unknown Attribute datatype (ADIOS1 Attribute flush)"); - default: - throw std::runtime_error( - "[ADIOS1] Datatype not implemented in ADIOS IO"); - } - - int status; - status = adios_define_attribute_byvalue( - group, - name.c_str(), - "", - getBP1DataType(att.dtype), - nelems, - values.get()); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to define ADIOS attribute by value"); - - if (att.dtype == Datatype::VEC_STRING) - { - auto ptr = reinterpret_cast(values.get()); - for (int i = 0; i < nelems; ++i) - delete[] ptr[i]; - } -} - -template -void CommonADIOS1IOHandlerImpl::createFile( - Writable *writable, Parameter const ¶meters) -{ - if (access::readOnly(m_handler->m_backendAccess)) - throw std::runtime_error( - "[ADIOS1] Creating a file in read-only mode is not possible."); - - if (!writable->written) - { - if (!auxiliary::directory_exists(m_handler->directory)) - { - bool success = auxiliary::create_directories(m_handler->directory); - VERIFY( - success, - "[ADIOS1] Internal error: Failed to create directories during " - "ADIOS file creation"); - } - - std::string name = m_handler->directory + parameters.name; - if (!auxiliary::ends_with(name, ".bp")) - name += ".bp"; - - if (m_handler->m_backendAccess == Access::APPEND && - auxiliary::file_exists(name)) - { - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "Appending to existing file on disk (use Access::CREATE to " - "overwrite)"); - } - - writable->written = true; - writable->abstractFilePosition = - std::make_shared("/"); - - m_filePaths[writable] = std::make_shared(name); - - /* our control flow allows for more than one open file handle - * if multiple files are opened with the same group, data might be lost - */ - - /* defer actually opening the file handle until the first - * Operation::WRITE_DATASET occurs */ - m_existsOnDisk[m_filePaths[writable]] = false; - - GetFileHandle(writable); - } -} - -template -void CommonADIOS1IOHandlerImpl::checkFile( - Writable *, Parameter ¶meter) -{ - *parameter.fileExists = - Parameter::FileExists::DontKnow; -} - -template -void CommonADIOS1IOHandlerImpl::createPath( - Writable *writable, Parameter const ¶meters) -{ - if (access::readOnly(m_handler->m_backendAccess)) - throw std::runtime_error( - "[ADIOS1] Creating a path in a file opened as read only is not " - "possible."); - - if (!writable->written) - { - /* Sanitize path */ - std::string path = parameters.path; - if (auxiliary::starts_with(path, '/')) - path = auxiliary::replace_first(path, "/", ""); - if (!auxiliary::ends_with(path, '/')) - path += '/'; - - /* ADIOS has no concept for explicitly creating paths. - * They are implicitly created with the paths of variables/attributes. - */ - - writable->written = true; - writable->abstractFilePosition = - std::make_shared(path); - - Writable *position; - if (writable->parent) - position = writable->parent; - else - position = writable; /* root does not have a parent but might still - have to be written */ - auto res = m_filePaths.find(position); - - m_filePaths[writable] = res->second; - } -} - -static std::optional datasetTransform(json::TracingJSON config) -{ - using ret_t = std::optional; - if (!config.json().contains("dataset")) - { - return ret_t{}; - } - config = config["dataset"]; - if (!config.json().contains("transform")) - { - return ret_t{}; - } - config = config["transform"]; - auto maybeRes = json::asStringDynamic(config.json()); - if (maybeRes.has_value()) - { - return std::move(maybeRes.value()); - } - else - { - error::throwBackendConfigSchema( - {"adios1", "dataset", "transform"}, - "Key must convertible to type string."); - } -} - -template -void CommonADIOS1IOHandlerImpl::createDataset( - Writable *writable, Parameter const ¶meters) -{ - if (access::readOnly(m_handler->m_backendAccess)) - throw std::runtime_error( - "[ADIOS1] Creating a dataset in a file opened as read only is not " - "possible."); - - if (!writable->written) - { - /* ADIOS variable definitions require the file to be (re-)opened to take - * effect/not cause errors */ - auto res = m_filePaths.find(writable->parent); - - int64_t group = m_groups[res->second]; - - /* Sanitize name */ - std::string name = parameters.name; - if (auxiliary::starts_with(name, '/')) - name = auxiliary::replace_first(name, "/", ""); - if (auxiliary::ends_with(name, '/')) - name = auxiliary::replace_last(name, "/", ""); - - std::string path = concrete_bp1_file_position(writable) + name; - - size_t ndims = parameters.extent.size(); - - std::vector chunkSize(ndims, ""); - std::vector chunkOffset(ndims, ""); - int64_t id; - for (size_t i = 0; i < ndims; ++i) - { - chunkSize[i] = "/tmp" + path + "_chunkSize" + std::to_string(i); - id = adios_define_var( - group, - chunkSize[i].c_str(), - "", - adios_unsigned_long, - "", - "", - ""); - VERIFY( - id != 0, - "[ADIOS1] Internal error: Failed to define ADIOS variable " - "during Dataset creation"); - chunkOffset[i] = "/tmp" + path + "_chunkOffset" + std::to_string(i); - id = adios_define_var( - group, - chunkOffset[i].c_str(), - "", - adios_unsigned_long, - "", - "", - ""); - VERIFY( - id != 0, - "[ADIOS1] Internal error: Failed to define ADIOS variable " - "during Dataset creation"); - } - - std::string chunkSizeParam = auxiliary::join(chunkSize, ","); - std::string globalSize = getBP1Extent(parameters.extent); - std::string chunkOffsetParam = auxiliary::join(chunkOffset, ","); - id = adios_define_var( - group, - path.c_str(), - "", - getBP1DataType(parameters.dtype), - chunkSizeParam.c_str(), - globalSize.c_str(), - chunkOffsetParam.c_str()); - VERIFY( - id != 0, - "[ADIOS1] Internal error: Failed to define ADIOS variable during " - "Dataset creation"); - - std::string transform = ""; - { - json::TracingJSON options = json::parseOptions( - parameters.options, /* considerFiles = */ false); - if (options.json().contains("adios1")) - { - options = options["adios1"]; - auto maybeTransform = datasetTransform(options); - if (maybeTransform.has_value()) - { - transform = maybeTransform.value(); - } - - parameters.warnUnusedParameters( - options, - "ADIOS1", - "Warning: parts of the backend configuration for " - "ADIOS1 dataset '" + - name + "' remain unused:\n"); - } - } - // Fallback: global option - if (transform.empty()) - { - transform = m_defaultTransform; - } - - if (!transform.empty()) - { - int status; - status = adios_set_transform(id, transform.c_str()); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to set ADIOS transform during " - "Dataset creation"); - } - - writable->written = true; - writable->abstractFilePosition = - std::make_shared(name); - - m_filePaths[writable] = res->second; - } -} - -template -void CommonADIOS1IOHandlerImpl::extendDataset( - Writable *, Parameter const &) -{ - throw std::runtime_error( - "[ADIOS1] Dataset extension not implemented in ADIOS backend"); -} - -template -void CommonADIOS1IOHandlerImpl::openFile( - Writable *writable, Parameter ¶meters) -{ - if (!auxiliary::directory_exists(m_handler->directory)) - error::throwReadError( - error::AffectedObject::File, - error::Reason::Inaccessible, - "ADIOS1", - "Supplied directory is not valid: " + m_handler->directory); - - std::string name = m_handler->directory + parameters.name; - if (!auxiliary::ends_with(name, ".bp")) - name += ".bp"; - - std::shared_ptr filePath; - auto it = std::find_if( - m_filePaths.begin(), - m_filePaths.end(), - [name](std::unordered_map >:: - value_type const &entry) { return *entry.second == name; }); - if (it == m_filePaths.end()) - filePath = std::make_shared(name); - else - filePath = it->second; - - if (m_handler->m_backendAccess == Access::CREATE) - { - // called at Series::flush for iterations that has been flushed before - // this is to make sure to point the Series.m_writer points to this - // iteration so when call Series.flushAttribute(), the attributes can be - // flushed to the iteration level file. - m_filePaths[writable] = filePath; - writable->written = true; - writable->abstractFilePosition = - std::make_shared("/"); - return; - } - /* close the handle that corresponds to the file we want to open */ - if (m_openWriteFileHandles.find(filePath) != m_openWriteFileHandles.end()) - { - close(m_openWriteFileHandles[filePath]); - m_openWriteFileHandles.erase(filePath); - } - - if (m_groups.find(filePath) == m_groups.end()) - m_groups[filePath] = - static_cast(this)->initialize_group(name); - - if (m_openReadFileHandles.find(filePath) == m_openReadFileHandles.end()) - { - ADIOS_FILE *f = static_cast(this)->open_read(name); - m_openReadFileHandles[filePath] = f; - } - - writable->written = true; - writable->abstractFilePosition = std::make_shared("/"); - - m_filePaths[writable] = filePath; - m_existsOnDisk[filePath] = true; -} - -template -void CommonADIOS1IOHandlerImpl::closeFile( - Writable *writable, Parameter const &) -{ - auto myFile = m_filePaths.find(writable); - if (myFile == m_filePaths.end()) - { - return; - } - - // finish write operations - auto myGroup = m_groups.find(myFile->second); - if (myGroup != m_groups.end()) - { - auto attributeWrites = m_attributeWrites.find(myGroup->second); - if (access::write(this->m_handler->m_backendAccess) && - attributeWrites != m_attributeWrites.end()) - { - for (auto &att : attributeWrites->second) - { - flush_attribute(myGroup->second, att.first, att.second); - } - m_attributeWrites.erase(attributeWrites); - } - m_groups.erase(myGroup); - } - - auto handle_write = m_openWriteFileHandles.find(myFile->second); - if (handle_write != m_openWriteFileHandles.end()) - { - close(handle_write->second); - m_openWriteFileHandles.erase(handle_write); - } - - // finish read operations - auto handle_read = m_openReadFileHandles.find(myFile->second); - if (handle_read != m_openReadFileHandles.end()) - { - auto scheduled = m_scheduledReads.find(handle_read->second); - if (scheduled != m_scheduledReads.end()) - { - auto status = adios_perform_reads(scheduled->first, 1); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to perform ADIOS reads during " - "dataset reading"); - - for (auto &sel : scheduled->second) - adios_selection_delete(sel.selection); - m_scheduledReads.erase(scheduled); - } - close(handle_read->second); - m_openReadFileHandles.erase(handle_read); - } - m_existsOnDisk.erase(myFile->second); - m_filePaths.erase(myFile); -} - -template -void CommonADIOS1IOHandlerImpl::availableChunks( - Writable *writable, Parameter ¶ms) -{ - ADIOS_FILE *f; - f = m_openReadFileHandles.at(m_filePaths.at(writable)); - std::string name = concrete_bp1_file_position(writable); - VERIFY( - std::strcmp(f->path, m_filePaths.at(writable)->c_str()) == 0, - "[ADIOS1] Internal Error: Invalid ADIOS read file handle"); - ADIOS_VARINFO *varinfo = adios_inq_var(f, name.c_str()); - VERIFY( - adios_errno == err_no_error, - "[ADIOS1] Internal error: Failed to inquire ADIOS variable while " - "querying available chunks."); - int err = adios_inq_var_blockinfo(f, varinfo); - VERIFY( - err == 0, - "[ADIOS1] Internal error: Failed to obtain ADIOS varinfo while " - "querying available chunks."); - int nblocks = varinfo->nblocks[0]; // we don't use steps, so index 0 is fine - int ndim = varinfo->ndim; - auto &table = *params.chunks; - table.reserve(nblocks); - for (int block = 0; block < nblocks; ++block) - { - ADIOS_VARBLOCK &varblock = varinfo->blockinfo[block]; - Offset offset(ndim); - Extent extent(ndim); - for (int i = 0; i < ndim; ++i) - { - offset[i] = varblock.start[i]; - extent[i] = varblock.count[i]; - } - table.emplace_back(offset, extent, int(varblock.process_id)); - } - adios_free_varinfo(varinfo); -} - -template -void CommonADIOS1IOHandlerImpl::openPath( - Writable *writable, Parameter const ¶meters) -{ - /* Sanitize path */ - std::string path = parameters.path; - if (!path.empty()) - { - if (auxiliary::starts_with(path, '/')) - path = auxiliary::replace_first(path, "/", ""); - if (!auxiliary::ends_with(path, '/')) - path += '/'; - } - - writable->written = true; - writable->abstractFilePosition = std::make_shared(path); - - auto res = writable->parent ? m_filePaths.find(writable->parent) - : m_filePaths.find(writable); - - m_filePaths[writable] = res->second; -} - -template -void CommonADIOS1IOHandlerImpl::openDataset( - Writable *writable, Parameter ¶meters) -{ - ADIOS_FILE *f; - auto res = m_filePaths.find(writable->parent); - f = m_openReadFileHandles.at(res->second); - - /* Sanitize name */ - std::string name = parameters.name; - if (auxiliary::starts_with(name, '/')) - name = auxiliary::replace_first(name, "/", ""); - - std::string datasetname = writable->abstractFilePosition - ? concrete_bp1_file_position(writable) - : concrete_bp1_file_position(writable) + name; - - ADIOS_VARINFO *vi; - vi = adios_inq_var(f, datasetname.c_str()); - std::string error_string("[ADIOS1] Internal error: "); - error_string.append("Failed to inquire about ADIOS variable '") - .append(datasetname) - .append("' during dataset opening"); - VERIFY(adios_errno == err_no_error, error_string); - VERIFY(vi != nullptr, error_string); - - Datatype dtype; - - // note the ill-named fixed-byte adios_... types - // https://github.com/ornladios/ADIOS/issues/187 - switch (vi->type) - { - using DT = Datatype; - case adios_byte: - dtype = DT::CHAR; - break; - case adios_short: - if (sizeof(short) == 2u) - dtype = DT::SHORT; - else if (sizeof(int) == 2u) - dtype = DT::INT; - else if (sizeof(long) == 2u) - dtype = DT::LONG; - else if (sizeof(long long) == 2u) - dtype = DT::LONGLONG; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "No native equivalent for Datatype adios_short found."); - break; - case adios_integer: - if (sizeof(short) == 4u) - dtype = DT::SHORT; - else if (sizeof(int) == 4u) - dtype = DT::INT; - else if (sizeof(long) == 4u) - dtype = DT::LONG; - else if (sizeof(long long) == 4u) - dtype = DT::LONGLONG; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "No native equivalent for Datatype adios_integer found."); - break; - case adios_long: - if (sizeof(short) == 8u) - dtype = DT::SHORT; - else if (sizeof(int) == 8u) - dtype = DT::INT; - else if (sizeof(long) == 8u) - dtype = DT::LONG; - else if (sizeof(long long) == 8u) - dtype = DT::LONGLONG; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "No native equivalent for Datatype adios_long found."); - break; - case adios_unsigned_byte: - dtype = DT::UCHAR; - break; - case adios_unsigned_short: - if (sizeof(unsigned short) == 2u) - dtype = DT::USHORT; - else if (sizeof(unsigned int) == 2u) - dtype = DT::UINT; - else if (sizeof(unsigned long) == 2u) - dtype = DT::ULONG; - else if (sizeof(unsigned long long) == 2u) - dtype = DT::ULONGLONG; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "No native equivalent for Datatype adios_unsigned_short " - "found."); - break; - case adios_unsigned_integer: - if (sizeof(unsigned short) == 4u) - dtype = DT::USHORT; - else if (sizeof(unsigned int) == 4u) - dtype = DT::UINT; - else if (sizeof(unsigned long) == 4u) - dtype = DT::ULONG; - else if (sizeof(unsigned long long) == 4u) - dtype = DT::ULONGLONG; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "No native equivalent for Datatype adios_unsigned_integer " - "found."); - break; - case adios_unsigned_long: - if (sizeof(unsigned short) == 8u) - dtype = DT::USHORT; - else if (sizeof(unsigned int) == 8u) - dtype = DT::UINT; - else if (sizeof(unsigned long) == 8u) - dtype = DT::ULONG; - else if (sizeof(unsigned long long) == 8u) - dtype = DT::ULONGLONG; - else - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "No native equivalent for Datatype adios_unsigned_long found."); - break; - case adios_real: - dtype = DT::FLOAT; - break; - case adios_double: - dtype = DT::DOUBLE; - break; - case adios_long_double: - dtype = DT::LONG_DOUBLE; - break; - case adios_complex: - dtype = DT::CFLOAT; - break; - case adios_double_complex: - dtype = DT::CDOUBLE; - break; - - case adios_string: - case adios_string_array: - default: - error::throwOperationUnsupportedInBackend( - "ADIOS1", - "[ADIOS1] Datatype not implemented for ADIOS dataset writing"); - } - *parameters.dtype = dtype; - - Extent e; - e.resize(vi->ndim); - for (int i = 0; i < vi->ndim; ++i) - e[i] = vi->dims[i]; - *parameters.extent = e; - - writable->written = true; - if (!writable->abstractFilePosition) - { - writable->abstractFilePosition = - std::make_shared(name); - } - - m_openReadFileHandles[res->second] = f; - m_filePaths[writable] = res->second; -} - -template -void CommonADIOS1IOHandlerImpl::deleteFile( - Writable *writable, Parameter const ¶meters) -{ - if (access::readOnly(m_handler->m_backendAccess)) - throw std::runtime_error( - "[ADIOS1] Deleting a file opened as read only is not possible."); - - if (writable->written) - { - auto path = m_filePaths.at(writable); - if (m_openReadFileHandles.find(path) != m_openReadFileHandles.end()) - { - close(m_openReadFileHandles.at(path)); - m_openReadFileHandles.erase(path); - } - if (m_openWriteFileHandles.find(path) != m_openWriteFileHandles.end()) - { - close(m_openWriteFileHandles.at(path)); - m_openWriteFileHandles.erase(path); - } - - std::string name = m_handler->directory + parameters.name; - if (!auxiliary::ends_with(name, ".bp")) - name += ".bp"; - - if (!auxiliary::file_exists(name)) - throw std::runtime_error("[ADIOS1] File does not exist: " + name); - - auxiliary::remove_file(name); - - writable->written = false; - writable->abstractFilePosition.reset(); - - m_filePaths.erase(writable); - } -} - -template -void CommonADIOS1IOHandlerImpl::deletePath( - Writable *, Parameter const &) -{ - throw std::runtime_error( - "[ADIOS1] Path deletion not implemented in ADIOS backend"); -} - -template -void CommonADIOS1IOHandlerImpl::deleteDataset( - Writable *, Parameter const &) -{ - throw std::runtime_error( - "[ADIOS1] Dataset deletion not implemented in ADIOS backend"); -} - -template -void CommonADIOS1IOHandlerImpl::deleteAttribute( - Writable *, Parameter const &) -{ - throw std::runtime_error( - "[ADIOS1] Attribute deletion not implemented in ADIOS backend"); -} - -template -int64_t CommonADIOS1IOHandlerImpl::GetFileHandle(Writable *writable) -{ - auto res = m_filePaths.find(writable); - if (res == m_filePaths.end()) - res = m_filePaths.find(writable->parent); - int64_t fd; - - if (m_openWriteFileHandles.find(res->second) == - m_openWriteFileHandles.end()) - { - std::string name = *(res->second); - m_groups[m_filePaths[writable]] = - static_cast(this)->initialize_group(name); - - fd = static_cast(this)->open_write(writable); - m_openWriteFileHandles[res->second] = fd; - } - else - fd = m_openWriteFileHandles.at(res->second); - - return fd; -} - -template -void CommonADIOS1IOHandlerImpl::writeDataset( - Writable *writable, Parameter ¶meters) -{ - if (access::readOnly(m_handler->m_backendAccess)) - throw std::runtime_error( - "[ADIOS1] Writing into a dataset in a file opened as read-only is " - "not possible."); - - int64_t fd = GetFileHandle(writable); - - std::string name = concrete_bp1_file_position(writable); - - size_t ndims = parameters.extent.size(); - - std::string chunkSize; - std::string chunkOffset; - int status; - for (size_t i = 0; i < ndims; ++i) - { - chunkSize = "/tmp" + name + "_chunkSize" + std::to_string(i); - status = adios_write(fd, chunkSize.c_str(), ¶meters.extent[i]); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to write ADIOS variable during " - "Dataset writing"); - chunkOffset = "/tmp" + name + "_chunkOffset" + std::to_string(i); - status = adios_write(fd, chunkOffset.c_str(), ¶meters.offset[i]); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to write ADIOS variable during " - "Dataset writing"); - } - - status = adios_write(fd, name.c_str(), parameters.data.get()); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to write ADIOS variable during " - "Dataset writing"); -} - -template -void CommonADIOS1IOHandlerImpl::writeAttribute( - Writable *writable, Parameter const ¶meters) -{ - if (parameters.changesOverSteps) - { - // cannot do this - return; - } - if (access::readOnly(m_handler->m_backendAccess)) - throw std::runtime_error( - "[ADIOS1] Writing an attribute in a file opened as read only is " - "not possible."); - - std::string name = concrete_bp1_file_position(writable); - if (!auxiliary::ends_with(name, '/')) - name += '/'; - name += parameters.name; - - auto res = m_filePaths.find(writable); - if (res == m_filePaths.end()) - res = m_filePaths.find(writable->parent); - GetFileHandle(writable); - - int64_t group = m_groups[res->second]; - - auto &attributes = m_attributeWrites[group]; - attributes.erase(name); - attributes.emplace(name, parameters.resource); -} - -template -void CommonADIOS1IOHandlerImpl::readDataset( - Writable *writable, Parameter ¶meters) -{ - switch (parameters.dtype) - { - using DT = Datatype; - case DT::DOUBLE: - case DT::FLOAT: - case DT::CDOUBLE: - case DT::CFLOAT: - case DT::SHORT: - case DT::INT: - case DT::LONG: - case DT::LONGLONG: - case DT::USHORT: - case DT::UINT: - case DT::ULONG: - case DT::ULONGLONG: - case DT::CHAR: - case DT::UCHAR: - case DT::SCHAR: - case DT::BOOL: - break; - case DT::UNDEFINED: - throw std::runtime_error( - "[ADIOS1] Unknown Attribute datatype (ADIOS1 Dataset read)"); - default: - throw std::runtime_error( - "[ADIOS1] Datatype not implemented in ADIOS1 IO"); - } - - ADIOS_FILE *f; - f = m_openReadFileHandles.at(m_filePaths.at(writable)); - VERIFY( - std::strcmp(f->path, m_filePaths.at(writable)->c_str()) == 0, - "[ADIOS1] Internal Error: Invalid ADIOS read file handle"); - - ADIOS_SELECTION *sel; - sel = adios_selection_boundingbox( - parameters.extent.size(), - parameters.offset.data(), - parameters.extent.data()); - VERIFY( - sel != nullptr, - "[ADIOS1] Internal error: Failed to select ADIOS bounding box during " - "dataset reading"); - VERIFY( - adios_errno == err_no_error, - "[ADIOS1] Internal error: Failed to select ADIOS bounding box during " - "dataset reading"); - - std::string varname = concrete_bp1_file_position(writable); - void *data = parameters.data.get(); - - int status; - status = adios_schedule_read(f, sel, varname.c_str(), 0, 1, data); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to schedule ADIOS read during dataset " - "reading"); - VERIFY( - adios_errno == err_no_error, - "[ADIOS1] Internal error: Failed to schedule ADIOS read during dataset " - "reading"); - - m_scheduledReads[f].push_back({sel, parameters.data}); -} - -template -void CommonADIOS1IOHandlerImpl::readAttribute( - Writable *writable, Parameter ¶meters) -{ - if (!writable->written) - throw std::runtime_error( - "[ADIOS1] Internal error: Writable not marked written during " - "attribute reading"); - - ADIOS_FILE *f; - f = m_openReadFileHandles.at(m_filePaths.at(writable)); - - std::string attrname = concrete_bp1_file_position(writable); - if (!auxiliary::ends_with(attrname, '/')) - attrname += "/"; - attrname += parameters.name; - - ADIOS_DATATYPES datatype = adios_unknown; - int size = 0; - void *data = nullptr; - - int status; - status = adios_get_attr(f, attrname.c_str(), &datatype, &size, &data); - if (status != 0) - { - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::NotFound, - "ADIOS1", - attrname); - } - if (datatype == adios_unknown) - { - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::UnexpectedContent, - "ADIOS1", - "Unknown datatype: " + attrname); - } - - // size is returned in number of allocated bytes - // note the ill-named fixed-byte adios_... types - // https://github.com/ornladios/ADIOS/issues/187 - switch (datatype) - { - case adios_byte: - break; - case adios_short: - size /= 2; - break; - case adios_integer: - size /= 4; - break; - case adios_long: - size /= 8; - break; - case adios_unsigned_byte: - break; - case adios_unsigned_short: - size /= 2; - break; - case adios_unsigned_integer: - size /= 4; - break; - case adios_unsigned_long: - size /= 8; - break; - case adios_real: - size /= 4; - break; - case adios_double: - size /= 8; - break; - case adios_long_double: - size /= sizeof(long double); - break; - case adios_complex: - size /= 8; - break; - case adios_double_complex: - size /= 16; - break; - case adios_string: - break; - case adios_string_array: - size /= sizeof(char *); - break; - - default: - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::UnexpectedContent, - "ADIOS1", - "Unsupported datatype: " + attrname); - } - - Datatype dtype; - Attribute a(0); - if (size == 1) - { - switch (datatype) - { - using DT = Datatype; - case adios_byte: - dtype = DT::CHAR; - a = Attribute(*reinterpret_cast(data)); - break; - case adios_short: - if (sizeof(short) == 2u) - { - dtype = DT::SHORT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(int) == 2u) - { - dtype = DT::INT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(long) == 2u) - { - dtype = DT::LONG; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(long long) == 2u) - { - dtype = DT::LONGLONG; - a = Attribute(*reinterpret_cast(data)); - } - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type adios_short: " + - attrname); - break; - case adios_integer: - if (sizeof(short) == 4u) - { - dtype = DT::SHORT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(int) == 4u) - { - dtype = DT::INT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(long) == 4u) - { - dtype = DT::LONG; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(long long) == 4u) - { - dtype = DT::LONGLONG; - a = Attribute(*reinterpret_cast(data)); - } - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type adios_integer: " + - attrname); - break; - case adios_long: - if (sizeof(short) == 8u) - { - dtype = DT::SHORT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(int) == 8u) - { - dtype = DT::INT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(long) == 8u) - { - dtype = DT::LONG; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(long long) == 8u) - { - dtype = DT::LONGLONG; - a = Attribute(*reinterpret_cast(data)); - } - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type adios_long: " + - attrname); - break; - case adios_unsigned_byte: - dtype = DT::UCHAR; - a = Attribute(*reinterpret_cast(data)); - break; - case adios_unsigned_short: - if (sizeof(unsigned short) == 2u) - { - dtype = DT::USHORT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned int) == 2u) - { - dtype = DT::UINT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned long) == 2u) - { - dtype = DT::ULONG; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned long long) == 2u) - { - dtype = DT::ULONGLONG; - a = Attribute(*reinterpret_cast(data)); - } - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type " - "adios_unsigned_short: " + - attrname); - break; - case adios_unsigned_integer: - if (sizeof(unsigned short) == 4u) - { - dtype = DT::USHORT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned int) == 4u) - { - dtype = DT::UINT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned long) == 4u) - { - dtype = DT::ULONG; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned long long) == 4u) - { - dtype = DT::ULONGLONG; - a = Attribute(*reinterpret_cast(data)); - } - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type " - "adios_unsigned_integer: " + - attrname); - break; - case adios_unsigned_long: - if (sizeof(unsigned short) == 8u) - { - dtype = DT::USHORT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned int) == 8u) - { - dtype = DT::UINT; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned long) == 8u) - { - dtype = DT::ULONG; - a = Attribute(*reinterpret_cast(data)); - } - else if (sizeof(unsigned long long) == 8u) - { - dtype = DT::ULONGLONG; - a = Attribute(*reinterpret_cast(data)); - } - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type " - "adios_unsigned_long: " + - attrname); - break; - case adios_real: - dtype = DT::FLOAT; - a = Attribute(*reinterpret_cast(data)); - break; - case adios_double: - dtype = DT::DOUBLE; - a = Attribute(*reinterpret_cast(data)); - break; - case adios_long_double: - dtype = DT::LONG_DOUBLE; - a = Attribute(*reinterpret_cast(data)); - break; - case adios_complex: - dtype = DT::CFLOAT; - a = Attribute(*reinterpret_cast *>(data)); - break; - case adios_double_complex: - dtype = DT::CDOUBLE; - a = Attribute(*reinterpret_cast *>(data)); - break; - case adios_string: { - dtype = DT::STRING; - auto c = reinterpret_cast(data); - a = Attribute( - auxiliary::strip(std::string(c, std::strlen(c)), {'\0'})); - break; - } - case adios_string_array: { - dtype = DT::VEC_STRING; - auto c = reinterpret_cast(data); - std::vector vs; - vs.resize(size); - for (int i = 0; i < size; ++i) - { - vs[i] = auxiliary::strip( - std::string(c[i], std::strlen(c[i])), {'\0'}); - /** @todo pointer should be freed, but this causes memory - * corruption */ - // free(c[i]); - } - a = Attribute(vs); - break; - } - default: - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "Unsupported ADIOS1 attribute datatype '" + - std::to_string(datatype) + - "' in scalar branch: " + attrname); - } - } - else - { - switch (datatype) - { - using DT = Datatype; - case adios_byte: { - dtype = DT::VEC_CHAR; - auto c = reinterpret_cast(data); - std::vector vc; - vc.resize(size); - for (int i = 0; i < size; ++i) - vc[i] = c[i]; - a = Attribute(vc); - break; - } - case adios_short: { - if (sizeof(short) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_SHORT); - else if (sizeof(int) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), DT::VEC_INT); - else if (sizeof(long) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_LONG); - else if (sizeof(long long) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_LONGLONG); - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type adios_short: " + - attrname); - break; - } - case adios_integer: { - if (sizeof(short) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_SHORT); - else if (sizeof(int) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), DT::VEC_INT); - else if (sizeof(long) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_LONG); - else if (sizeof(long long) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_LONGLONG); - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type adios_integer: " + - attrname); - break; - } - case adios_long: { - if (sizeof(short) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_SHORT); - else if (sizeof(int) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), DT::VEC_INT); - else if (sizeof(long) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_LONG); - else if (sizeof(long long) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_LONGLONG); - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type adios_long: " + - attrname); - break; - } - case adios_unsigned_byte: { - dtype = DT::VEC_UCHAR; - auto uc = reinterpret_cast(data); - std::vector vuc; - vuc.resize(size); - for (int i = 0; i < size; ++i) - vuc[i] = uc[i]; - a = Attribute(vuc); - break; - } - case adios_unsigned_short: { - if (sizeof(unsigned short) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_USHORT); - else if (sizeof(unsigned int) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_UINT); - else if (sizeof(unsigned long) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_ULONG); - else if (sizeof(unsigned long long) == 2u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_ULONGLONG); - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type " - "adios_unsigned_short: " + - attrname); - break; - } - case adios_unsigned_integer: { - if (sizeof(unsigned short) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_USHORT); - else if (sizeof(unsigned int) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_UINT); - else if (sizeof(unsigned long) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_ULONG); - else if (sizeof(unsigned long long) == 4u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_ULONGLONG); - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type " - "adios_unsigned_integer: " + - attrname); - break; - } - case adios_unsigned_long: { - if (sizeof(unsigned short) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_USHORT); - else if (sizeof(unsigned int) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_UINT); - else if (sizeof(unsigned long) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_ULONG); - else if (sizeof(unsigned long long) == 8u) - std::tie(a, dtype) = std::make_tuple( - readVectorAttributeInternal(data, size), - DT::VEC_ULONGLONG); - else - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "No native equivalent found for type " - "adios_unsigned_long: " + - attrname); - break; - } - case adios_real: { - dtype = DT::VEC_FLOAT; - auto f4 = reinterpret_cast(data); - std::vector vf; - vf.resize(size); - for (int i = 0; i < size; ++i) - vf[i] = f4[i]; - a = Attribute(vf); - break; - } - case adios_double: { - dtype = DT::VEC_DOUBLE; - auto d8 = reinterpret_cast(data); - std::vector vd; - vd.resize(size); - for (int i = 0; i < size; ++i) - vd[i] = d8[i]; - a = Attribute(vd); - break; - } - case adios_long_double: { - dtype = DT::VEC_LONG_DOUBLE; - auto ld = reinterpret_cast(data); - std::vector vld; - vld.resize(size); - for (int i = 0; i < size; ++i) - vld[i] = ld[i]; - a = Attribute(vld); - break; - } - /* not supported by ADIOS 1.13.1: VEC_CFLOAT, VEC_CDOUBLE, - * VEC_CLONG_DOUBLE https://github.com/ornladios/ADIOS/issues/212 - */ - case adios_string: { - dtype = DT::STRING; - a = Attribute(auxiliary::strip( - std::string(reinterpret_cast(data), size), {'\0'})); - break; - } - case adios_string_array: { - dtype = DT::VEC_STRING; - auto c = reinterpret_cast(data); - std::vector vs; - vs.resize(size); - for (int i = 0; i < size; ++i) - { - vs[i] = auxiliary::strip( - std::string(c[i], std::strlen(c[i])), {'\0'}); - /** @todo pointer should be freed, but this causes memory - * corruption */ - // free(c[i]); - } - a = Attribute(vs); - break; - } - - default: - error::throwReadError( - error::AffectedObject::Attribute, - error::Reason::Other, - "ADIOS1", - "Unsupported ADIOS1 attribute datatype '" + - std::to_string(datatype) + - "' in vector branch: " + attrname); - } - } - - free(data); - - *parameters.dtype = dtype; - *parameters.resource = a.getResource(); -} - -template -void CommonADIOS1IOHandlerImpl::listPaths( - Writable *writable, Parameter ¶meters) -{ - if (!writable->written) - throw std::runtime_error( - "[ADIOS1] Internal error: Writable not marked written during path " - "listing"); - - ADIOS_FILE *f; - f = m_openReadFileHandles.at(m_filePaths.at(writable)); - - std::string name = concrete_bp1_file_position(writable); - - std::unordered_set paths; - std::unordered_set variables; - for (int i = 0; i < f->nvars; ++i) - { - char *str = f->var_namelist[i]; - std::string s(str, std::strlen(str)); - if (auxiliary::starts_with(s, name)) - { - /* remove the writable's path from the name */ - s = auxiliary::replace_first(s, name, ""); - variables.emplace(s); - if (std::any_of( - s.begin(), s.end(), [](char c) { return c == '/'; })) - { - /* there are more path levels after the current writable */ - s = s.substr(0, s.find_first_of('/')); - paths.emplace(s); - } - } - } - for (int i = 0; i < f->nattrs; ++i) - { - char *str = f->attr_namelist[i]; - std::string s(str, std::strlen(str)); - if (auxiliary::starts_with(s, name)) - { - /* remove the writable's path from the name */ - s = auxiliary::replace_first(s, name, ""); - if (std::any_of( - s.begin(), s.end(), [](char c) { return c == '/'; })) - { - /* remove the attribute name */ - s = s.substr(0, s.find_last_of('/')); - if (!std::any_of( - variables.begin(), - variables.end(), - [&s](std::string const &var) { - return auxiliary::starts_with(var, s); - })) - { - /* this is either a group or a constant scalar */ - s = s.substr(0, s.find_first_of('/')); - paths.emplace(s); - } - } - } - } - - *parameters.paths = std::vector(paths.begin(), paths.end()); -} - -template -void CommonADIOS1IOHandlerImpl::listDatasets( - Writable *writable, Parameter ¶meters) -{ - if (!writable->written) - throw std::runtime_error( - "[ADIOS1] Internal error: Writable not marked written during " - "dataset listing"); - - ADIOS_FILE *f; - f = m_openReadFileHandles.at(m_filePaths.at(writable)); - - std::string name = concrete_bp1_file_position(writable); - - std::unordered_set paths; - for (int i = 0; i < f->nvars; ++i) - { - char *str = f->var_namelist[i]; - std::string s(str, std::strlen(str)); - if (auxiliary::starts_with(s, name)) - { - /* remove the writable's path from the name */ - s = auxiliary::replace_first(s, name, ""); - if (std::none_of( - s.begin(), s.end(), [](char c) { return c == '/'; })) - { - /* this is a dataset of the writable */ - paths.emplace(s); - } - } - } - - *parameters.datasets = std::vector(paths.begin(), paths.end()); -} - -template -void CommonADIOS1IOHandlerImpl::listAttributes( - Writable *writable, Parameter ¶meters) -{ - if (!writable->written) - throw std::runtime_error( - "[ADIOS1] Internal error: Writable not marked written during " - "attribute listing"); - - ADIOS_FILE *f; - f = m_openReadFileHandles.at(m_filePaths.at(writable)); - - std::string name = concrete_bp1_file_position(writable); - - if (!auxiliary::ends_with(name, '/')) - { - /* writable is a dataset and corresponds to an ADIOS variable */ - ADIOS_VARINFO *info; - info = adios_inq_var(f, name.c_str()); - VERIFY( - adios_errno == err_no_error, - "[ADIOS1] Internal error: Failed to inquire ADIOS variable during " - "attribute listing"); - VERIFY( - info != nullptr, - "[ADIOS1] Internal error: Failed to inquire ADIOS variable during " - "attribute listing"); - - name += '/'; - parameters.attributes->reserve(info->nattrs); - for (int i = 0; i < info->nattrs; ++i) - { - char *c = f->attr_namelist[info->attr_ids[i]]; - parameters.attributes->push_back(auxiliary::replace_first( - std::string(c, std::strlen(c)), name, "")); - } - - adios_free_varinfo(info); - } - else - { - /* there is no ADIOS variable associated with the writable */ - std::unordered_set attributes; - for (int i = 0; i < f->nattrs; ++i) - { - char *str = f->attr_namelist[i]; - std::string s(str, std::strlen(str)); - if (auxiliary::starts_with(s, name)) - { - /* remove the writable's path from the name */ - s = auxiliary::replace_first(s, name, ""); - if (std::none_of( - s.begin(), s.end(), [](char c) { return c == '/'; })) - { - /* this is an attribute of the writable */ - attributes.insert(s); - } - } - } - *parameters.attributes = - std::vector(attributes.begin(), attributes.end()); - } -} - -template -void CommonADIOS1IOHandlerImpl::deregister( - Writable *writable, Parameter const &) -{ - m_filePaths.erase(writable); -} - -template -void CommonADIOS1IOHandlerImpl::initJson(json::TracingJSON config) -{ - if (!config.json().contains("adios1")) - { - return; - } - auto maybeTransform = datasetTransform(config["adios1"]); - if (maybeTransform.has_value()) - { - m_defaultTransform = std::move(maybeTransform.value()); - } - auto shadow = config.invertShadow(); - if (shadow.size() > 0) - { - switch (config.originallySpecifiedAs) - { - case json::SupportedLanguages::JSON: - std::cerr << "Warning: parts of the JSON configuration for ADIOS1 " - "remain unused:\n" - << shadow << std::endl; - break; - case json::SupportedLanguages::TOML: { - auto asToml = json::jsonToToml(shadow); - std::cerr << "Warning: parts of the JSON configuration for ADIOS1 " - "remain unused:\n" - << asToml << std::endl; - break; - } - } - } -} - -/* - * Sic! - * The ADIOS1 IO Handler is built as two CMake targets: serial and parallel. - * One receives the definition openPMD_HAVE_MPI=0, the other receives - * openPMD_HAVE_MPI=1. - * So, if openPMD_HAVE_MPI is true, then we do not need to instantiate both - * the serial and the parallel handler down here, since the serial handler will - * be instantiated in another target. - */ -#if openPMD_HAVE_MPI -template class CommonADIOS1IOHandlerImpl; -#else -template class CommonADIOS1IOHandlerImpl; -#endif // openPMD_HAVE_MPI - -} // namespace openPMD -#endif // openPMD_HAVE_ADIOS1 diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp deleted file mode 100644 index 396a9106d8..0000000000 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ /dev/null @@ -1,511 +0,0 @@ -/* Copyright 2017-2021 Fabian Koller, Axel Huebl - * - * This file is part of openPMD-api. - * - * openPMD-api is free software: you can redistribute it and/or modify - * it under the terms of of either the GNU General Public License or - * the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * openPMD-api is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License and the GNU Lesser General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License - * and the GNU Lesser General Public License along with openPMD-api. - * If not, see . - */ -#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" -#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp" - -#if openPMD_HAVE_MPI && openPMD_HAVE_ADIOS1 -#include "openPMD/IO/IOTask.hpp" -#include -#include -#include -#include -#include -#include -#endif - -namespace openPMD -{ -#if openPMD_HAVE_ADIOS1 && openPMD_HAVE_MPI -#if openPMD_USE_VERIFY -#define VERIFY(CONDITION, TEXT) \ - { \ - if (!(CONDITION)) \ - throw std::runtime_error((TEXT)); \ - } -#else -#define VERIFY(CONDITION, TEXT) \ - do \ - { \ - (void)sizeof(CONDITION); \ - } while (0) -#endif - -ParallelADIOS1IOHandlerImpl::ParallelADIOS1IOHandlerImpl( - AbstractIOHandler *handler, json::TracingJSON json, MPI_Comm comm) - : Base_t{handler}, m_mpiInfo{MPI_INFO_NULL} -{ - int status = MPI_SUCCESS; - status = MPI_Comm_dup(comm, &m_mpiComm); - VERIFY( - status == MPI_SUCCESS, - "[ADIOS1] Internal error: Failed to duplicate MPI communicator"); - initJson(std::move(json)); -} - -ParallelADIOS1IOHandlerImpl::~ParallelADIOS1IOHandlerImpl() -{ - for (auto &f : m_openReadFileHandles) - close(f.second); - m_openReadFileHandles.clear(); - - if (this->m_handler->m_backendAccess != Access::READ_ONLY) - { - for (auto &group : m_attributeWrites) - for (auto &att : group.second) - flush_attribute(group.first, att.first, att.second); - - // unordered map caused the value of the same container - // stored with different orders in different processors. - // which caused trouble with close(), which is collective - // so I just sort by file name to force all processors close - // all the fids in the same order - std::map allFiles; - for (auto &f : m_openWriteFileHandles) - allFiles[*(f.first)] = f.second; - - for (auto const &p : allFiles) - { - auto const fid = p.second; - close(fid); - } - - m_openWriteFileHandles.clear(); - } - - int status; - MPI_Barrier(m_mpiComm); - status = adios_read_finalize_method(m_readMethod); - if (status != err_no_error) - std::cerr << "Internal error: Failed to finalize ADIOS reading method " - "(parallel)\n"; - - MPI_Barrier(m_mpiComm); - int rank = 0; - MPI_Comm_rank(m_mpiComm, &rank); - status = adios_finalize(rank); - if (status != err_no_error) - std::cerr << "Internal error: Failed to finalize ADIOS (parallel)\n"; - - MPI_Comm_free(&m_mpiComm); -} - -std::future ParallelADIOS1IOHandlerImpl::flush() -{ - using namespace auxiliary; - - auto handler = dynamic_cast(m_handler); - while (!handler->m_setup.empty()) - { - IOTask &i = handler->m_setup.front(); - try - { - switch (i.operation) - { - using O = Operation; - case O::CREATE_FILE: - createFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CHECK_FILE: - checkFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CREATE_PATH: - createPath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::OPEN_PATH: - openPath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CREATE_DATASET: - createDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::WRITE_ATT: - writeAttribute( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::OPEN_FILE: - openFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::KEEP_SYNCHRONOUS: - keepSynchronous( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DEREGISTER: - deregister( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - default: - VERIFY( - false, - "[ADIOS1] Internal error: Wrong operation in ADIOS setup " - "queue"); - } - } - catch (...) - { - std::cerr << "[AbstractIOHandlerImpl] IO Task " - << internal::operationAsString(i.operation) - << " failed with exception. Clearing IO queue and " - "passing on the exception." - << std::endl; - while (!m_handler->m_work.empty()) - { - m_handler->m_work.pop(); - } - throw; - } - handler->m_setup.pop(); - } - - while (!handler->m_work.empty()) - { - IOTask &i = handler->m_work.front(); - try - { - switch (i.operation) - { - using O = Operation; - case O::EXTEND_DATASET: - extendDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CLOSE_PATH: - closePath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::OPEN_DATASET: - openDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::CLOSE_FILE: - closeFile( - i.writable, - *dynamic_cast *>( - i.parameter.get())); - break; - case O::DELETE_FILE: - deleteFile( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DELETE_PATH: - deletePath( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DELETE_DATASET: - deleteDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::DELETE_ATT: - deleteAttribute( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::WRITE_DATASET: - writeDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::READ_DATASET: - readDataset( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::GET_BUFFER_VIEW: - getBufferView( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::READ_ATT: - readAttribute( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::LIST_PATHS: - listPaths( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::LIST_DATASETS: - listDatasets( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::LIST_ATTS: - listAttributes( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::ADVANCE: - advance( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - case O::AVAILABLE_CHUNKS: - availableChunks( - i.writable, - deref_dynamic_cast>( - i.parameter.get())); - break; - default: - VERIFY( - false, - "[ADIOS1] Internal error: Wrong operation in ADIOS work " - "queue"); - } - } - catch (...) - { - std::cerr << "[AbstractIOHandlerImpl] IO Task " - << internal::operationAsString(i.operation) - << " failed with exception. Clearing IO queue and " - "passing on the exception." - << std::endl; - while (!m_handler->m_work.empty()) - { - m_handler->m_work.pop(); - } - throw; - } - handler->m_work.pop(); - } - - int status; - for (auto &file : m_scheduledReads) - { - status = adios_perform_reads(file.first, 1); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to perform ADIOS reads during " - "dataset reading"); - - for (auto &sel : file.second) - adios_selection_delete(sel.selection); - } - m_scheduledReads.clear(); - - return std::future(); -} - -void ParallelADIOS1IOHandlerImpl::init() -{ - int status; - status = adios_init_noxml(m_mpiComm); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to initialize ADIOS"); - - /** @todo ADIOS_READ_METHOD_BP_AGGREGATE */ - m_readMethod = ADIOS_READ_METHOD_BP; - status = adios_read_init_method(m_readMethod, m_mpiComm, ""); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to initialize ADIOS reading method"); -} - -ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( - std::string path, Access at, json::TracingJSON json, MPI_Comm comm) - : AbstractIOHandler(std::move(path), at, comm) - , m_impl{new ParallelADIOS1IOHandlerImpl(this, std::move(json), comm)} -{ - m_impl->init(); -} - -ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; - -std::future ParallelADIOS1IOHandler::flush(internal::ParsedFlushParams &) -{ - return m_impl->flush(); -} - -void ParallelADIOS1IOHandler::enqueue(IOTask const &i) -{ - switch (i.operation) - { - case Operation::CREATE_FILE: - case Operation::CHECK_FILE: - case Operation::CREATE_PATH: - case Operation::OPEN_PATH: - case Operation::CREATE_DATASET: - case Operation::OPEN_FILE: - case Operation::WRITE_ATT: - case Operation::KEEP_SYNCHRONOUS: - case Operation::DEREGISTER: - m_setup.push(i); - return; - default: - m_work.push(i); - return; - } -} - -int64_t ParallelADIOS1IOHandlerImpl::open_write(Writable *writable) -{ - auto res = m_filePaths.find(writable); - if (res == m_filePaths.end()) - res = m_filePaths.find(writable->parent); - - std::string mode; - if (m_existsOnDisk[res->second]) - { - mode = "u"; - /* close the handle that corresponds to the file we want to append to */ - if (m_openReadFileHandles.find(res->second) != - m_openReadFileHandles.end()) - { - close(m_openReadFileHandles[res->second]); - m_openReadFileHandles.erase(res->second); - } - } - else - { - mode = "w"; - m_existsOnDisk[res->second] = true; - } - - int64_t fd; - int status; - status = adios_open( - &fd, - res->second->c_str(), - res->second->c_str(), - mode.c_str(), - m_mpiComm); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to open_write ADIOS file"); - - return fd; -} - -ADIOS_FILE *ParallelADIOS1IOHandlerImpl::open_read(std::string const &name) -{ - ADIOS_FILE *f; - f = adios_read_open_file(name.c_str(), m_readMethod, m_mpiComm); - VERIFY( - adios_errno != err_file_not_found, - "[ADIOS1] Internal error: ADIOS file not found"); - VERIFY( - f != nullptr, - "[ADIOS1] Internal error: Failed to open_read ADIOS file"); - - return f; -} - -int64_t ParallelADIOS1IOHandlerImpl::initialize_group(std::string const &name) -{ - std::stringstream params; - params << "num_aggregators=" - << getEnvNum("OPENPMD_ADIOS_NUM_AGGREGATORS", "1") - << ";num_ost=" << getEnvNum("OPENPMD_ADIOS_NUM_OST", "0") - << ";have_metadata_file=" - << getEnvNum("OPENPMD_ADIOS_HAVE_METADATA_FILE", "1") - << ";verbose=2"; - std::string params_str = params.str(); // important: copy out of temporary! - - int status; - int64_t group; - ADIOS_STATISTICS_FLAG noStatistics = adios_stat_no; - status = adios_declare_group(&group, name.c_str(), "", noStatistics); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to declare ADIOS group"); - status = - adios_select_method(group, "MPI_AGGREGATE", params_str.c_str(), ""); - VERIFY( - status == err_no_error, - "[ADIOS1] Internal error: Failed to select ADIOS method"); - return group; -} - -#else // openPMD_HAVE_ADIOS1 && openPMD_HAVE_MPI -#if openPMD_HAVE_MPI -ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( - std::string path, Access at, json::TracingJSON, MPI_Comm comm) - : AbstractIOHandler(std::move(path), at, comm) -{ - throw std::runtime_error("openPMD-api built without ADIOS1 support"); -} -#else -ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( - std::string path, Access at, json::TracingJSON) - : AbstractIOHandler(std::move(path), at) -{ - throw std::runtime_error( - "openPMD-api built without parallel ADIOS1 support"); -} -#endif // openPMD_HAVE_MPI - -ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; - -std::future ParallelADIOS1IOHandler::flush(internal::ParsedFlushParams &) -{ - return std::future(); -} - -#if openPMD_HAVE_ADIOS1 -void ParallelADIOS1IOHandler::enqueue(IOTask const &) -{} -#endif // openPMD_HAVE_ADIOS1 -#endif // openPMD_HAVE_ADIOS1 && openPMD_HAVE_MPI -} // namespace openPMD diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 27efe722c1..c58528f568 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -23,9 +23,7 @@ #include "openPMD/config.hpp" #include "openPMD/Error.hpp" -#include "openPMD/IO/ADIOS/ADIOS1IOHandler.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" -#include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" #include "openPMD/IO/DummyIOHandler.hpp" #include "openPMD/IO/HDF5/HDF5IOHandler.hpp" #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" @@ -62,22 +60,6 @@ namespace } throw "Unreachable"; } - - constexpr char const *adios1Deprecation = R"( -[Deprecation warning] - Development on the ADIOS1 IO library has ceased. - Support for ADIOS1 in the openPMD-api has been deprecated - and will be removed in a future version. - - Please consider switching to ADIOS2. - We recommend checking your ADIOS1 datasets for compatibility with ADIOS2. - Conversion of data from one backend to another may optionally be achieved - by using the `openpmd-pipe` tool.) - - Suppress this warning via `export OPENPMD_ADIOS_SUPPRESS_DEPRECATED_WARNING=1`.)"; - - constexpr char const *suppressAdios1DeprecationWarning = - "OPENPMD_ADIOS_SUPPRESS_DEPRECATED_WARNING"; } // namespace #if openPMD_HAVE_MPI @@ -97,18 +79,6 @@ std::unique_ptr createIOHandler( case Format::HDF5: return constructIOHandler( "HDF5", path, access, comm, std::move(options)); - case Format::ADIOS1: - if (auxiliary::getEnvNum(suppressAdios1DeprecationWarning, 0) == 0) - { - int rank; - MPI_Comm_rank(comm, &rank); - if (rank == 0) - { - std::cerr << adios1Deprecation << std::endl; - } - } - return constructIOHandler( - "ADIOS1", path, access, std::move(options), comm); case Format::ADIOS2_BP: return constructIOHandler( "ADIOS2", @@ -175,13 +145,6 @@ std::unique_ptr createIOHandler( case Format::HDF5: return constructIOHandler( "HDF5", path, access, std::move(options)); - case Format::ADIOS1: - if (auxiliary::getEnvNum(suppressAdios1DeprecationWarning, 0) == 0) - { - std::cerr << adios1Deprecation << std::endl; - } - return constructIOHandler( - "ADIOS1", path, access, std::move(options)); case Format::ADIOS2_BP: return constructIOHandler( "ADIOS2", diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 26ab93940e..3e6560e4c9 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -219,20 +219,6 @@ void Iteration::flushFileBased( } else { - // operations for create mode - if ((IOHandler()->m_frontendAccess == Access::CREATE) && - ((IOHandler()->backendName() == "MPI_ADIOS1") || - (IOHandler()->backendName() == "ADIOS1"))) - { - Parameter fOpen; - fOpen.name = filename; - fOpen.encoding = IterationEncoding::fileBased; - IOHandler()->enqueue(IOTask(&s.writable(), fOpen)); - flush(flushParams); - - return; - } - // operations for read/read-write mode /* open file */ s.openIteration(i, *this); diff --git a/src/Series.cpp b/src/Series.cpp index 779ba97906..39c46f30ba 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -2155,7 +2155,6 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) { std::map const backendDescriptors{ {"hdf5", Format::HDF5}, - {"adios1", Format::ADIOS1}, {"adios2", Format::ADIOS2_BP}, {"json", Format::JSON}}; std::string backend; diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index f57cf7455a..c04e672ae6 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -501,7 +501,7 @@ std::optional asLowerCaseStringDynamic(nlohmann::json const &value) std::vector backendKeys() { - return {"adios1", "adios2", "json", "hdf5"}; + return {"adios2", "json", "hdf5"}; } void warnGlobalUnusedOptions(TracingJSON const &config) diff --git a/src/config.cpp b/src/config.cpp index b9e27be752..a44925287a 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -34,7 +34,7 @@ std::map openPMD::getVariants() {"mpi", bool(openPMD_HAVE_MPI)}, {"json", true}, {"hdf5", bool(openPMD_HAVE_HDF5)}, - {"adios1", bool(openPMD_HAVE_ADIOS1)}, + {"adios1", false}, {"adios2", bool(openPMD_HAVE_ADIOS2)}}; } @@ -42,7 +42,7 @@ std::vector openPMD::getFileExtensions() { std::vector fext; fext.emplace_back("json"); -#if openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2 +#if openPMD_HAVE_ADIOS2 fext.emplace_back("bp"); #endif #if openPMD_HAVE_ADIOS2 diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 496cb0a36f..f4ad2ce110 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1095,25 +1095,6 @@ TEST_CASE("backend_via_json", "[core]") REQUIRE(auxiliary::directory_exists( "../samples/optionsViaJsonOverwritesAutomaticDetectionBp4.sst")); -#if openPMD_HAVE_ADIOS1 - setenv("OPENPMD_BP_BACKEND", "ADIOS1", 1); - { - /* - * ADIOS2 backend should be selected even if OPENPMD_BP_BACKEND is set - * as ADIOS1 - * JSON config overwrites environment variables - */ - Series series( - "../samples/optionsPreferJsonOverEnvVar.bp", - Access::CREATE, - R"({"backend": "ADIOS2"})"); - REQUIRE(series.backend() == "ADIOS2"); - } - // unset again - unsetenv("OPENPMD_BP_BACKEND"); - REQUIRE(auxiliary::directory_exists( - "../samples/optionsPreferJsonOverEnvVar.bp")); -#endif #endif std::string encodingFileBased = R"({"backend": "json", "iteration_encoding": "file_based"})"; @@ -1317,18 +1298,6 @@ TEST_CASE("DoConvert_single_value_to_vector", "[core]") TEST_CASE("unavailable_backend", "[core]") { -#if !openPMD_HAVE_ADIOS1 - { - auto fail = []() { - Series( - "unavailable.bp", Access::CREATE, R"({"backend": "ADIOS1"})"); - }; - REQUIRE_THROWS_WITH( - fail(), - "Wrong API usage: openPMD-api built without support for backend " - "'ADIOS1'."); - } -#endif #if !openPMD_HAVE_ADIOS2 { auto fail = []() { @@ -1341,7 +1310,7 @@ TEST_CASE("unavailable_backend", "[core]") "'ADIOS2'."); } #endif -#if !openPMD_HAVE_ADIOS1 && !openPMD_HAVE_ADIOS2 +#if !openPMD_HAVE_ADIOS2 { auto fail = []() { Series("unavailable.bp", Access::CREATE); }; REQUIRE_THROWS_WITH( diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 5ace4a2cd4..b6621ea8f1 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -34,7 +34,7 @@ std::vector getBackends() // first component: backend file ending // second component: whether to test 128 bit values std::vector res; -#if openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2 +#if openPMD_HAVE_ADIOS2 res.emplace_back("bp"); #endif #if openPMD_HAVE_HDF5 @@ -71,14 +71,6 @@ TEST_CASE("parallel_multi_series_test", "[parallel]") auto myBackends = getBackends(); - // this test demonstrates an ADIOS1 (upstream) bug, comment this section to - // trigger it - auto const rmEnd = std::remove_if( - myBackends.begin(), myBackends.end(), [](std::string const &beit) { - return beit == "bp" && determineFormat("test.bp") == Format::ADIOS1; - }); - myBackends.erase(rmEnd, myBackends.end()); - // have multiple serial series alive at the same time for (auto const sn : {1, 2, 3}) { @@ -385,8 +377,7 @@ TEST_CASE("no_parallel_hdf5", "[parallel][hdf5]") #endif -// this one works for both ADIOS1 and ADIOS2 -#if (openPMD_HAVE_ADIOS1 || openPMD_HAVE_ADIOS2) && openPMD_HAVE_MPI +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI void available_chunks_test(std::string file_ending) { int r_mpi_rank{-1}, r_mpi_size{-1}; @@ -481,11 +472,6 @@ void extendDataset(std::string const &ext, std::string const &jsonConfig) std::iota(data2.begin(), data2.end(), 25); { Series write(filename, Access::CREATE, MPI_COMM_WORLD, jsonConfig); - if (ext == "bp" && write.backend() != "ADIOS2") - { - // dataset resizing unsupported in ADIOS1 - return; - } Dataset ds1{Datatype::INT, {mpi_size, 25}}; Dataset ds2{{mpi_size, 50}}; @@ -525,7 +511,7 @@ TEST_CASE("extend_dataset", "[parallel]") } #endif -#if openPMD_HAVE_ADIOS1 && openPMD_HAVE_MPI +#if openPMD_HAVE_ADIOS2 && openPMD_HAVE_MPI TEST_CASE("adios_write_test", "[parallel][adios]") { Series o = @@ -538,7 +524,7 @@ TEST_CASE("adios_write_test", "[parallel][adios]") auto mpi_size = static_cast(size); auto mpi_rank = static_cast(rank); - o.setAuthor("Parallel ADIOS1"); + o.setAuthor("Parallel ADIOS2"); ParticleSpecies &e = o.iterations[1].particles["e"]; std::vector position_global(mpi_size); @@ -587,7 +573,7 @@ TEST_CASE("adios_write_test_skip_declare", "[parallel][adios]") write_test_zero_extent(true, "bp", false, false); } -TEST_CASE("hzdr_adios_sample_content_test", "[parallel][adios1]") +TEST_CASE("hzdr_adios_sample_content_test", "[parallel][adios2][bp3]") { int mpi_rank{-1}; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); @@ -720,7 +706,6 @@ void close_iteration_test(std::string file_ending) std::vector data{2, 4, 6, 8}; // { // we do *not* need these parentheses Series write(name, Access::CREATE, MPI_COMM_WORLD); - bool isAdios1 = write.backend() == "MPI_ADIOS1"; { Iteration it0 = write.iterations[0]; auto E_x = it0.meshes["E"]["x"]; @@ -731,14 +716,6 @@ void close_iteration_test(std::string file_ending) write.flush(); // } - if (isAdios1) - { - // run a simplified test for Adios1 since Adios1 has issues opening - // twice in the same process - REQUIRE(auxiliary::file_exists( - "../samples/close_iterations_parallel_0.bp")); - } - else { Series read(name, Access::READ_ONLY, MPI_COMM_WORLD); Iteration it0 = read.iterations[0]; @@ -764,14 +741,6 @@ void close_iteration_test(std::string file_ending) REQUIRE_THROWS(write.flush()); } - if (isAdios1) - { - // run a simplified test for Adios1 since Adios1 has issues opening - // twice in the same process - REQUIRE(auxiliary::file_exists( - "../samples/close_iterations_parallel_1.bp")); - } - else { Series read(name, Access::READ_ONLY, MPI_COMM_WORLD); Iteration it1 = read.iterations[1]; @@ -1075,11 +1044,6 @@ void adios2_streaming(bool variableBasedLayout) int rank{-1}; MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1") - { - // run this test for ADIOS2 only - return; - } if (size < 2 || rank > 1) { @@ -1179,11 +1143,6 @@ TEST_CASE("adios2_streaming", "[pseudoserial][adios2]") TEST_CASE("parallel_adios2_json_config", "[parallel][adios2]") { - if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1") - { - // run this test for ADIOS2 only - return; - } int size{-1}; int rank{-1}; MPI_Comm_size(MPI_COMM_WORLD, &size); @@ -1327,11 +1286,6 @@ void adios2_ssc() int global_rank{-1}; MPI_Comm_size(MPI_COMM_WORLD, &global_size); MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); - if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1") - { - // run this test for ADIOS2 only - return; - } if (global_size < 2) { @@ -1489,16 +1443,6 @@ void append_mode( { write.setIterationEncoding(IterationEncoding::variableBased); } - if (write.backend() == "MPI_ADIOS1") - { - REQUIRE_THROWS_WITH( - write.flush(), - Catch::Equals( - "Operation unsupported in ADIOS1: Appending to existing " - "file on disk (use Access::CREATE to overwrite)")); - // destructor will be noisy now - return; - } writeSomeIterations( write.writeIterations(), std::vector{3, 2}); @@ -1518,16 +1462,6 @@ void append_mode( { write.setIterationEncoding(IterationEncoding::variableBased); } - if (write.backend() == "MPI_ADIOS1") - { - REQUIRE_THROWS_WITH( - write.flush(), - Catch::Equals( - "Operation unsupported in ADIOS1: Appending to existing " - "file on disk (use Access::CREATE to overwrite)")); - // destructor will be noisy now - return; - } writeSomeIterations( write.writeIterations(), std::vector{4, 3, 10}); @@ -1540,13 +1474,6 @@ void append_mode( { write.setIterationEncoding(IterationEncoding::variableBased); } - if (write.backend() == "MPI_ADIOS1") - { - REQUIRE_THROWS_AS( - write.flush(), error::OperationUnsupportedInBackend); - // destructor will be noisy now - return; - } writeSomeIterations( write.writeIterations(), std::vector{7, 1, 11}); @@ -1681,17 +1608,6 @@ void append_mode( { write.setIterationEncoding(IterationEncoding::variableBased); } - if (write.backend() == "ADIOS1") - { - REQUIRE_THROWS_WITH( - write.flush(), - Catch::Equals( - "Operation unsupported in ADIOS1: Appending to " - "existing " - "file on disk (use Access::CREATE to overwrite)")); - // destructor will be noisy now - return; - } writeSomeIterations( write.writeIterations(), std::vector{4, 5}); @@ -1808,21 +1724,6 @@ TEST_CASE("append_mode", "[serial]") TEST_CASE("unavailable_backend", "[core][parallel]") { -#if !openPMD_HAVE_ADIOS1 - { - auto fail = []() { - Series( - "unavailable.bp", - Access::CREATE, - MPI_COMM_WORLD, - R"({"backend": "ADIOS1"})"); - }; - REQUIRE_THROWS_WITH( - fail(), - "Wrong API usage: openPMD-api built without support for backend " - "'ADIOS1'."); - } -#endif #if !openPMD_HAVE_ADIOS2 { auto fail = []() { @@ -1838,7 +1739,7 @@ TEST_CASE("unavailable_backend", "[core][parallel]") "'ADIOS2'."); } #endif -#if !openPMD_HAVE_ADIOS1 && !openPMD_HAVE_ADIOS2 +#if !openPMD_HAVE_ADIOS2 { auto fail = []() { Series("unavailable.bp", Access::CREATE, MPI_COMM_WORLD); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 3d48b01789..e58db2bff0 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -57,10 +57,7 @@ std::vector testedBackends() { auto variants = getVariants(); std::map extensions{ - {"json", "json"}, - {"adios1", "adios1.bp"}, - {"adios2", "bp"}, - {"hdf5", "h5"}}; + {"json", "json"}, {"adios2", "bp"}, {"hdf5", "h5"}}; std::vector res; for (auto const &pair : variants) { @@ -175,10 +172,6 @@ TEST_CASE("adios2_char_portability", "[serial][adios2]") } { - if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "ADIOS2") != "ADIOS2") - { - return; - } Series read( "../samples/adios2_char_portability.bp", Access::READ_ONLY, config); auto signedVectorAttribute = read.getAttribute("signedVector"); @@ -301,16 +294,6 @@ TEST_CASE("multi_series_test", "[serial]") auto myfileExtensions = testedFileExtensions(); - // this test demonstrates an ADIOS1 (upstream) bug, comment this section to - // trigger it - auto const rmEnd = std::remove_if( - myfileExtensions.begin(), - myfileExtensions.end(), - [](std::string const &beit) { - return beit == "bp" && determineFormat("test.bp") == Format::ADIOS1; - }); - myfileExtensions.erase(rmEnd, myfileExtensions.end()); - // have multiple serial series alive at the same time for (auto const sn : {1, 2, 3}) { @@ -642,9 +625,6 @@ void close_iteration_interleaved_test( TEST_CASE("close_iteration_interleaved_test", "[serial]") { - bool const bp_prefer_adios1 = - (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1"); - for (auto const &t : testedFileExtensions()) { close_iteration_interleaved_test(t, IterationEncoding::fileBased); @@ -653,8 +633,6 @@ TEST_CASE("close_iteration_interleaved_test", "[serial]") // run this test for ADIOS2 & JSON only if (t == "h5") continue; - if (t == "bp" && bp_prefer_adios1) - continue; close_iteration_interleaved_test(t, IterationEncoding::variableBased); } } @@ -4400,8 +4378,6 @@ TEST_CASE("adios2_engines_and_file_endings") std::string const &requiredEngine, std::string const &filesystemExt, std::string const &jsonCfg = "{}") mutable { - // Env. var. OPENPMD_BP_BACKEND does not matter for this test as - // we always override it in the JSON config auto basename = "../samples/file_endings/groupbased" + std::to_string(filenameCounter++); auto name = basename + ext; @@ -4480,10 +4456,7 @@ TEST_CASE("adios2_engines_and_file_endings") { Series write(name, Access::CREATE, jsonCfg); } - bool isThisADIOS1 = - auxiliary::getEnvString("OPENPMD_BP_BACKEND", "") == "ADIOS1" && - ext == ".bp"; - if (directory && !isThisADIOS1) + if (directory) { REQUIRE(auxiliary::directory_exists(filesystemname)); } @@ -4630,10 +4603,7 @@ TEST_CASE("adios2_engines_and_file_endings") Series write(name, Access::CREATE, jsonCfg); write.writeIterations()[0]; } - bool isThisADIOS1 = - auxiliary::getEnvString("OPENPMD_BP_BACKEND", "") == "ADIOS1" && - ext == ".bp"; - if (directory && !isThisADIOS1) + if (directory) { REQUIRE(auxiliary::directory_exists(filesystemname)); } @@ -4695,11 +4665,6 @@ TEST_CASE("adios2_engines_and_file_endings") TEST_CASE("serial_adios2_backend_config", "[serial][adios2]") { - if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") == "ADIOS1") - { - // run this test for ADIOS2 only - return; - } std::string writeConfigBP3 = R"END( unused = "global parameter" diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index db1948ef3a..0aa71cf7a8 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -220,17 +220,12 @@ def attributeRoundTrip(self, file_ending): np.array([4.5, 6.7], dtype=np.double)) series.set_attribute("nparr_longdouble", np.array([8.9, 7.6], dtype=np.longdouble)) - # note: looks like ADIOS 1.13.1 cannot write arrays of complex - # as attributes (writes 1st value for single and crashes - # in write for complex double) - # https://github.com/ornladios/ADIOS/issues/212 - if series.backend != "ADIOS1": - series.set_attribute("nparr_csingle", - np.array([1.2 - 0.3j, 2.3 + 4.2j], - dtype=np.complex64)) - series.set_attribute("nparr_cdouble", - np.array([4.5 + 1.1j, 6.7 - 2.2j], - dtype=np.complex128)) + series.set_attribute("nparr_csingle", + np.array([1.2 - 0.3j, 2.3 + 4.2j], + dtype=np.complex64)) + series.set_attribute("nparr_cdouble", + np.array([4.5 + 1.1j, 6.7 - 2.2j], + dtype=np.complex128)) if file_ending not in ["bp", "bp4", "bp5"]: series.set_attribute("nparr_clongdouble", np.array([8.9 + 7.8j, 7.6 + 9.2j], @@ -335,15 +330,13 @@ def attributeRoundTrip(self, file_ending): series.get_attribute("nparr_double"), [4.5, 6.7]) np.testing.assert_almost_equal( series.get_attribute("nparr_longdouble"), [8.9, 7.6]) - # see https://github.com/ornladios/ADIOS/issues/212 - if series.backend != "ADIOS1": - np.testing.assert_almost_equal( - series.get_attribute("nparr_csingle"), - np.array([1.2 - 0.3j, 2.3 + 4.2j], - dtype=np.complex64)) - np.testing.assert_almost_equal( - series.get_attribute("nparr_cdouble"), - [4.5 + 1.1j, 6.7 - 2.2j]) + np.testing.assert_almost_equal( + series.get_attribute("nparr_csingle"), + np.array([1.2 - 0.3j, 2.3 + 4.2j], + dtype=np.complex64)) + np.testing.assert_almost_equal( + series.get_attribute("nparr_cdouble"), + [4.5 + 1.1j, 6.7 - 2.2j]) # not in ADIOS 1.13.1 nor ADIOS 2.7.0 if file_ending not in ["bp", "bp4", "bp5"]: np.testing.assert_almost_equal( @@ -811,7 +804,6 @@ def testEmptyRecords(self): backend_filesupport = { 'json': 'json', 'hdf5': 'h5', - 'adios1': 'bp', 'adios2': 'bp' } for b in io.variants: @@ -1709,25 +1701,22 @@ def makeCloseIterationRoundTrip(self, file_ending): E_x = it0.meshes["E"]["x"] E_x.reset_dataset(DS(np.dtype("int"), extent)) E_x.store_chunk(data, [0], extent) - is_adios1 = series.backend == 'ADIOS1' it0.close(flush=True) - # not supported in ADIOS1: can only open one ADIOS1 series at a time - if not is_adios1: - read = io.Series( - "../samples/unittest_closeIteration_%T." + file_ending, - io.Access_Type.read_only - ) - it0 = read.iterations[0] - E_x = it0.meshes["E"]["x"] - chunk = E_x.load_chunk([0], extent) - it0.close() # flush = True <- default argument + read = io.Series( + "../samples/unittest_closeIteration_%T." + file_ending, + io.Access_Type.read_only + ) + it0 = read.iterations[0] + E_x = it0.meshes["E"]["x"] + chunk = E_x.load_chunk([0], extent) + it0.close() # flush = True <- default argument - for i in range(len(data)): - self.assertEqual(data[i], chunk[i]) - self.assertTrue(read) - read.close() - self.assertFalse(read) + for i in range(len(data)): + self.assertEqual(data[i], chunk[i]) + self.assertTrue(read) + read.close() + self.assertFalse(read) it1 = series.iterations[1] E_x = it1.meshes["E"]["x"] @@ -1736,22 +1725,21 @@ def makeCloseIterationRoundTrip(self, file_ending): it1.close(flush=False) series.flush() - if not is_adios1: - read = io.Series( - "../samples/unittest_closeIteration_%T." + file_ending, - io.Access_Type.read_only - ) - it1 = read.iterations[1] - E_x = it1.meshes["E"]["x"] - chunk = E_x.load_chunk([0], extent) - it1.close(flush=False) - read.flush() + read = io.Series( + "../samples/unittest_closeIteration_%T." + file_ending, + io.Access_Type.read_only + ) + it1 = read.iterations[1] + E_x = it1.meshes["E"]["x"] + chunk = E_x.load_chunk([0], extent) + it1.close(flush=False) + read.flush() - for i in range(len(data)): - self.assertEqual(data[i], chunk[i]) - self.assertTrue(read) - read.close() - self.assertFalse(read) + for i in range(len(data)): + self.assertEqual(data[i], chunk[i]) + self.assertTrue(read) + read.close() + self.assertFalse(read) def testCloseIteration(self): for ext in tested_file_extensions: @@ -1829,7 +1817,6 @@ def testIterator(self): backend_filesupport = { 'json': 'json', 'hdf5': 'h5', - 'adios1': 'bp', 'adios2': 'bp' } for b in io.variants: @@ -2003,9 +1990,6 @@ def testJsonConfigADIOS2(self): "../samples/unittest_jsonConfiguredBP3.bp", io.Access_Type.create, global_config) - if series.backend != 'ADIOS2': - # might happen, if env. var. OPENPMD_BP_BACKEND is used - return DS = io.Dataset data = np.array(range(1000), dtype=np.dtype("double")) From 1e2e6dd9984c626da377377f950cd67739223ef5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Apr 2023 09:57:38 +0200 Subject: [PATCH 2/3] Fix forward declaration of ADIOS2IOHandlerImpl in Writable.hpp --- include/openPMD/backend/Writable.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 7ee1eaa341..c585893fc3 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -78,7 +78,6 @@ class Writable final friend class Series; friend class Record; friend class AbstractIOHandlerImpl; - template friend class ADIOS2IOHandlerImpl; friend class HDF5IOHandlerImpl; friend class ParallelHDF5IOHandlerImpl; From 9f0e1c4fe8a307de471a7324d1a03bd4746729b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Apr 2023 10:17:35 +0200 Subject: [PATCH 3/3] Remove mentions of ADIOS1 from Serial testing --- test/SerialIOTest.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index e58db2bff0..0607a19a4a 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -4468,10 +4468,8 @@ TEST_CASE("adios2_engines_and_file_endings") Series read( name, Access::READ_ONLY, - isThisADIOS1 - ? "backend = \"adios1\"" - : "backend = \"adios2\"\nadios2.engine.type = \"" + - requiredEngine + "\""); + "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); } }; @@ -4625,10 +4623,8 @@ TEST_CASE("adios2_engines_and_file_endings") Series read( name, Access::READ_ONLY, - isThisADIOS1 - ? "backend = \"adios1\"" - : "backend = \"adios2\"\nadios2.engine.type = \"" + - requiredEngine + "\""); + "backend = \"adios2\"\nadios2.engine.type = \"" + + requiredEngine + "\""); } };