Use ADIOS variables for openPMD attributes

.github/workflows/unix.yml

@@ -78,6 +78,30 @@ jobs:
         export OPENPMD_BP_BACKEND=ADIOS1
         ctest --output-on-failure
 
+  clang5_nopy_ompi_h5_ad2_newLayout_bp3:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Spack Cache
+      uses: actions/cache@v2
+      with: {path: /opt/spack, key: clang5_nopy_ompi_h5_ad1_ad2_bp3 }
+    - name: Install
+      run: |
+        sudo apt-get install clang-5.0 gfortran libopenmpi-dev python3
+        sudo .github/workflows/dependencies/install_spack
+    - name: Build
+      env: {CC: clang-5.0, CXX: clang++-5.0, CXXFLAGS: -Werror -Wno-deprecated-declarations}
+      run: |
+        eval $(spack env activate --sh .github/ci/spack-envs/clang5_nopy_ompi_h5_ad1_ad2_bp3/)
+        spack install
+
+        mkdir build && cd build
+        ../share/openPMD/download_samples.sh && chmod u-w samples/git-sample/*.h5
+        cmake -S .. -B . -DopenPMD_USE_PYTHON=OFF -DopenPMD_USE_MPI=ON -DopenPMD_USE_HDF5=ON -DopenPMD_USE_ADIOS1=OFF -DopenPMD_USE_ADIOS2=ON -DopenPMD_USE_INVASIVE_TESTS=ON
+        cmake --build . --parallel 2
+        export OPENPMD_NEW_ATTRIBUTE_LAYOUT=1
+        ctest --output-on-failure
+
 # TODO
 #  clang6_py36_nompi_h5_ad1_ad2_libcpp
 

CMakeLists.txt

@@ -370,6 +370,7 @@ set(IO_SOURCE
         src/IO/JSON/JSONFilePosition.cpp
         src/IO/ADIOS/ADIOS2IOHandler.cpp
         src/IO/ADIOS/ADIOS2Auxiliary.cpp
+        src/IO/ADIOS/ADIOS2PreloadAttributes.cpp
         src/IO/InvalidatableFile.cpp)
 set(IO_ADIOS1_SEQUENTIAL_SOURCE
         src/auxiliary/Filesystem.cpp

docs/source/backends/adios2.rst

@@ -75,6 +75,29 @@ A good number for substreams is usually the number of contributing nodes divided
 For fine-tuning at extreme scale or for exotic systems, please refer to the ADIOS2 manual and talk to your filesystem admins and the ADIOS2 authors.
 Be aware that extreme-scale I/O is a research topic after all.
 
+Experimental new attribute layout
+---------------------------------
+
+We are experimenting with a breaking change to our layout of openPMD datasets in ADIOS2.
+It is likely that we will in future use ADIOS attributes only for a handful of internal flags.
+Actual openPMD attributes will be modeled by ADIOS variables of the same name.
+In order to distinguish datasets from attributes, datasets will be suffixed by ``/__data__``.
+
+We hope that this will bring several advantages:
+
+* Unlike ADIOS attributes, ADIOS variables are mutable.
+* ADIOS variables are more closely related to the concept of ADIOS steps.
+  An ADIOS variable that is not written to in one step is not seen by the reader.
+  This will bring more manageable amounts of metadata for readers to parse through.
+
+The new layout may be activated **for experimental purposes** in two ways:
+
+* Via the JSON parameter ``adios2.new_attribute_layout = true``.
+* Via the environment variable ``export OPENPMD_NEW_ATTRIBUTE_LAYOUT=1``.
+
+The classical and the new layout are absolutely incompatible with one another.
+The ADIOS2 backend will **not** (yet) automatically recognize the layout that has been used by a writer when reading a dataset.
+
 Selected References
 -------------------
 

include/openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp

@@ -23,12 +23,15 @@
 
 #include "openPMD/config.hpp"
 #if openPMD_HAVE_ADIOS2
-#include "openPMD/Datatype.hpp"
-#include <adios2.h>
-#include <complex>
-#include <stdexcept>
-#include <utility>
-#include <vector>
+# include "openPMD/Dataset.hpp"
+# include "openPMD/Datatype.hpp"
+
+# include <adios2.h>
+
+# include <complex>
+# include <stdexcept>
+# include <utility>
+# include <vector>
 
 namespace openPMD
 {
@@ -75,16 +78,25 @@ namespace detail
      */
     Datatype fromADIOS2Type( std::string const & dt );
 
+    enum class VariableOrAttribute : unsigned char
+    {
+        Variable,
+        Attribute
+    };
+
     template < typename T > struct AttributeInfoHelper
     {
-        static typename std::vector< T >::size_type
-        getSize( adios2::IO &, std::string const & attributeName );
+        static Extent
+        getSize(
+            adios2::IO &,
+            std::string const & attributeName,
+            VariableOrAttribute );
     };
 
     template < > struct AttributeInfoHelper< std::complex< long double > >
     {
-        static typename std::vector< long double >::size_type
-        getSize( adios2::IO &, std::string const & )
+        static Extent
+        getSize( adios2::IO &, std::string const &, VariableOrAttribute )
         {
             throw std::runtime_error(
                 "[ADIOS2] Internal error: no support for long double complex attribute types" );
@@ -93,14 +105,17 @@ namespace detail
 
     template < typename T > struct AttributeInfoHelper< std::vector< T > >
     {
-        static typename std::vector< T >::size_type
-        getSize( adios2::IO &, std::string const & attributeName );
+        static Extent
+        getSize(
+            adios2::IO &,
+            std::string const & attributeName,
+            VariableOrAttribute );
     };
 
     template < > struct AttributeInfoHelper< std::vector< std::complex< long double > > >
     {
-        static typename std::vector< std::complex< long double > >::size_type
-        getSize( adios2::IO &, std::string const & )
+        static Extent
+        getSize( adios2::IO &, std::string const &, VariableOrAttribute )
         {
             throw std::runtime_error(
                 "[ADIOS2] Internal error: no support for long double complex vector attribute types" );
@@ -110,27 +125,33 @@ namespace detail
     template < typename T, std::size_t n >
     struct AttributeInfoHelper< std::array< T, n > >
     {
-        static typename std::vector< T >::size_type
-        getSize( adios2::IO & IO, std::string const & attributeName )
+        static Extent
+        getSize(
+            adios2::IO & IO,
+            std::string const & attributeName,
+            VariableOrAttribute voa )
         {
-            return AttributeInfoHelper< T >::getSize( IO, attributeName );
+            return AttributeInfoHelper< T >::getSize( IO, attributeName, voa );
         }
     };
 
     template <> struct AttributeInfoHelper< bool >
     {
-        static typename std::vector< bool_representation >::size_type
-        getSize( adios2::IO &, std::string const & attributeName );
+        static Extent
+        getSize(
+            adios2::IO &,
+            std::string const & attributeName,
+            VariableOrAttribute );
     };
 
     struct AttributeInfo
     {
-        template < typename T >
-        typename std::vector< T >::size_type
-        operator( )( adios2::IO &, std::string const & attributeName );
+        template< typename T, typename... Params >
+        Extent
+        operator()( Params &&... );
 
         template < int n, typename... Params >
-        size_t operator( )( Params &&... );
+        Extent operator( )( Params &&... );
     };
 
     /**
@@ -146,7 +167,44 @@ namespace detail
     attributeInfo(
         adios2::IO & IO,
         std::string const & attributeName,
-        bool verbose );
+        bool verbose,
+        VariableOrAttribute = VariableOrAttribute::Attribute );
+
+    template< typename T >
+    struct IsTrivialType
+    {
+        constexpr static bool val = true;
+    };
+
+    template< typename T >
+    struct IsTrivialType< std::vector< T > >
+    {
+        constexpr static bool val = false;
+    };
+
+    template< typename T, size_t n >
+    struct IsTrivialType< std::array< T, n > >
+    {
+        constexpr static bool val = false;
+    };
+
+    template<>
+    struct IsTrivialType< bool >
+    {
+        constexpr static bool val = false;
+    };
+
+    template<>
+    struct IsTrivialType< std::complex< long double > >
+    {
+        constexpr static bool val = false;
+    };
+
+    template<>
+    struct IsTrivialType< std::vector< std::complex< long double > > >
+    {
+        constexpr static bool val = false;
+    };
 } // namespace detail
 
 } // namespace openPMD