ADIOS1: Create Attribute-Only Files

CHANGELOG.rst

@@ -26,6 +26,9 @@ Bug Fixes
 """""""""
 
 - ``std::ostream& operator<<`` overloads are not declared in namespace ``std`` anymore #662
+- ADIOS1:
+
+  - ensure creation of files that only contain attributes #674
 
 Other
 """""

include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp

@@ -52,6 +52,8 @@ namespace openPMD
         ADIOS1IOHandler(std::string path, AccessType);
         ~ADIOS1IOHandler() override;
 
+        std::string backendName() const override { return "ADIOS1"; }
+
         std::future< void > flush() override;
 
         void enqueue(IOTask const&) override;
@@ -69,6 +71,8 @@ namespace openPMD
         ADIOS1IOHandler(std::string path, AccessType);
         ~ADIOS1IOHandler() override;
 
+        std::string backendName() const override { return "DUMMY_ADIOS1"; }
+
         std::future< void > flush() override;
 
     private:

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -658,6 +658,8 @@ friend class ADIOS2IOHandlerImpl;
 
     ADIOS2IOHandler( std::string path, AccessType );
 
+    std::string backendName() const override { return "ADIOS2"; }
+
     std::future< void > flush( ) override;
 }; // ADIOS2IOHandler
 } // namespace openPMD

include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp

@@ -55,6 +55,8 @@ namespace openPMD
 #   endif
         ~ParallelADIOS1IOHandler() override;
 
+        std::string backendName() const override { return "MPI_ADIOS1"; }
+
         std::future< void > flush() override;
 #if openPMD_HAVE_ADIOS1
         void enqueue(IOTask const&) override;

include/openPMD/IO/AbstractIOHandler.hpp

@@ -97,6 +97,9 @@ class AbstractIOHandler
      */
     virtual std::future< void > flush() = 0;
 
+    /** The currently used backend */
+    virtual std::string backendName() const = 0;
+
     std::string const directory;
     AccessType const accessTypeBackend;
     AccessType const accessTypeFrontend;

include/openPMD/IO/HDF5/HDF5IOHandler.hpp

@@ -37,6 +37,8 @@ class HDF5IOHandler : public AbstractIOHandler
     HDF5IOHandler(std::string path, AccessType);
     ~HDF5IOHandler() override;
 
+    std::string backendName() const override { return "HDF5"; }
+
     std::future< void > flush() override;
 
 private:

include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp

@@ -42,6 +42,8 @@ namespace openPMD
     #endif
         ~ParallelHDF5IOHandler() override;
 
+        std::string backendName() const override { return "MPI_HDF5"; }
+
         std::future< void > flush() override;
 
     private:

include/openPMD/IO/JSON/JSONIOHandler.hpp

@@ -39,6 +39,8 @@ namespace openPMD
 
         ~JSONIOHandler( ) override;
 
+        std::string backendName() const override { return "JSON"; }
+
         std::future< void > flush( ) override;
 
     private:

include/openPMD/Series.hpp

@@ -237,6 +237,9 @@ class Series : public Attributable
      */
     Series& setName(std::string const& name);
 
+    /** The currently used backend */
+    std::string backend() const;
+
     /** Execute all required remaining IO operations to write or read data.
      */
     void flush();

src/IO/ADIOS/CommonADIOS1IOHandler.cpp

@@ -756,7 +756,7 @@ CommonADIOS1IOHandlerImpl::writeDataset(Writable* writable,
     if( m_handler->accessTypeBackend == AccessType::READ_ONLY )
         throw std::runtime_error("[ADIOS1] Writing into a dataset in a file opened as read only is not possible.");
 
-    /* file opening is deferred until the first dataset write to a file occurs */
+    /* file opening is deferred until the first dataset/attribute write to a file occurs */
     auto res = m_filePaths.find(writable);
     if( res == m_filePaths.end() )
         res = m_filePaths.find(writable->parent);
@@ -803,9 +803,18 @@ CommonADIOS1IOHandlerImpl::writeAttribute(Writable* writable,
         name += '/';
     name += parameters.name;
 
+    /* file opening is deferred until the first dataset/attribute write to a file occurs */
     auto res = m_filePaths.find(writable);
     if( res == m_filePaths.end() )
         res = m_filePaths.find(writable->parent);
+    int64_t fd;
+    if( m_openWriteFileHandles.find(res->second) == m_openWriteFileHandles.end() )
+    {
+        fd = open_write(writable);
+        m_openWriteFileHandles[res->second] = fd;
+    } else
+        fd = m_openWriteFileHandles.at(res->second);
+
     int64_t group = m_groups[res->second];
 
     auto& attributes = m_attributeWrites[group];

src/Series.cpp

@@ -353,6 +353,12 @@ Series::setName(std::string const& n)
     return *this;
 }
 
+std::string
+Series::backend() const
+{
+    return IOHandler->backendName();
+}
+
 void
 Series::flush()
 {

test/SerialIOTest.cpp

@@ -673,6 +673,7 @@ void fileBased_write_test(const std::string & backend)
         std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t);
         e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4}));
 
+        //o.setOpenPMDextension(1); // this would be before the first flush
         for( uint64_t i = 0; i < 4; ++i )
         {
             *position_local_1 = position_global[i];
@@ -723,6 +724,9 @@ void fileBased_write_test(const std::string & backend)
 
         o.setOpenPMDextension(1);
         o.iterations[3].setTime(static_cast< double >(3));
+        o.iterations[4].setTime(static_cast< double >(4));
+        o.flush();
+        o.iterations[5].setTime(static_cast< double >(5));
     }
     REQUIRE((auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000001." + backend)
         || auxiliary::directory_exists("../samples/subdir/serial_fileBased_write00000001." + backend)));
@@ -734,10 +738,12 @@ void fileBased_write_test(const std::string & backend)
     {
         Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY);
 
-        REQUIRE(o.iterations.size() == 3);
+        REQUIRE(o.iterations.size() == 5);
         REQUIRE(o.iterations.count(1) == 1);
         REQUIRE(o.iterations.count(2) == 1);
         REQUIRE(o.iterations.count(3) == 1);
+        REQUIRE(o.iterations.count(4) == 1);
+        REQUIRE(o.iterations.count(5) == 1);
 
 #if openPMD_USE_INVASIVE_TESTS
         REQUIRE(*o.m_filenamePadding == 8);
@@ -759,6 +765,10 @@ void fileBased_write_test(const std::string & backend)
             REQUIRE(it.dt< double >() == 1.);
             REQUIRE(it.time< double >() == static_cast< double >(entry.first));
             REQUIRE(it.timeUnitSI() == 1.);
+
+            if( entry.first > 3 )
+                continue; // empty iterations
+
             auto& pos = it.particles.at("e").at("position");
             REQUIRE(pos.timeOffset< float >() == 0.f);
             REQUIRE(pos.unitDimension() == udim);
@@ -788,15 +798,18 @@ void fileBased_write_test(const std::string & backend)
               REQUIRE(positionOffset_raw[j] == j + (entry.first-1)*4);
             }
         }
+        REQUIRE(o.iterations[3].time< double >() == 3.0);
+        REQUIRE(o.iterations[4].time< double >() == 4.0);
+        REQUIRE(o.iterations[5].time< double >() == 5.0);
     }
 
     // extend existing series with new step and auto-detection of iteration padding
     {
         Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_WRITE);
 
-        REQUIRE(o.iterations.size() == 3);
-        o.iterations[4];
-        REQUIRE(o.iterations.size() == 4);
+        REQUIRE(o.iterations.size() == 5);
+        o.iterations[6];
+        REQUIRE(o.iterations.size() == 6);
     }
     REQUIRE((auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000004." + backend)
         || auxiliary::directory_exists("../samples/subdir/serial_fileBased_write00000004." + backend)));
@@ -822,7 +835,7 @@ void fileBased_write_test(const std::string & backend)
     // read back with auto-detection and non-fixed padding
     {
         Series s = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY);
-        REQUIRE(s.iterations.size() == 5);
+        REQUIRE(s.iterations.size() == 7);
     }
 
     // write with auto-detection and in-consistent padding
@@ -834,7 +847,7 @@ void fileBased_write_test(const std::string & backend)
     // read back with auto-detection and fixed padding
     {
         Series s = Series("../samples/subdir/serial_fileBased_write%08T." + backend, AccessType::READ_ONLY);
-        REQUIRE(s.iterations.size() == 4);
+        REQUIRE(s.iterations.size() == 6);
     }
 }