diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d5928babc..52f25c1746 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -26,6 +26,9 @@ Bug Fixes """"""""" - ``std::ostream& operator<<`` overloads are not declared in namespace ``std`` anymore #662 +- ADIOS1: + + - ensure creation of files that only contain attributes #674 Other """"" diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp index 136e9c6937..fe9c2419d8 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp @@ -52,6 +52,8 @@ namespace openPMD ADIOS1IOHandler(std::string path, AccessType); ~ADIOS1IOHandler() override; + std::string backendName() const override { return "ADIOS1"; } + std::future< void > flush() override; void enqueue(IOTask const&) override; @@ -69,6 +71,8 @@ namespace openPMD ADIOS1IOHandler(std::string path, AccessType); ~ADIOS1IOHandler() override; + std::string backendName() const override { return "DUMMY_ADIOS1"; } + std::future< void > flush() override; private: diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index f142c0556b..1c8bce6df5 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -658,6 +658,8 @@ friend class ADIOS2IOHandlerImpl; ADIOS2IOHandler( std::string path, AccessType ); + std::string backendName() const override { return "ADIOS2"; } + std::future< void > flush( ) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp index 1c1cedbd5d..7be83bb691 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp @@ -55,6 +55,8 @@ namespace openPMD # endif ~ParallelADIOS1IOHandler() override; + std::string backendName() const override { return "MPI_ADIOS1"; } + std::future< void > flush() override; #if openPMD_HAVE_ADIOS1 void enqueue(IOTask const&) override; diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 5e41b9cf3f..596b609a98 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -97,6 +97,9 @@ class AbstractIOHandler */ virtual std::future< void > flush() = 0; + /** The currently used backend */ + virtual std::string backendName() const = 0; + std::string const directory; AccessType const accessTypeBackend; AccessType const accessTypeFrontend; diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 7a5ea07e4d..76bfed8a1a 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -37,6 +37,8 @@ class HDF5IOHandler : public AbstractIOHandler HDF5IOHandler(std::string path, AccessType); ~HDF5IOHandler() override; + std::string backendName() const override { return "HDF5"; } + std::future< void > flush() override; private: diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index 32253f7235..614a69851d 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -42,6 +42,8 @@ namespace openPMD #endif ~ParallelHDF5IOHandler() override; + std::string backendName() const override { return "MPI_HDF5"; } + std::future< void > flush() override; private: diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 9c974ebce0..0b6b3a1dc9 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -39,6 +39,8 @@ namespace openPMD ~JSONIOHandler( ) override; + std::string backendName() const override { return "JSON"; } + std::future< void > flush( ) override; private: diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 57231309aa..be8f3400c0 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -237,6 +237,9 @@ class Series : public Attributable */ Series& setName(std::string const& name); + /** The currently used backend */ + std::string backend() const; + /** Execute all required remaining IO operations to write or read data. */ void flush(); diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index c9db60b87e..88277b805d 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -756,7 +756,7 @@ CommonADIOS1IOHandlerImpl::writeDataset(Writable* writable, if( m_handler->accessTypeBackend == AccessType::READ_ONLY ) throw std::runtime_error("[ADIOS1] Writing into a dataset in a file opened as read only is not possible."); - /* file opening is deferred until the first dataset write to a file occurs */ + /* file opening is deferred until the first dataset/attribute write to a file occurs */ auto res = m_filePaths.find(writable); if( res == m_filePaths.end() ) res = m_filePaths.find(writable->parent); @@ -803,9 +803,18 @@ CommonADIOS1IOHandlerImpl::writeAttribute(Writable* writable, name += '/'; name += parameters.name; + /* file opening is deferred until the first dataset/attribute write to a file occurs */ auto res = m_filePaths.find(writable); if( res == m_filePaths.end() ) res = m_filePaths.find(writable->parent); + int64_t fd; + if( m_openWriteFileHandles.find(res->second) == m_openWriteFileHandles.end() ) + { + fd = open_write(writable); + m_openWriteFileHandles[res->second] = fd; + } else + fd = m_openWriteFileHandles.at(res->second); + int64_t group = m_groups[res->second]; auto& attributes = m_attributeWrites[group]; diff --git a/src/Series.cpp b/src/Series.cpp index c055d5c62e..2c3d341e39 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -353,6 +353,12 @@ Series::setName(std::string const& n) return *this; } +std::string +Series::backend() const +{ + return IOHandler->backendName(); +} + void Series::flush() { diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index d8dd903486..adddb6784f 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -673,6 +673,7 @@ void fileBased_write_test(const std::string & backend) std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); + //o.setOpenPMDextension(1); // this would be before the first flush for( uint64_t i = 0; i < 4; ++i ) { *position_local_1 = position_global[i]; @@ -723,6 +724,9 @@ void fileBased_write_test(const std::string & backend) o.setOpenPMDextension(1); o.iterations[3].setTime(static_cast< double >(3)); + o.iterations[4].setTime(static_cast< double >(4)); + o.flush(); + o.iterations[5].setTime(static_cast< double >(5)); } REQUIRE((auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000001." + backend) || auxiliary::directory_exists("../samples/subdir/serial_fileBased_write00000001." + backend))); @@ -734,10 +738,12 @@ void fileBased_write_test(const std::string & backend) { Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY); - REQUIRE(o.iterations.size() == 3); + REQUIRE(o.iterations.size() == 5); REQUIRE(o.iterations.count(1) == 1); REQUIRE(o.iterations.count(2) == 1); REQUIRE(o.iterations.count(3) == 1); + REQUIRE(o.iterations.count(4) == 1); + REQUIRE(o.iterations.count(5) == 1); #if openPMD_USE_INVASIVE_TESTS REQUIRE(*o.m_filenamePadding == 8); @@ -759,6 +765,10 @@ void fileBased_write_test(const std::string & backend) REQUIRE(it.dt< double >() == 1.); REQUIRE(it.time< double >() == static_cast< double >(entry.first)); REQUIRE(it.timeUnitSI() == 1.); + + if( entry.first > 3 ) + continue; // empty iterations + auto& pos = it.particles.at("e").at("position"); REQUIRE(pos.timeOffset< float >() == 0.f); REQUIRE(pos.unitDimension() == udim); @@ -788,15 +798,18 @@ void fileBased_write_test(const std::string & backend) REQUIRE(positionOffset_raw[j] == j + (entry.first-1)*4); } } + REQUIRE(o.iterations[3].time< double >() == 3.0); + REQUIRE(o.iterations[4].time< double >() == 4.0); + REQUIRE(o.iterations[5].time< double >() == 5.0); } // extend existing series with new step and auto-detection of iteration padding { Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_WRITE); - REQUIRE(o.iterations.size() == 3); - o.iterations[4]; - REQUIRE(o.iterations.size() == 4); + REQUIRE(o.iterations.size() == 5); + o.iterations[6]; + REQUIRE(o.iterations.size() == 6); } REQUIRE((auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000004." + backend) || auxiliary::directory_exists("../samples/subdir/serial_fileBased_write00000004." + backend))); @@ -822,7 +835,7 @@ void fileBased_write_test(const std::string & backend) // read back with auto-detection and non-fixed padding { Series s = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY); - REQUIRE(s.iterations.size() == 5); + REQUIRE(s.iterations.size() == 7); } // write with auto-detection and in-consistent padding @@ -834,7 +847,7 @@ void fileBased_write_test(const std::string & backend) // read back with auto-detection and fixed padding { Series s = Series("../samples/subdir/serial_fileBased_write%08T." + backend, AccessType::READ_ONLY); - REQUIRE(s.iterations.size() == 4); + REQUIRE(s.iterations.size() == 6); } }