From 8fe3157dbe063721164055f1ece335f50a5ad163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 11 Mar 2025 17:51:11 +0100 Subject: [PATCH 01/14] Steal ADIOS2PreloadAttributes from the past MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Totgesagte leben länger Removed in #1310 --- CMakeLists.txt | 1 + .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 139 ++++++++ src/IO/ADIOS/ADIOS2PreloadAttributes.cpp | 314 ++++++++++++++++++ 3 files changed, 454 insertions(+) create mode 100644 include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp create mode 100644 src/IO/ADIOS/ADIOS2PreloadAttributes.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3af9872195..367bc15e92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -437,6 +437,7 @@ set(IO_SOURCE src/IO/JSON/JSONIOHandlerImpl.cpp src/IO/JSON/JSONFilePosition.cpp src/IO/ADIOS/ADIOS2IOHandler.cpp + src/IO/ADIOS/ADIOS2PreloadAttributes.cpp src/IO/ADIOS/ADIOS2File.cpp src/IO/ADIOS/ADIOS2Auxiliary.cpp src/IO/InvalidatableFile.cpp) diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp new file mode 100644 index 0000000000..f831f543c2 --- /dev/null +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -0,0 +1,139 @@ +/* Copyright 2020-2021 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/config.hpp" +#if openPMD_HAVE_ADIOS2 + +#include +#include +#include +#include +#include +#include + +#include "openPMD/Datatype.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" + +namespace openPMD::detail +{ +/** + * @brief Pointer to an attribute's data along with its shape. + * + * @tparam T Underlying attribute data type. + */ +template +struct AttributeWithShape +{ + adios2::Dims shape; + T const *data; +}; + +/** + * Class that is responsible for scheduling and buffering openPMD attribute + * loads from ADIOS2, if using ADIOS variables to store openPMD attributes. + * + * Reasoning: ADIOS variables can be of any shape and size, and ADIOS cannot + * know which variables to buffer. While it will preload and buffer scalar + * variables, openPMD also stores vector-type attributes which are not + * preloaded. Since in Streaming setups, every variable load requires full + * communication back to the writer, this can quickly become very expensive. + * Hence, do this manually. + * + */ +class PreloadAdiosAttributes +{ +public: + /** + * Internally used struct to store meta information on a buffered + * attribute. Public for simplicity (helper struct in the + * implementation uses it). + */ + struct AttributeLocation + { + adios2::Dims shape; + size_t offset; + Datatype dt; + char *destroy = nullptr; + + AttributeLocation() = delete; + AttributeLocation(adios2::Dims shape, size_t offset, Datatype dt); + + AttributeLocation(AttributeLocation const &other) = delete; + AttributeLocation &operator=(AttributeLocation const &other) = delete; + + AttributeLocation(AttributeLocation &&other); + AttributeLocation &operator=(AttributeLocation &&other); + + ~AttributeLocation(); + }; + +private: + /* + * Allocate one large buffer instead of hundreds of single heap + * allocations. + * This will comply with alignment requirements, since + * std::allocator::allocate() will call the untyped new operator + * ::operator new(std::size_t) + * https://en.cppreference.com/w/cpp/memory/allocator/allocate + */ + std::vector m_rawBuffer; + std::map m_offsets; + +public: + explicit PreloadAdiosAttributes() = default; + PreloadAdiosAttributes(PreloadAdiosAttributes const &other) = delete; + PreloadAdiosAttributes & + operator=(PreloadAdiosAttributes const &other) = delete; + + PreloadAdiosAttributes(PreloadAdiosAttributes &&other) = default; + PreloadAdiosAttributes &operator=(PreloadAdiosAttributes &&other) = default; + + /** + * @brief Schedule attributes for preloading. + * + * This will invalidate all previously buffered attributes. + * This will *not* flush the scheduled loads. This way, attributes can + * be loaded along with the next adios2::Engine flush. + * + * @param IO + * @param engine + */ + void preloadAttributes(adios2::IO &IO, adios2::Engine &engine); + + /** + * @brief Get an attribute that has been buffered previously. + * + * @tparam T The underlying primitive datatype of the attribute. + * Will fail if the type found in ADIOS does not match. + * @param name The full name of the attribute. + * @return Pointer to the buffered attribute along with information on + * the attribute's shape. Valid only until any non-const member + * of PreloadAdiosAttributes is called. + */ + template + AttributeWithShape getAttribute(std::string const &name) const; + + Datatype attributeType(std::string const &name) const; +}; +} // namespace openPMD::detail + +#endif // openPMD_HAVE_ADIOS2 diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp new file mode 100644 index 0000000000..88ca80d7f8 --- /dev/null +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -0,0 +1,314 @@ +/* Copyright 2020-2021 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/config.hpp" +#if openPMD_HAVE_ADIOS2 + +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" + +#include "openPMD/Datatype.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" +#include "openPMD/auxiliary/StringManip.hpp" + +#include +#include +#include +#include +#include + +namespace openPMD::detail +{ +namespace +{ + struct GetAlignment + { + template + static constexpr size_t call() + { + return alignof(T); + } + + template + static constexpr size_t call(Args &&...) + { + return alignof(std::max_align_t); + } + }; + + struct GetSize + { + template + static constexpr size_t call() + { + return sizeof(T); + } + + template + static constexpr size_t call(Args &&...) + { + return 0; + } + }; + + struct ScheduleLoad + { + template + static void call( + adios2::IO &IO, + adios2::Engine &engine, + std::string const &name, + char *buffer, + PreloadAdiosAttributes::AttributeLocation &location) + { + adios2::Variable var = IO.InquireVariable(name); + if (!var) + { + throw std::runtime_error( + "[ADIOS2] Variable not found: " + name); + } + adios2::Dims const &shape = location.shape; + adios2::Dims offset(shape.size(), 0); + if (shape.size() > 0) + { + var.SetSelection({offset, shape}); + } + T *dest = reinterpret_cast(buffer); + size_t numItems = 1; + for (auto extent : shape) + { + numItems *= extent; + } + /* + * MSVC does not like placement new of arrays, so we do it + * in a loop instead. + * https://developercommunity.visualstudio.com/t/c-placement-new-is-incorrectly-compiled/206439 + */ + for (size_t i = 0; i < numItems; ++i) + { + new (dest + i) T(); + } + location.destroy = buffer; + engine.Get(var, dest, adios2::Mode::Deferred); + } + + static constexpr char const *errorMsg = "ADIOS2"; + }; + + struct VariableShape + { + template + static adios2::Dims call(adios2::IO &IO, std::string const &name) + { + auto var = IO.InquireVariable(name); + if (!var) + { + throw std::runtime_error( + "[ADIOS2] Variable not found: " + name); + } + return var.Shape(); + } + + template + static adios2::Dims call(Args &&...) + { + return {}; + } + }; + + struct AttributeLocationDestroy + { + template + static void call(char *ptr, size_t numItems) + { + T *destroy = reinterpret_cast(ptr); + for (size_t i = 0; i < numItems; ++i) + { + destroy[i].~T(); + } + } + + template + static void call(Args &&...) + {} + }; +} // namespace + +using AttributeLocation = PreloadAdiosAttributes::AttributeLocation; + +AttributeLocation::AttributeLocation( + adios2::Dims shape_in, size_t offset_in, Datatype dt_in) + : shape(std::move(shape_in)), offset(offset_in), dt(dt_in) +{} + +AttributeLocation::AttributeLocation(AttributeLocation &&other) + : shape{std::move(other.shape)} + , offset{std::move(other.offset)} + , dt{std::move(other.dt)} + , destroy{std::move(other.destroy)} +{ + other.destroy = nullptr; +} + +AttributeLocation &AttributeLocation::operator=(AttributeLocation &&other) +{ + this->shape = std::move(other.shape); + this->offset = std::move(other.offset); + this->dt = std::move(other.dt); + this->destroy = std::move(other.destroy); + other.destroy = nullptr; + return *this; +} + +PreloadAdiosAttributes::AttributeLocation::~AttributeLocation() +{ + /* + * If the object has been moved from, this may be empty. + * Or else, if no custom destructor has been emplaced. + */ + if (destroy) + { + size_t length = 1; + for (auto ext : shape) + { + length *= ext; + } + switchAdios2AttributeType( + dt, destroy, length); + } +} + +void PreloadAdiosAttributes::preloadAttributes( + adios2::IO &IO, adios2::Engine &engine) +{ + m_offsets.clear(); + std::map > attributesByType; + auto addAttribute = [&attributesByType](Datatype dt, std::string name) { + constexpr size_t reserve = 10; + auto it = attributesByType.find(dt); + if (it == attributesByType.end()) + { + it = attributesByType.emplace_hint( + it, dt, std::vector()); + it->second.reserve(reserve); + } + it->second.push_back(std::move(name)); + }; + // PHASE 1: collect names of available attributes by ADIOS datatype + for (auto &variable : IO.AvailableVariables()) + { + if (auxiliary::ends_with(variable.first, "/__data__")) + { + continue; + } + // this will give us basic types only, no fancy vectors or similar + Datatype dt = fromADIOS2Type(IO.VariableType(variable.first)); + addAttribute(dt, std::move(variable.first)); + } + + // PHASE 2: get offsets for attributes in buffer + std::map offsets; + size_t currentOffset = 0; + for (auto &pair : attributesByType) + { + size_t alignment = switchAdios2AttributeType(pair.first); + size_t size = switchAdios2AttributeType(pair.first); + // go to next offset with valid alignment + size_t modulus = currentOffset % alignment; + if (modulus > 0) + { + currentOffset += alignment - modulus; + } + for (std::string &name : pair.second) + { + adios2::Dims shape = + switchAdios2AttributeType(pair.first, IO, name); + size_t elements = 1; + for (auto extent : shape) + { + elements *= extent; + } + m_offsets.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(name)), + std::forward_as_tuple( + std::move(shape), currentOffset, pair.first)); + currentOffset += elements * size; + } + } + // now, currentOffset is the number of bytes that we need to allocate + // PHASE 3: allocate new buffer and schedule loads + m_rawBuffer.resize(currentOffset); + for (auto &pair : m_offsets) + { + switchAdios2AttributeType( + pair.second.dt, + IO, + engine, + pair.first, + &m_rawBuffer[pair.second.offset], + pair.second); + } +} + +template +AttributeWithShape +PreloadAdiosAttributes::getAttribute(std::string const &name) const +{ + auto it = m_offsets.find(name); + if (it == m_offsets.end()) + { + throw std::runtime_error( + "[ADIOS2] Requested attribute not found: " + name); + } + AttributeLocation const &location = it->second; + Datatype determinedDatatype = determineDatatype(); + if (location.dt != determinedDatatype) + { + std::stringstream errorMsg; + errorMsg << "[ADIOS2] Wrong datatype for attribute: " << name + << "(location.dt=" << location.dt + << ", T=" << determineDatatype() << ")"; + throw std::runtime_error(errorMsg.str()); + } + AttributeWithShape res; + res.shape = location.shape; + res.data = reinterpret_cast(&m_rawBuffer[location.offset]); + return res; +} + +#define OPENPMD_INSTANTIATE_GETATTRIBUTE(type) \ + template AttributeWithShape PreloadAdiosAttributes::getAttribute( \ + std::string const &name) const; +ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(OPENPMD_INSTANTIATE_GETATTRIBUTE) +#undef OPENPMD_INSTANTIATE_GETATTRIBUTE + +Datatype PreloadAdiosAttributes::attributeType(std::string const &name) const +{ + auto it = m_offsets.find(name); + if (it == m_offsets.end()) + { + return Datatype::UNDEFINED; + } + return it->second.dt; +} +} // namespace openPMD::detail + +#endif // openPMD_HAVE_ADIOS2 From 49a078eb9d66af20332da69271fdf6f69307ef2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 12 Mar 2025 10:41:23 +0100 Subject: [PATCH 02/14] WIP: Rewrite preload class for attributes --- .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 20 ++++- src/IO/ADIOS/ADIOS2PreloadAttributes.cpp | 90 +++++++------------ 2 files changed, 46 insertions(+), 64 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index f831f543c2..7bc3679fa1 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -43,7 +43,7 @@ namespace openPMD::detail template struct AttributeWithShape { - adios2::Dims shape; + size_t len; T const *data; }; @@ -69,13 +69,13 @@ class PreloadAdiosAttributes */ struct AttributeLocation { - adios2::Dims shape; + size_t len; size_t offset; Datatype dt; char *destroy = nullptr; AttributeLocation() = delete; - AttributeLocation(adios2::Dims shape, size_t offset, Datatype dt); + AttributeLocation(size_t len, size_t offset, Datatype dt); AttributeLocation(AttributeLocation const &other) = delete; AttributeLocation &operator=(AttributeLocation const &other) = delete; @@ -117,7 +117,7 @@ class PreloadAdiosAttributes * @param IO * @param engine */ - void preloadAttributes(adios2::IO &IO, adios2::Engine &engine); + void preloadAttributes(adios2::IO &IO); /** * @brief Get an attribute that has been buffered previously. @@ -134,6 +134,18 @@ class PreloadAdiosAttributes Datatype attributeType(std::string const &name) const; }; + +struct AdiosAttributes +{ + using RandomAccess_t = std::vector; + struct StreamAccess_t + { + size_t m_currentStep = 0; + std::optional> m_attributes; + }; + + std::variant m_data; +}; } // namespace openPMD::detail #endif // openPMD_HAVE_ADIOS2 diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp index 88ca80d7f8..3971e2e8aa 100644 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -20,6 +20,7 @@ */ #include "openPMD/config.hpp" +#include #if openPMD_HAVE_ADIOS2 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" @@ -73,61 +74,50 @@ namespace template static void call( adios2::IO &IO, - adios2::Engine &engine, std::string const &name, char *buffer, PreloadAdiosAttributes::AttributeLocation &location) { - adios2::Variable var = IO.InquireVariable(name); - if (!var) + adios2::Attribute attr = IO.InquireAttribute(name); + if (!attr) { throw std::runtime_error( "[ADIOS2] Variable not found: " + name); } - adios2::Dims const &shape = location.shape; - adios2::Dims offset(shape.size(), 0); - if (shape.size() > 0) - { - var.SetSelection({offset, shape}); - } - T *dest = reinterpret_cast(buffer); - size_t numItems = 1; - for (auto extent : shape) - { - numItems *= extent; - } /* * MSVC does not like placement new of arrays, so we do it * in a loop instead. * https://developercommunity.visualstudio.com/t/c-placement-new-is-incorrectly-compiled/206439 */ - for (size_t i = 0; i < numItems; ++i) + T *dest = reinterpret_cast(buffer); + for (size_t i = 0; i < location.len; ++i) { new (dest + i) T(); } location.destroy = buffer; - engine.Get(var, dest, adios2::Mode::Deferred); + auto data = attr.Data(); + std::copy_n(data.begin(), data.size(), dest); } static constexpr char const *errorMsg = "ADIOS2"; }; - struct VariableShape + struct AttributeLen { template - static adios2::Dims call(adios2::IO &IO, std::string const &name) + static size_t call(adios2::IO &IO, std::string const &name) { - auto var = IO.InquireVariable(name); - if (!var) + auto attr = IO.InquireAttribute(name); + if (!attr) { throw std::runtime_error( "[ADIOS2] Variable not found: " + name); } - return var.Shape(); + return attr.Data().size(); } template - static adios2::Dims call(Args &&...) + static size_t call(Args &&...) { return {}; } @@ -154,25 +144,22 @@ namespace using AttributeLocation = PreloadAdiosAttributes::AttributeLocation; AttributeLocation::AttributeLocation( - adios2::Dims shape_in, size_t offset_in, Datatype dt_in) - : shape(std::move(shape_in)), offset(offset_in), dt(dt_in) + size_t len_in, size_t offset_in, Datatype dt_in) + : len(len_in), offset(offset_in), dt(dt_in) {} AttributeLocation::AttributeLocation(AttributeLocation &&other) - : shape{std::move(other.shape)} - , offset{std::move(other.offset)} - , dt{std::move(other.dt)} - , destroy{std::move(other.destroy)} + : len{other.len}, offset{other.offset}, dt{other.dt}, destroy{other.destroy} { other.destroy = nullptr; } AttributeLocation &AttributeLocation::operator=(AttributeLocation &&other) { - this->shape = std::move(other.shape); - this->offset = std::move(other.offset); - this->dt = std::move(other.dt); - this->destroy = std::move(other.destroy); + this->len = other.len; + this->offset = other.offset; + this->dt = other.dt; + this->destroy = other.destroy; other.destroy = nullptr; return *this; } @@ -185,18 +172,11 @@ PreloadAdiosAttributes::AttributeLocation::~AttributeLocation() */ if (destroy) { - size_t length = 1; - for (auto ext : shape) - { - length *= ext; - } - switchAdios2AttributeType( - dt, destroy, length); + switchAdios2AttributeType(dt, destroy, len); } } -void PreloadAdiosAttributes::preloadAttributes( - adios2::IO &IO, adios2::Engine &engine) +void PreloadAdiosAttributes::preloadAttributes(adios2::IO &IO) { m_offsets.clear(); std::map > attributesByType; @@ -212,15 +192,11 @@ void PreloadAdiosAttributes::preloadAttributes( it->second.push_back(std::move(name)); }; // PHASE 1: collect names of available attributes by ADIOS datatype - for (auto &variable : IO.AvailableVariables()) + for (auto &attribute : IO.AvailableAttributes()) { - if (auxiliary::ends_with(variable.first, "/__data__")) - { - continue; - } // this will give us basic types only, no fancy vectors or similar - Datatype dt = fromADIOS2Type(IO.VariableType(variable.first)); - addAttribute(dt, std::move(variable.first)); + Datatype dt = fromADIOS2Type(IO.AttributeType(attribute.first)); + addAttribute(dt, attribute.first); } // PHASE 2: get offsets for attributes in buffer @@ -238,18 +214,13 @@ void PreloadAdiosAttributes::preloadAttributes( } for (std::string &name : pair.second) { - adios2::Dims shape = - switchAdios2AttributeType(pair.first, IO, name); - size_t elements = 1; - for (auto extent : shape) - { - elements *= extent; - } + size_t elements = + switchAdios2AttributeType(pair.first, IO, name); + m_offsets.emplace( std::piecewise_construct, std::forward_as_tuple(std::move(name)), - std::forward_as_tuple( - std::move(shape), currentOffset, pair.first)); + std::forward_as_tuple(elements, currentOffset, pair.first)); currentOffset += elements * size; } } @@ -261,7 +232,6 @@ void PreloadAdiosAttributes::preloadAttributes( switchAdios2AttributeType( pair.second.dt, IO, - engine, pair.first, &m_rawBuffer[pair.second.offset], pair.second); @@ -289,7 +259,7 @@ PreloadAdiosAttributes::getAttribute(std::string const &name) const throw std::runtime_error(errorMsg.str()); } AttributeWithShape res; - res.shape = location.shape; + res.len = location.len; res.data = reinterpret_cast(&m_rawBuffer[location.offset]); return res; } From 72be3cbabb52c0a20c1f19996557675cd0bfb8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 12 Mar 2025 11:38:52 +0100 Subject: [PATCH 03/14] Use AdiosAttributes struct --- include/openPMD/IO/ADIOS/ADIOS2File.hpp | 10 +-- .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 35 ++++++++++- src/IO/ADIOS/ADIOS2File.cpp | 61 +++++++------------ src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 - src/IO/ADIOS/ADIOS2PreloadAttributes.cpp | 4 -- 5 files changed, 58 insertions(+), 53 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 6595a9baa2..ac4b36b11a 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -21,6 +21,7 @@ #pragma once #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" @@ -325,16 +326,9 @@ class ADIOS2File */ void drop(); - AttributeMap_t const &availableAttributes(); - std::vector availableAttributesPrefixed(std::string const &prefix); - /* - * See description below. - */ - void invalidateAttributesMap(); - AttributeMap_t const &availableVariables(); std::vector @@ -442,7 +436,7 @@ class ADIOS2File * the map that would be returned by a call to * IO::Available(Attributes|Variables). */ - std::optional m_availableAttributes; + AdiosAttributes m_attributes; std::optional m_availableVariables; std::set m_pathsMarkedAsActive; diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index 7bc3679fa1..5760fcdb83 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -20,7 +20,9 @@ */ #pragma once +#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/config.hpp" +#include #if openPMD_HAVE_ADIOS2 #include @@ -133,6 +135,12 @@ class PreloadAdiosAttributes AttributeWithShape getAttribute(std::string const &name) const; Datatype attributeType(std::string const &name) const; + + std::map const & + availableAttributes() const & + { + return m_offsets; + } }; struct AdiosAttributes @@ -144,7 +152,32 @@ struct AdiosAttributes std::optional> m_attributes; }; - std::variant m_data; + std::variant m_data = StreamAccess_t{}; + + template + auto withAvailableAttributes(size_t step, adios2::IO &IO, Functor &&f) + -> decltype(std::forward(f)( + std::declval &>())) + { + using ret_t = decltype(std::forward(f)( + std::declval &>())); + return std::visit( + auxiliary::overloaded{ + [step, &f](RandomAccess_t &ra) -> ret_t { + auto &attribute_data = ra.at(step); + return std::forward(f)( + attribute_data.availableAttributes()); + }, + [step, &f, &IO](StreamAccess_t &sa) -> ret_t { + if (!sa.m_attributes.has_value() || + sa.m_currentStep != step) + { + sa = StreamAccess_t{step, IO.AvailableAttributes()}; + } + return std::forward(f)(*sa.m_attributes); + }}, + m_data); + } }; } // namespace openPMD::detail diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 71800bdfc6..500362c768 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -759,19 +759,21 @@ void ADIOS2File::configure_IO() auto ADIOS2File::detectGroupTable() -> UseGroupTable { - auto const &attributes = availableAttributes(); - auto lower_bound = - attributes.lower_bound(adios_defaults::str_activeTablePrefix); - if (lower_bound != attributes.end() && - auxiliary::starts_with( - lower_bound->first, adios_defaults::str_activeTablePrefix)) - { - return UseGroupTable::Yes; - } - else - { - return UseGroupTable::No; - } + return m_attributes.withAvailableAttributes( + currentStep(), m_IO, [&](auto const &attributes) { + auto lower_bound = + attributes.lower_bound(adios_defaults::str_activeTablePrefix); + if (lower_bound != attributes.end() && + auxiliary::starts_with( + lower_bound->first, adios_defaults::str_activeTablePrefix)) + { + return UseGroupTable::Yes; + } + else + { + return UseGroupTable::No; + } + }); } adios2::Engine &ADIOS2File::getEngine() @@ -1264,7 +1266,6 @@ AdvanceStatus ADIOS2File::advance(AdvanceMode mode) case adios2::StepStatus::OtherError: throw std::runtime_error("[ADIOS2] Unexpected step status."); } - invalidateAttributesMap(); invalidateVariablesMap(); m_pathsMarkedAsActive.clear(); return res; @@ -1280,13 +1281,11 @@ void ADIOS2File::drop() assert(m_buffer.empty()); } +template static std::vector availableAttributesOrVariablesPrefixed( - std::string const &prefix, - ADIOS2File::AttributeMap_t const &(ADIOS2File::*getBasicMap)(), - ADIOS2File &ba) + std::string const &prefix, AttributesMap const &attributes) { std::string var = auxiliary::ends_with(prefix, '/') ? prefix : prefix + '/'; - ADIOS2File::AttributeMap_t const &attributes = (ba.*getBasicMap)(); std::vector ret; for (auto it = attributes.lower_bound(prefix); it != attributes.end(); ++it) { @@ -1305,33 +1304,17 @@ static std::vector availableAttributesOrVariablesPrefixed( std::vector ADIOS2File::availableAttributesPrefixed(std::string const &prefix) { - return availableAttributesOrVariablesPrefixed( - prefix, &ADIOS2File::availableAttributes, *this); + return m_attributes.withAvailableAttributes( + currentStep(), m_IO, [&](auto const &attributes) { + return availableAttributesOrVariablesPrefixed(prefix, attributes); + }); } std::vector ADIOS2File::availableVariablesPrefixed(std::string const &prefix) { return availableAttributesOrVariablesPrefixed( - prefix, &ADIOS2File::availableVariables, *this); -} - -void ADIOS2File::invalidateAttributesMap() -{ - m_availableAttributes = std::optional(); -} - -ADIOS2File::AttributeMap_t const &ADIOS2File::availableAttributes() -{ - if (m_availableAttributes) - { - return m_availableAttributes.value(); - } - else - { - m_availableAttributes = std::make_optional(m_IO.AvailableAttributes()); - return m_availableAttributes.value(); - } + prefix, ADIOS2File::availableVariables()); } void ADIOS2File::invalidateVariablesMap() diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index c91ff6b272..827cf67054 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -2148,7 +2148,6 @@ namespace detail auto &filedata = impl->getFileData( file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError); - filedata.invalidateAttributesMap(); adios2::IO IO = filedata.m_IO; impl->m_dirty.emplace(std::move(file)); diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp index 3971e2e8aa..5d97c0071a 100644 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -27,13 +27,9 @@ #include "openPMD/Datatype.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" -#include "openPMD/auxiliary/StringManip.hpp" #include #include -#include -#include -#include namespace openPMD::detail { From efc72b0f74e31ead913bc6fb32b2a7634988e127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 12 Mar 2025 15:15:25 +0100 Subject: [PATCH 04/14] Go through AdiosAttributes for attribute reading --- include/openPMD/IO/ADIOS/ADIOS2File.hpp | 5 ++ include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 22 +++++- .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 39 +++++++++++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 70 +++++++++++++------ src/IO/ADIOS/ADIOS2PreloadAttributes.cpp | 35 ++++++++-- 5 files changed, 143 insertions(+), 28 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index ac4b36b11a..8a0f011db6 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -411,6 +411,11 @@ class ADIOS2File void setStepSelection(std::optional); [[nodiscard]] std::optional stepSelection() const; + [[nodiscard]] detail::AdiosAttributes const &attributes() const + { + return m_attributes; + } + private: ADIOS2IOHandlerImpl *m_impl; std::optional m_engine; //! ADIOS engine diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 355d1aa87f..da63b1196a 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -23,6 +23,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" @@ -541,9 +542,26 @@ namespace detail struct AttributeReader { + struct GetAttribute + { + size_t step; + adios2::IO &IO; + detail::AdiosAttributes const &attributes; + template + auto call(std::string const &name) const + -> detail::AttributeWithShapeAndResource + { + return attributes.getAttribute(step, IO, name); + } + }; + template - static Datatype - call(adios2::IO &IO, std::string name, Attribute::resource &resource); + static Datatype call( + size_t step, + adios2::IO &IO, + std::string name, + Attribute::resource &resource, + detail::AdiosAttributes const &); template static Datatype call(Params &&...); diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index 5760fcdb83..4162c5d768 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -143,6 +143,41 @@ class PreloadAdiosAttributes } }; +template +struct AttributeWithShapeAndResource : AttributeWithShape +{ + AttributeWithShapeAndResource(AttributeWithShape parent) + : AttributeWithShape(std::move(parent)) + {} + AttributeWithShapeAndResource( + size_t len_in, + T const *data_in, + std::optional> resource_in) + : AttributeWithShape{len_in, data_in} + , resource{std::move(resource_in)} + {} + explicit AttributeWithShapeAndResource() : AttributeWithShape(0, nullptr) + {} + AttributeWithShapeAndResource(adios2::Attribute attr) + { + if (!attr) + { + this->data = nullptr; + this->len = 0; + return; + } + auto vec = attr.Data(); + this->len = vec.size(); + this->data = vec.data(); + this->resource = std::move(vec); + } + operator bool() const + { + return this->data; + } + std::optional> resource; +}; + struct AdiosAttributes { using RandomAccess_t = std::vector; @@ -178,6 +213,10 @@ struct AdiosAttributes }}, m_data); } + + template + AttributeWithShapeAndResource + getAttribute(size_t step, adios2::IO &IO, std::string const &name) const; }; } // namespace openPMD::detail diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 827cf67054..33e865abdb 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -27,6 +27,7 @@ #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Streaming.hpp" @@ -1236,7 +1237,12 @@ void ADIOS2IOHandlerImpl::readAttribute( } Datatype ret = switchType( - type, ba.m_IO, name, *parameters.resource); + type, + ba.currentStep(), + ba.m_IO, + name, + *parameters.resource, + ba.attributes()); *parameters.dtype = ret; } @@ -1246,9 +1252,12 @@ namespace Functor fun will be called with the value of the retrieved attribute; both functions use different logic for processing the retrieved values. */ - template - Datatype - genericReadAttribute(Functor &&fun, adios2::IO &IO, std::string const &name) + template + Datatype genericReadAttribute( + Functor &&fun, + adios2::IO &IO, + std::string const &name, + GetAttribute const &getAttribute) { /* * If we store an attribute of boolean type, we store an additional @@ -1259,7 +1268,7 @@ namespace if constexpr (std::is_same::value) { - auto attr = IO.InquireAttribute(name); + auto attr = getAttribute.template call(name); if (!attr) { throw std::runtime_error( @@ -1286,11 +1295,11 @@ namespace if (meta.Data().size() == 1 && meta.Data()[0] == 1) { std::forward(fun)( - detail::bool_repr::fromRep(attr.Data()[0])); + detail::bool_repr::fromRep(attr.data[0])); return determineDatatype(); } } - std::forward(fun)(attr.Data()[0]); + std::forward(fun)(attr.data[0]); } else if constexpr (detail::IsUnsupportedComplex_v) { @@ -1300,27 +1309,30 @@ namespace } else if constexpr (auxiliary::IsVector_v) { - auto attr = IO.InquireAttribute(name); + auto attr = + getAttribute.template call(name); if (!attr) { throw std::runtime_error( "[ADIOS2] Internal error: Failed reading attribute '" + name + "'."); } - std::forward(fun)(attr.Data()); + std::forward(fun)(std::vector( + attr.data, attr.data + attr.len)); } else if constexpr (auxiliary::IsArray_v) { - auto attr = IO.InquireAttribute(name); + auto attr = + getAttribute.template call(name); if (!attr) { throw std::runtime_error( "[ADIOS2] Internal error: Failed reading attribute '" + name + "'."); } - auto data = attr.Data(); + auto data = attr.data; T res; - for (size_t i = 0; i < data.size(); i++) + for (size_t i = 0; i < attr.len; i++) { res[i] = data[i]; } @@ -1333,14 +1345,14 @@ namespace } else { - auto attr = IO.InquireAttribute(name); + auto attr = getAttribute.template call(name); if (!attr) { throw std::runtime_error( "[ADIOS2] Internal error: Failed reading attribute '" + name + "'."); } - std::forward(fun)(attr.Data()[0]); + std::forward(fun)(attr.data[0]); } return determineDatatype(); @@ -1348,6 +1360,17 @@ namespace struct ReadAttributeAllsteps { + struct GetAttribute + { + adios2::IO &IO; + template + [[nodiscard]] auto call(std::string const &name) const + -> detail::AttributeWithShapeAndResource + { + return {IO.InquireAttribute(name)}; + } + }; + template static void call( adios2::IO &IO, @@ -1379,7 +1402,8 @@ namespace } }, IO, - name); + name, + GetAttribute{IO}); engine.EndStep(); status = engine.BeginStep(); } @@ -1657,11 +1681,12 @@ void ADIOS2IOHandlerImpl::listPaths( detail::ADIOS2File::StreamStatus::DuringStep) { auto currentStep = fileData.currentStep(); + auto &IO = fileData.m_IO; for (auto const &attrName : attrs) { using table_t = unsigned long long; - auto attr = fileData.m_IO.InquireAttribute( - tablePrefix + attrName); + auto attr = fileData.attributes().getAttribute( + currentStep, IO, tablePrefix + attrName); if (!attr) { std::cerr << "[ADIOS2 backend] Unable to inquire group " @@ -1671,7 +1696,7 @@ void ADIOS2IOHandlerImpl::listPaths( << std::endl; continue; } - if (attr.Data()[0] != currentStep) + if (attr.data[0] != currentStep) { // group wasn't defined in current step continue; @@ -2113,14 +2138,19 @@ namespace detail { template Datatype AttributeReader::call( - adios2::IO &IO, std::string name, Attribute::resource &resource) + size_t step, + adios2::IO &IO, + std::string name, + Attribute::resource &resource, + detail::AdiosAttributes const &attributes) { return genericReadAttribute( [&resource](auto &&value) { resource = static_cast(value); }, IO, - name); + name, + GetAttribute{step, IO, attributes}); } template diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp index 5d97c0071a..204e254338 100644 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -260,12 +260,6 @@ PreloadAdiosAttributes::getAttribute(std::string const &name) const return res; } -#define OPENPMD_INSTANTIATE_GETATTRIBUTE(type) \ - template AttributeWithShape PreloadAdiosAttributes::getAttribute( \ - std::string const &name) const; -ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(OPENPMD_INSTANTIATE_GETATTRIBUTE) -#undef OPENPMD_INSTANTIATE_GETATTRIBUTE - Datatype PreloadAdiosAttributes::attributeType(std::string const &name) const { auto it = m_offsets.find(name); @@ -275,6 +269,35 @@ Datatype PreloadAdiosAttributes::attributeType(std::string const &name) const } return it->second.dt; } + +template +auto AdiosAttributes::getAttribute( + size_t step, adios2::IO &IO, std::string const &name) const + -> AttributeWithShapeAndResource +{ + return std::visit( + auxiliary::overloaded{ + [step, &name]( + RandomAccess_t const &ra) -> AttributeWithShapeAndResource { + auto &attribute_data = ra.at(step); + return attribute_data.getAttribute(name); + }, + [&name, + &IO](StreamAccess_t const &) -> AttributeWithShapeAndResource { + auto attr = IO.InquireAttribute(name); + return {std::move(attr)}; + }}, + m_data); +} + +#define OPENPMD_INSTANTIATE_GETATTRIBUTE(type) \ + template AttributeWithShape PreloadAdiosAttributes::getAttribute( \ + std::string const &name) const; \ + template auto AdiosAttributes::getAttribute( \ + size_t step, adios2::IO &IO, std::string const &name) const \ + -> AttributeWithShapeAndResource; +ADIOS2_FOREACH_TYPE_1ARG(OPENPMD_INSTANTIATE_GETATTRIBUTE) +#undef OPENPMD_INSTANTIATE_GETATTRIBUTE } // namespace openPMD::detail #endif // openPMD_HAVE_ADIOS2 From 4e69a6e43f68f27cd3e4a2314e9e1654c8b1eb47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 12 Mar 2025 15:57:31 +0100 Subject: [PATCH 05/14] WIP: Tie things together --- include/openPMD/IO/ADIOS/ADIOS2File.hpp | 4 + .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 6 +- src/IO/ADIOS/ADIOS2IOHandler.cpp | 104 +++++++++++++++++- 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 8a0f011db6..206ca411e7 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -415,6 +415,10 @@ class ADIOS2File { return m_attributes; } + [[nodiscard]] detail::AdiosAttributes &attributes() + { + return m_attributes; + } private: ADIOS2IOHandlerImpl *m_impl; diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index 4162c5d768..8fc367453f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -22,18 +22,14 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/config.hpp" +#include #include #if openPMD_HAVE_ADIOS2 #include -#include #include -#include -#include -#include #include "openPMD/Datatype.hpp" -#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" namespace openPMD::detail { diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 33e865abdb..22da2f0d2b 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1291,7 +1291,7 @@ namespace if (type == determineDatatype()) { - auto meta = IO.InquireAttribute(metaAttr); + auto meta = IO.template InquireAttribute(metaAttr); if (meta.Data().size() == 1 && meta.Data()[0] == 1) { std::forward(fun)( @@ -1427,6 +1427,59 @@ namespace static constexpr char const *errorMsg = "ReadAttributeAllsteps"; }; + struct ReadAttributeAllstepsFullPreparsing + { + struct GetAttribute + { + detail::PreloadAdiosAttributes const &p; + template + [[nodiscard]] auto call(std::string const &name) const + -> detail::AttributeWithShapeAndResource + { + return p.getAttribute(name); + } + }; + + template + static void call( + std::vector const &preload, + adios2::IO &IO, + std::string const &name, + Parameter::result_type + &put_result_here) + { + std::vector res; + res.reserve(preload.size()); + for (auto const &p : preload) + { + genericReadAttribute( + [&res](auto &&val) { + using type = std::remove_reference_t; + if constexpr (std::is_same_v) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "ADIOS2", + "[ReadAttributeAllsteps] No support for " + "Boolean attributes."); + } + else + { + res.emplace_back(static_cast(val)); + } + }, + IO, + name, + GetAttribute{p}); + } + put_result_here = std::move(res); + } + + static constexpr char const *errorMsg = + "ReadAttributeAllstepsFullPreparsing"; + }; + #if openPMD_HAVE_MPI struct DistributeToAllRanks { @@ -1561,6 +1614,8 @@ Use Access::READ_LINEAR to retrieve those values if needed. } } // namespace +#define OPENPMD_PREPARSE_EVERYTHING 1 + void ADIOS2IOHandlerImpl::readAttributeAllsteps( Writable *writable, Parameter ¶m) { @@ -1568,6 +1623,52 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto pos = setAndGetFilePosition(writable); auto name = nameOfAttribute(writable, param.name); +#if OPENPMD_PREPARSE_EVERYTHING + detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); +#if openPMD_HAVE_MPI + auto adios = [&]() { + if (m_communicator.has_value()) + { + return adios2::ADIOS(*m_communicator); + } + else + { + return adios2::ADIOS{}; + } + }(); +#else + adios2::ADIOS adios; +#endif + auto IO = adios.DeclareIO("PreparseSnapshots"); + // @todo check engine type + IO.SetEngine(realEngineType()); + IO.SetParameter("StreamReader", "ON"); // this be for BP4 + auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); + std::vector preload; + preload.reserve(engine.Steps()); + adios2::StepStatus status; + while ((status = engine.BeginStep()) == adios2::StepStatus::OK) + { + auto &new_entry = preload.emplace_back(); + new_entry.preloadAttributes(IO); + engine.EndStep(); + } + if (status != adios2::StepStatus::EndOfStream) + { + throw std::runtime_error( + "[ADIOS2IOHandlerImpl::readAttributeAllsteps] Unexpected step " + "status while beginning a step."); + } + engine.Close(); + auto &attributes = ba.attributes(); + switchType( + preload.at(0).attributeType(param.name), + preload, + IO, + param.name, + *param.resource); + attributes.m_data = std::move(preload); +#else auto read_from_file_in_serial = [&]() { adios2::ADIOS adios; auto IO = adios.DeclareIO("PreparseSnapshots"); @@ -1603,6 +1704,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( #else read_from_file_in_serial(); #endif +#endif } void ADIOS2IOHandlerImpl::listPaths( From 5eaf97bf5cc1d7512daf55c04cb776dc563d2af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 12 Mar 2025 17:03:52 +0100 Subject: [PATCH 06/14] Fixes --- src/IO/ADIOS/ADIOS2File.cpp | 6 +++++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 7 ++----- test/SerialIOTest.cpp | 7 ++----- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 500362c768..27edec51a8 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -344,7 +344,11 @@ namespace size_t ADIOS2File::currentStep() { - if (nonpersistentEngine(m_impl->m_engineType)) + if (auto step_selection = stepSelection(); step_selection.has_value()) + { + return *step_selection; + } + else if (nonpersistentEngine(m_impl->m_engineType)) { return m_currentStep; } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 22da2f0d2b..ac38a551d2 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1625,6 +1625,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( #if OPENPMD_PREPARSE_EVERYTHING detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); + auto type = detail::attributeInfo(ba.m_IO, name, /* verbose = */ true); #if openPMD_HAVE_MPI auto adios = [&]() { if (m_communicator.has_value()) @@ -1662,11 +1663,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( engine.Close(); auto &attributes = ba.attributes(); switchType( - preload.at(0).attributeType(param.name), - preload, - IO, - param.name, - *param.resource); + type, preload, IO, name, *param.resource); attributes.m_data = std::move(preload); #else auto read_from_file_in_serial = [&]() { diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 40aba2c8c1..c011cbe91a 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5950,11 +5950,8 @@ void variableBasedSeries(std::string const &file) { REQUIRE( iteration.getAttribute("changing_value").get() == - (supportsModifiableAttributes - ? (access == Access::READ_LINEAR - ? iteration.iterationIndex - : 9) - : 0)); + (supportsModifiableAttributes ? iteration.iterationIndex + : 0)); } auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); From 20a1a512b46572650a6809d783066032fa132881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Mar 2025 13:22:27 +0100 Subject: [PATCH 07/14] Fix path reading --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 9 ++++++--- test/SerialIOTest.cpp | 9 ++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index ac38a551d2..3287f104e0 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1591,6 +1591,9 @@ namespace }; #endif +#define OPENPMD_PREPARSE_EVERYTHING 1 + +#if !OPENPMD_PREPARSE_EVERYTHING void warn_ignored_modifiable_attributes(adios2::IO &IO) { auto modifiable_flag = IO.InquireAttribute( @@ -1612,10 +1615,9 @@ Use Access::READ_LINEAR to retrieve those values if needed. print_warning("File uses modifiable attributes."); } } +#endif } // namespace -#define OPENPMD_PREPARSE_EVERYTHING 1 - void ADIOS2IOHandlerImpl::readAttributeAllsteps( Writable *writable, Parameter ¶m) { @@ -1777,7 +1779,8 @@ void ADIOS2IOHandlerImpl::listPaths( std::vector attrs = fileData.availableAttributesPrefixed(tablePrefix); if (fileData.streamStatus == - detail::ADIOS2File::StreamStatus::DuringStep) + detail::ADIOS2File::StreamStatus::DuringStep || + fileData.stepSelection().has_value()) { auto currentStep = fileData.currentStep(); auto &IO = fileData.m_IO; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index c011cbe91a..14e46f90c8 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5971,13 +5971,12 @@ void variableBasedSeries(std::string const &file) last_iteration_index = iteration.iterationIndex; - if (access == Access::READ_RANDOM_ACCESS) - { - continue; - } - // this loop ensures that only the recordcomponent ["E"]["i"] is // present where i == iteration.iterationIndex + // Note that this works for ReadRandomAccess as well since constant + // components contain no datasets. The ADIOS2 backend is however not + // (yet) able to deal with array datasets that are present only in a + // subselection of steps. for (uint64_t otherIteration = 0; otherIteration < 10; ++otherIteration) { From eb95a872e41e0238a5a912deaa656aa9228bd4d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Mar 2025 14:37:53 +0100 Subject: [PATCH 08/14] Use parameters from actual IO --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 3287f104e0..ec8916946a 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1624,9 +1624,9 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); auto pos = setAndGetFilePosition(writable); auto name = nameOfAttribute(writable, param.name); + detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); #if OPENPMD_PREPARSE_EVERYTHING - detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); auto type = detail::attributeInfo(ba.m_IO, name, /* verbose = */ true); #if openPMD_HAVE_MPI auto adios = [&]() { @@ -1643,10 +1643,11 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( adios2::ADIOS adios; #endif auto IO = adios.DeclareIO("PreparseSnapshots"); - // @todo check engine type - IO.SetEngine(realEngineType()); + IO.SetEngine(ba.m_IO.EngineType()); + IO.SetParameters(ba.m_IO.Parameters()); IO.SetParameter("StreamReader", "ON"); // this be for BP4 auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); + std::vector preload; preload.reserve(engine.Steps()); adios2::StepStatus status; @@ -1671,8 +1672,8 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto read_from_file_in_serial = [&]() { adios2::ADIOS adios; auto IO = adios.DeclareIO("PreparseSnapshots"); - // @todo check engine type - IO.SetEngine(realEngineType()); + IO.SetEngine(ba.m_IO.EngineType()); + IO.SetParameters(ba.m_IO.Parameters()); IO.SetParameter("StreamReader", "ON"); // this be for BP4 auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); auto status = engine.BeginStep(); From 3741a7ea5ccab64989a4a4be9724d155f3a2c66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Mar 2025 15:15:42 +0100 Subject: [PATCH 09/14] CI fixes --- .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 1 - src/IO/ADIOS/ADIOS2IOHandler.cpp | 81 ++++++++++--------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index 8fc367453f..5cfe71896f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -113,7 +113,6 @@ class PreloadAdiosAttributes * be loaded along with the next adios2::Engine flush. * * @param IO - * @param engine */ void preloadAttributes(adios2::IO &IO); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index ec8916946a..af1f39c566 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1246,6 +1246,7 @@ void ADIOS2IOHandlerImpl::readAttribute( *parameters.dtype = ret; } +#define openPMD_PREPARSE_EVERYTHING 1 namespace { /* Used by both readAttribute() and readAttributeAllsteps() tasks. @@ -1358,31 +1359,32 @@ namespace return determineDatatype(); } - struct ReadAttributeAllsteps +#if openPMD_PREPARSE_EVERYTHING + + struct ReadAttributeAllstepsFullPreparsing { struct GetAttribute { - adios2::IO &IO; + detail::PreloadAdiosAttributes const &p; template [[nodiscard]] auto call(std::string const &name) const -> detail::AttributeWithShapeAndResource { - return {IO.InquireAttribute(name)}; + return p.getAttribute(name); } }; template static void call( + std::vector const &preload, adios2::IO &IO, - adios2::Engine &engine, std::string const &name, - adios2::StepStatus status, Parameter::result_type &put_result_here) { std::vector res; - res.reserve(engine.Steps()); - while (status == adios2::StepStatus::OK) + res.reserve(preload.size()); + for (auto const &p : preload) { genericReadAttribute( [&res](auto &&val) { @@ -1403,54 +1405,41 @@ namespace }, IO, name, - GetAttribute{IO}); - engine.EndStep(); - status = engine.BeginStep(); - } - switch (status) - { - case adios2::StepStatus::OK: - throw error::Internal("Control flow error."); - case adios2::StepStatus::NotReady: - case adios2::StepStatus::OtherError: - throw error::ReadError( - error::AffectedObject::File, - error::Reason::CannotRead, - "ADIOS2", - "Unexpected step status while preparsing snapshots."); - case adios2::StepStatus::EndOfStream: - break; + GetAttribute{p}); } put_result_here = std::move(res); } - static constexpr char const *errorMsg = "ReadAttributeAllsteps"; + static constexpr char const *errorMsg = + "ReadAttributeAllstepsFullPreparsing"; }; - struct ReadAttributeAllstepsFullPreparsing +#else + struct ReadAttributeAllsteps { struct GetAttribute { - detail::PreloadAdiosAttributes const &p; + adios2::IO &IO; template [[nodiscard]] auto call(std::string const &name) const -> detail::AttributeWithShapeAndResource { - return p.getAttribute(name); + return {IO.InquireAttribute(name)}; } }; template static void call( - std::vector const &preload, adios2::IO &IO, + adios2::Engine &engine, std::string const &name, + adios2::StepStatus status, Parameter::result_type &put_result_here) { std::vector res; - res.reserve(preload.size()); - for (auto const &p : preload) + res.reserve(engine.Steps()); + while (status == adios2::StepStatus::OK) { genericReadAttribute( [&res](auto &&val) { @@ -1471,13 +1460,28 @@ namespace }, IO, name, - GetAttribute{p}); + GetAttribute{IO}); + engine.EndStep(); + status = engine.BeginStep(); + } + switch (status) + { + case adios2::StepStatus::OK: + throw error::Internal("Control flow error."); + case adios2::StepStatus::NotReady: + case adios2::StepStatus::OtherError: + throw error::ReadError( + error::AffectedObject::File, + error::Reason::CannotRead, + "ADIOS2", + "Unexpected step status while preparsing snapshots."); + case adios2::StepStatus::EndOfStream: + break; } put_result_here = std::move(res); } - static constexpr char const *errorMsg = - "ReadAttributeAllstepsFullPreparsing"; + static constexpr char const *errorMsg = "ReadAttributeAllsteps"; }; #if openPMD_HAVE_MPI @@ -1589,11 +1593,7 @@ namespace } static constexpr char const *errorMsg = "DistributeToAllRanks"; }; -#endif - -#define OPENPMD_PREPARSE_EVERYTHING 1 -#if !OPENPMD_PREPARSE_EVERYTHING void warn_ignored_modifiable_attributes(adios2::IO &IO) { auto modifiable_flag = IO.InquireAttribute( @@ -1615,7 +1615,8 @@ Use Access::READ_LINEAR to retrieve those values if needed. print_warning("File uses modifiable attributes."); } } -#endif +#endif // openPMD_HAVE_MPI +#endif // openPMD_PREPARSE_EVERYTHING } // namespace void ADIOS2IOHandlerImpl::readAttributeAllsteps( @@ -1626,7 +1627,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto name = nameOfAttribute(writable, param.name); detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); -#if OPENPMD_PREPARSE_EVERYTHING +#if openPMD_PREPARSE_EVERYTHING auto type = detail::attributeInfo(ba.m_IO, name, /* verbose = */ true); #if openPMD_HAVE_MPI auto adios = [&]() { From 1de01de57ab0e3d1380ec20689c93644e6c408a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Mar 2025 15:15:53 +0100 Subject: [PATCH 10/14] Fix old implementation in case we want to keep it --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index af1f39c566..ff706d2fb9 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -52,6 +52,7 @@ #include #include #include +#include namespace openPMD { @@ -1782,7 +1783,10 @@ void ADIOS2IOHandlerImpl::listPaths( fileData.availableAttributesPrefixed(tablePrefix); if (fileData.streamStatus == detail::ADIOS2File::StreamStatus::DuringStep || - fileData.stepSelection().has_value()) + (fileData.stepSelection().has_value() && + std::holds_alternative< + detail::AdiosAttributes::RandomAccess_t>( + fileData.attributes().m_data))) { auto currentStep = fileData.currentStep(); auto &IO = fileData.m_IO; From 5fc8384b845ecc846eeab370bfb93a700ea03069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Mar 2025 17:34:20 +0100 Subject: [PATCH 11/14] Update documentation --- docs/source/usage/workflow.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index a69a498cb4..8ae2a18dba 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -77,13 +77,15 @@ The openPMD-api distinguishes between a number of different access modes: When using such a backend, the access mode will be coerced automatically to *linear read mode*. Use of Series::readIterations() is mandatory for access. 4. *Random-access read mode* for a variable-based Series is currently experimental. - There is currently only very restricted support for metadata definitions that change across steps: + Support for metadata definitions that change across steps is (currently) restricted: - 1. Modifiable attributes (except ``/data/snapshot``) can currently not be read. Attributes such as ``/data/time`` that naturally change their value across Iterations will hence not be really well-usable; the last Iteration's value will currently leak into all other Iterations. - 2. There is no support for datasets that do not exist in all Iterations. The internal Iteration layouts should be homogeneous. + 1. There is no support for datasets that do not exist in all Iterations. The internal Iteration layouts should be homogeneous. If you need this feature, please contact the openPMD-api developers; implementing this is currently not a priority. Datasets that do not exist in all steps will be skipped at read time (with an error). - 3. Datasets with changing extents are supported. + + This restriction affects only datasets that contain array data. + Datasets defined exlusively in terms of attributes (especially: constant components) may be defined and read in a subselection of Iterations. + 2. Datasets with changing extents are supported under the condition that they do not change their dimensionality. * **Read/Write mode**: Creates a new Series if not existing, otherwise opens an existing Series for reading and writing. New datasets and iterations will be inserted as needed. From f636145bdce5eb8e0375af956b7c6dfa107a3374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 13 Mar 2025 17:36:05 +0100 Subject: [PATCH 12/14] CI fixes: noexcept constructor --- include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp | 4 ++-- src/IO/ADIOS/ADIOS2PreloadAttributes.cpp | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index 5cfe71896f..bdfe9d3e3d 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -78,8 +78,8 @@ class PreloadAdiosAttributes AttributeLocation(AttributeLocation const &other) = delete; AttributeLocation &operator=(AttributeLocation const &other) = delete; - AttributeLocation(AttributeLocation &&other); - AttributeLocation &operator=(AttributeLocation &&other); + AttributeLocation(AttributeLocation &&other) noexcept; + AttributeLocation &operator=(AttributeLocation &&other) noexcept; ~AttributeLocation(); }; diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp index 204e254338..efe8df5690 100644 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -144,13 +144,14 @@ AttributeLocation::AttributeLocation( : len(len_in), offset(offset_in), dt(dt_in) {} -AttributeLocation::AttributeLocation(AttributeLocation &&other) +AttributeLocation::AttributeLocation(AttributeLocation &&other) noexcept : len{other.len}, offset{other.offset}, dt{other.dt}, destroy{other.destroy} { other.destroy = nullptr; } -AttributeLocation &AttributeLocation::operator=(AttributeLocation &&other) +AttributeLocation & +AttributeLocation::operator=(AttributeLocation &&other) noexcept { this->len = other.len; this->offset = other.offset; From b69abc5c251621647e6b9aaaf1a40300e215860f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 24 Mar 2025 11:29:00 +0100 Subject: [PATCH 13/14] Remove old implementation --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 254 +------------------------------ 1 file changed, 3 insertions(+), 251 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index ff706d2fb9..860e159b16 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1247,7 +1247,6 @@ void ADIOS2IOHandlerImpl::readAttribute( *parameters.dtype = ret; } -#define openPMD_PREPARSE_EVERYTHING 1 namespace { /* Used by both readAttribute() and readAttributeAllsteps() tasks. @@ -1360,9 +1359,7 @@ namespace return determineDatatype(); } -#if openPMD_PREPARSE_EVERYTHING - - struct ReadAttributeAllstepsFullPreparsing + struct ReadAttributeAllsteps { struct GetAttribute { @@ -1383,7 +1380,7 @@ namespace Parameter::result_type &put_result_here) { - std::vector res; + auto &res = put_result_here.emplace>(); res.reserve(preload.size()); for (auto const &p : preload) { @@ -1408,216 +1405,10 @@ namespace name, GetAttribute{p}); } - put_result_here = std::move(res); - } - - static constexpr char const *errorMsg = - "ReadAttributeAllstepsFullPreparsing"; - }; - -#else - struct ReadAttributeAllsteps - { - struct GetAttribute - { - adios2::IO &IO; - template - [[nodiscard]] auto call(std::string const &name) const - -> detail::AttributeWithShapeAndResource - { - return {IO.InquireAttribute(name)}; - } - }; - - template - static void call( - adios2::IO &IO, - adios2::Engine &engine, - std::string const &name, - adios2::StepStatus status, - Parameter::result_type - &put_result_here) - { - std::vector res; - res.reserve(engine.Steps()); - while (status == adios2::StepStatus::OK) - { - genericReadAttribute( - [&res](auto &&val) { - using type = std::remove_reference_t; - if constexpr (std::is_same_v) - { - throw error::ReadError( - error::AffectedObject::Attribute, - error::Reason::UnexpectedContent, - "ADIOS2", - "[ReadAttributeAllsteps] No support for " - "Boolean attributes."); - } - else - { - res.emplace_back(static_cast(val)); - } - }, - IO, - name, - GetAttribute{IO}); - engine.EndStep(); - status = engine.BeginStep(); - } - switch (status) - { - case adios2::StepStatus::OK: - throw error::Internal("Control flow error."); - case adios2::StepStatus::NotReady: - case adios2::StepStatus::OtherError: - throw error::ReadError( - error::AffectedObject::File, - error::Reason::CannotRead, - "ADIOS2", - "Unexpected step status while preparsing snapshots."); - case adios2::StepStatus::EndOfStream: - break; - } - put_result_here = std::move(res); } static constexpr char const *errorMsg = "ReadAttributeAllsteps"; }; - -#if openPMD_HAVE_MPI - struct DistributeToAllRanks - { - template - static void call( - Parameter::result_type - &put_result_here_in, - MPI_Comm comm, - int rank) - { - if (rank != 0) - { - put_result_here_in = std::vector{}; - } - std::vector &put_result_here = - std::get>(put_result_here_in); - size_t num_items = put_result_here.size(); - MPI_CHECK(MPI_Bcast( - &num_items, 1, auxiliary::openPMD_MPI_type(), 0, comm)); - if constexpr ( - std::is_same_v || - std::is_same_v> || - std::is_same_v || - std::is_same_v> || - auxiliary::IsArray_v || isComplexFloatingPoint()) - { - throw error::OperationUnsupportedInBackend( - "ADIOS2", - "[readAttributeAllsteps] No support for attributes of type " - "std::string, bool, std::complex or std::array in " - "parallel."); - } - else if constexpr ( - // auxiliary::IsArray_v || - auxiliary::IsVector_v) - { - std::vector sizes; - sizes.reserve(num_items); - if (rank == 0) - { - std::transform( - put_result_here.begin(), - put_result_here.end(), - std::back_inserter(sizes), - [](T const &arr) { return arr.size(); }); - } - sizes.resize(num_items); - MPI_CHECK(MPI_Bcast( - sizes.data(), - num_items, - auxiliary::openPMD_MPI_type(), - 0, - comm)); - size_t total_flat_size = - std::accumulate(sizes.begin(), sizes.end(), size_t(0)); - using flat_type = typename T::value_type; - std::vector flat_vector; - flat_vector.reserve(total_flat_size); - if (rank == 0) - { - for (auto const &arr : put_result_here) - { - for (auto val : arr) - { - flat_vector.push_back(val); - } - } - } - flat_vector.resize(total_flat_size); - MPI_CHECK(MPI_Bcast( - flat_vector.data(), - total_flat_size, - auxiliary::openPMD_MPI_type(), - 0, - comm)); - if (rank != 0) - { - size_t offset = 0; - put_result_here.reserve(num_items); - for (size_t current_extent : sizes) - { - put_result_here.emplace_back( - flat_vector.begin() + offset, - flat_vector.begin() + offset + current_extent); - offset += current_extent; - } - } - } - else - { - std::vector receive; - if (rank != 0) - { - receive.resize(num_items); - } - MPI_CHECK(MPI_Bcast( - rank == 0 ? put_result_here.data() : receive.data(), - num_items, - auxiliary::openPMD_MPI_type(), - 0, - comm)); - if (rank != 0) - { - put_result_here = std::move(receive); - } - } - } - static constexpr char const *errorMsg = "DistributeToAllRanks"; - }; - - void warn_ignored_modifiable_attributes(adios2::IO &IO) - { - auto modifiable_flag = IO.InquireAttribute( - adios_defaults::str_useModifiableAttributes); - auto print_warning = [](std::string const ¬e) { - std::cerr << "Warning: " << note << R"( -Random-access for variable-encoding in ADIOS2 is currently -experimental. Support for modifiable attributes is currently not implemented -yet, meaning that attributes such as /data/time will show useless values. -Use Access::READ_LINEAR to retrieve those values if needed. -)"; - }; - if (!modifiable_flag) - { - print_warning("File might be using modifiable attributes."); - } - else if (modifiable_flag.Data().at(0) != 0) - { - print_warning("File uses modifiable attributes."); - } - } -#endif // openPMD_HAVE_MPI -#endif // openPMD_PREPARSE_EVERYTHING } // namespace void ADIOS2IOHandlerImpl::readAttributeAllsteps( @@ -1628,7 +1419,6 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto name = nameOfAttribute(writable, param.name); detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); -#if openPMD_PREPARSE_EVERYTHING auto type = detail::attributeInfo(ba.m_IO, name, /* verbose = */ true); #if openPMD_HAVE_MPI auto adios = [&]() { @@ -1667,46 +1457,8 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( } engine.Close(); auto &attributes = ba.attributes(); - switchType( - type, preload, IO, name, *param.resource); + switchType(type, preload, IO, name, *param.resource); attributes.m_data = std::move(preload); -#else - auto read_from_file_in_serial = [&]() { - adios2::ADIOS adios; - auto IO = adios.DeclareIO("PreparseSnapshots"); - IO.SetEngine(ba.m_IO.EngineType()); - IO.SetParameters(ba.m_IO.Parameters()); - IO.SetParameter("StreamReader", "ON"); // this be for BP4 - auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); - auto status = engine.BeginStep(); - warn_ignored_modifiable_attributes(IO); - auto type = detail::attributeInfo(IO, name, /* verbose = */ true); - switchType( - type, IO, engine, name, status, *param.resource); - engine.Close(); - return type; - }; -#if openPMD_HAVE_MPI - if (!m_communicator.has_value()) - { - read_from_file_in_serial(); - return; - } - int rank, size; - MPI_Comm_rank(*m_communicator, &rank); - MPI_Comm_size(*m_communicator, &size); - Datatype type; - if (rank == 0) - { - type = read_from_file_in_serial(); - } - MPI_CHECK(MPI_Bcast(&type, 1, MPI_INT, 0, *m_communicator)); - switchType( - type, *param.resource, *m_communicator, rank); -#else - read_from_file_in_serial(); -#endif -#endif } void ADIOS2IOHandlerImpl::listPaths( From 361363a7737b1cb65fe7112428b1bd44565128ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 24 Mar 2025 13:01:38 +0100 Subject: [PATCH 14/14] Cleanup retrigger CI --- .../IO/ADIOS/ADIOS2PreloadAttributes.hpp | 99 +++++++++---------- src/IO/ADIOS/ADIOS2IOHandler.cpp | 12 ++- src/IO/ADIOS/ADIOS2PreloadAttributes.cpp | 57 ++++++++++- 3 files changed, 112 insertions(+), 56 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp index bdfe9d3e3d..c6c2271b9f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -1,4 +1,4 @@ -/* Copyright 2020-2021 Franz Poeschel +/* Copyright 2020-2025 Franz Poeschel * * This file is part of openPMD-api. * @@ -20,16 +20,16 @@ */ #pragma once -#include "openPMD/auxiliary/Variant.hpp" #include "openPMD/config.hpp" -#include -#include #if openPMD_HAVE_ADIOS2 #include #include +#include +#include #include "openPMD/Datatype.hpp" +#include "openPMD/auxiliary/Variant.hpp" namespace openPMD::detail { @@ -46,24 +46,21 @@ struct AttributeWithShape }; /** - * Class that is responsible for scheduling and buffering openPMD attribute - * loads from ADIOS2, if using ADIOS variables to store openPMD attributes. + * Class that is responsible for buffering loaded openPMD attributes from + * ADIOS2. * - * Reasoning: ADIOS variables can be of any shape and size, and ADIOS cannot - * know which variables to buffer. While it will preload and buffer scalar - * variables, openPMD also stores vector-type attributes which are not - * preloaded. Since in Streaming setups, every variable load requires full - * communication back to the writer, this can quickly become very expensive. - * Hence, do this manually. + * This is used for reading variable-encoded files in random-access mode. + * Random-access mode in ADIOS2 has the restriction that modifiable attributes + * can only be recovered in normal non-random-access read mode, so we open the + * file first in that mode, store all attribute metadata in this class and only + * then continue in random-access mode. * */ class PreloadAdiosAttributes { public: /** - * Internally used struct to store meta information on a buffered - * attribute. Public for simplicity (helper struct in the - * implementation uses it). + * Meta information on a buffered attribute. */ struct AttributeLocation { @@ -106,11 +103,7 @@ class PreloadAdiosAttributes PreloadAdiosAttributes &operator=(PreloadAdiosAttributes &&other) = default; /** - * @brief Schedule attributes for preloading. - * - * This will invalidate all previously buffered attributes. - * This will *not* flush the scheduled loads. This way, attributes can - * be loaded along with the next adios2::Engine flush. + * @brief Load attributes from the current step into the buffer. * * @param IO */ @@ -131,45 +124,27 @@ class PreloadAdiosAttributes Datatype attributeType(std::string const &name) const; - std::map const & - availableAttributes() const & - { - return m_offsets; - } + std::map const &availableAttributes() const; }; template struct AttributeWithShapeAndResource : AttributeWithShape { - AttributeWithShapeAndResource(AttributeWithShape parent) - : AttributeWithShape(std::move(parent)) - {} + AttributeWithShapeAndResource(AttributeWithShape parent); AttributeWithShapeAndResource( size_t len_in, T const *data_in, - std::optional> resource_in) - : AttributeWithShape{len_in, data_in} - , resource{std::move(resource_in)} - {} - explicit AttributeWithShapeAndResource() : AttributeWithShape(0, nullptr) - {} - AttributeWithShapeAndResource(adios2::Attribute attr) - { - if (!attr) - { - this->data = nullptr; - this->len = 0; - return; - } - auto vec = attr.Data(); - this->len = vec.size(); - this->data = vec.data(); - this->resource = std::move(vec); - } - operator bool() const - { - return this->data; - } + std::optional> resource_in); + AttributeWithShapeAndResource(adios2::Attribute attr); + operator bool() const; + +private: + /* + * Users should still use the API of AttributeWithShape (parent type), we + * just need somewhere to store the std::vector returned by + * Attribute::Data(). This field will not be used when using preparsing, + * because the data pointer will go right into the preparse buffer. + */ std::optional> resource; }; @@ -178,12 +153,34 @@ struct AdiosAttributes using RandomAccess_t = std::vector; struct StreamAccess_t { + /* + * These are only buffered for performance reasons. + * IO::AvailableAttributes() returns by value, so we should avoid + * calling it too often. Instead, store the returned value along with + * the step, so we know when we need to update again (i.e. when the + * current step changes). + */ size_t m_currentStep = 0; std::optional> m_attributes; }; + /* + * Variant RandomAcces_t has to be initialized explicitly by + * ADIOS2IOHandlerImpl::readAttributeAllsteps(), so we use StreamAccess_t by + * default. + */ std::variant m_data = StreamAccess_t{}; + /* + * Needs to be this somewhat ugly API since the AvailableAttributes map has + * a different type depending if we use preparsing or not. If we don't use + * preparsing, we just use the returned std::map from IO::AvailableAttributes(). Otherwise, we use the + * std::map map from the + * PreloadAdiosAttributes class. The functor f will be called either with + * the one or the other, so needs to be written such that the mapped-to type + * does not matter. + */ template auto withAvailableAttributes(size_t step, adios2::IO &IO, Functor &&f) -> decltype(std::forward(f)( diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 860e159b16..647c31d856 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1252,6 +1252,9 @@ namespace /* Used by both readAttribute() and readAttributeAllsteps() tasks. Functor fun will be called with the value of the retrieved attribute; both functions use different logic for processing the retrieved values. + Functor getAttribute is called for retrieving an attribute (Different + wrappers around IO::InquireAttribute<>()::Data(), together with + buffering). */ template Datatype genericReadAttribute( @@ -1364,6 +1367,7 @@ namespace struct GetAttribute { detail::PreloadAdiosAttributes const &p; + template [[nodiscard]] auto call(std::string const &name) const -> detail::AttributeWithShapeAndResource @@ -1533,9 +1537,15 @@ void ADIOS2IOHandlerImpl::listPaths( auto tablePrefix = adios_defaults::str_activeTablePrefix + myName; std::vector attrs = fileData.availableAttributesPrefixed(tablePrefix); - if (fileData.streamStatus == + if ( + // either a step is currently active... + fileData.streamStatus == detail::ADIOS2File::StreamStatus::DuringStep || + // ...or a step selection is currently active (fileData.stepSelection().has_value() && + // This check is currently redundant, but may be relevant if we + // (re-)introduce a RandomAccessMode lite that does no + // preparsing of attributes. std::holds_alternative< detail::AdiosAttributes::RandomAccess_t>( fileData.attributes().m_data))) diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp index efe8df5690..c3c2fbfbd7 100644 --- a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -1,4 +1,4 @@ -/* Copyright 2020-2021 Franz Poeschel +/* Copyright 2020-2025 Franz Poeschel * * This file is part of openPMD-api. * @@ -20,7 +20,6 @@ */ #include "openPMD/config.hpp" -#include #if openPMD_HAVE_ADIOS2 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" @@ -28,8 +27,10 @@ #include "openPMD/Datatype.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" +#include #include #include +#include namespace openPMD::detail { @@ -176,7 +177,7 @@ PreloadAdiosAttributes::AttributeLocation::~AttributeLocation() void PreloadAdiosAttributes::preloadAttributes(adios2::IO &IO) { m_offsets.clear(); - std::map > attributesByType; + std::map> attributesByType; auto addAttribute = [&attributesByType](Datatype dt, std::string name) { constexpr size_t reserve = 10; auto it = attributesByType.find(dt); @@ -271,6 +272,12 @@ Datatype PreloadAdiosAttributes::attributeType(std::string const &name) const return it->second.dt; } +std::map const & +PreloadAdiosAttributes::availableAttributes() const +{ + return m_offsets; +} + template auto AdiosAttributes::getAttribute( size_t step, adios2::IO &IO, std::string const &name) const @@ -291,12 +298,54 @@ auto AdiosAttributes::getAttribute( m_data); } +template +AttributeWithShapeAndResource::AttributeWithShapeAndResource( + AttributeWithShape parent) + : AttributeWithShape(std::move(parent)) +{} +template +AttributeWithShapeAndResource::AttributeWithShapeAndResource( + size_t len_in, T const *data_in, std::optional> resource_in) + : AttributeWithShape{len_in, data_in}, resource{std::move(resource_in)} +{} +template +AttributeWithShapeAndResource::AttributeWithShapeAndResource( + adios2::Attribute attr) +{ + if (!attr) + { + this->data = nullptr; + this->len = 0; + return; + } + auto vec = attr.Data(); + this->len = vec.size(); + this->data = vec.data(); + this->resource = std::move(vec); +} +template +AttributeWithShapeAndResource::operator bool() const +{ + return this->data; +} + #define OPENPMD_INSTANTIATE_GETATTRIBUTE(type) \ template AttributeWithShape PreloadAdiosAttributes::getAttribute( \ std::string const &name) const; \ template auto AdiosAttributes::getAttribute( \ size_t step, adios2::IO &IO, std::string const &name) const \ - -> AttributeWithShapeAndResource; + -> AttributeWithShapeAndResource; \ + template AttributeWithShapeAndResource< \ + type>::AttributeWithShapeAndResource(AttributeWithShape parent); \ + template AttributeWithShapeAndResource:: \ + AttributeWithShapeAndResource( \ + size_t len_in, \ + type const \ + *data_in, /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ + std::optional> resource_in); \ + template AttributeWithShapeAndResource< \ + type>::AttributeWithShapeAndResource(adios2::Attribute attr); \ + template AttributeWithShapeAndResource::operator bool() const; ADIOS2_FOREACH_TYPE_1ARG(OPENPMD_INSTANTIATE_GETATTRIBUTE) #undef OPENPMD_INSTANTIATE_GETATTRIBUTE } // namespace openPMD::detail