diff --git a/CMakeLists.txt b/CMakeLists.txt index 3af9872195..367bc15e92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -437,6 +437,7 @@ set(IO_SOURCE src/IO/JSON/JSONIOHandlerImpl.cpp src/IO/JSON/JSONFilePosition.cpp src/IO/ADIOS/ADIOS2IOHandler.cpp + src/IO/ADIOS/ADIOS2PreloadAttributes.cpp src/IO/ADIOS/ADIOS2File.cpp src/IO/ADIOS/ADIOS2Auxiliary.cpp src/IO/InvalidatableFile.cpp) diff --git a/docs/source/usage/workflow.rst b/docs/source/usage/workflow.rst index a69a498cb4..8ae2a18dba 100644 --- a/docs/source/usage/workflow.rst +++ b/docs/source/usage/workflow.rst @@ -77,13 +77,15 @@ The openPMD-api distinguishes between a number of different access modes: When using such a backend, the access mode will be coerced automatically to *linear read mode*. Use of Series::readIterations() is mandatory for access. 4. *Random-access read mode* for a variable-based Series is currently experimental. - There is currently only very restricted support for metadata definitions that change across steps: + Support for metadata definitions that change across steps is (currently) restricted: - 1. Modifiable attributes (except ``/data/snapshot``) can currently not be read. Attributes such as ``/data/time`` that naturally change their value across Iterations will hence not be really well-usable; the last Iteration's value will currently leak into all other Iterations. - 2. There is no support for datasets that do not exist in all Iterations. The internal Iteration layouts should be homogeneous. + 1. There is no support for datasets that do not exist in all Iterations. The internal Iteration layouts should be homogeneous. If you need this feature, please contact the openPMD-api developers; implementing this is currently not a priority. Datasets that do not exist in all steps will be skipped at read time (with an error). - 3. Datasets with changing extents are supported. + + This restriction affects only datasets that contain array data. + Datasets defined exlusively in terms of attributes (especially: constant components) may be defined and read in a subselection of Iterations. + 2. Datasets with changing extents are supported under the condition that they do not change their dimensionality. * **Read/Write mode**: Creates a new Series if not existing, otherwise opens an existing Series for reading and writing. New datasets and iterations will be inserted as needed. diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 6595a9baa2..206ca411e7 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -21,6 +21,7 @@ #pragma once #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" @@ -325,16 +326,9 @@ class ADIOS2File */ void drop(); - AttributeMap_t const &availableAttributes(); - std::vector availableAttributesPrefixed(std::string const &prefix); - /* - * See description below. - */ - void invalidateAttributesMap(); - AttributeMap_t const &availableVariables(); std::vector @@ -417,6 +411,15 @@ class ADIOS2File void setStepSelection(std::optional); [[nodiscard]] std::optional stepSelection() const; + [[nodiscard]] detail::AdiosAttributes const &attributes() const + { + return m_attributes; + } + [[nodiscard]] detail::AdiosAttributes &attributes() + { + return m_attributes; + } + private: ADIOS2IOHandlerImpl *m_impl; std::optional m_engine; //! ADIOS engine @@ -442,7 +445,7 @@ class ADIOS2File * the map that would be returned by a call to * IO::Available(Attributes|Variables). */ - std::optional m_availableAttributes; + AdiosAttributes m_attributes; std::optional m_availableVariables; std::set m_pathsMarkedAsActive; diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 355d1aa87f..da63b1196a 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -23,6 +23,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/ADIOS/macros.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" @@ -541,9 +542,26 @@ namespace detail struct AttributeReader { + struct GetAttribute + { + size_t step; + adios2::IO &IO; + detail::AdiosAttributes const &attributes; + template + auto call(std::string const &name) const + -> detail::AttributeWithShapeAndResource + { + return attributes.getAttribute(step, IO, name); + } + }; + template - static Datatype - call(adios2::IO &IO, std::string name, Attribute::resource &resource); + static Datatype call( + size_t step, + adios2::IO &IO, + std::string name, + Attribute::resource &resource, + detail::AdiosAttributes const &); template static Datatype call(Params &&...); diff --git a/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp new file mode 100644 index 0000000000..c6c2271b9f --- /dev/null +++ b/include/openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp @@ -0,0 +1,215 @@ +/* Copyright 2020-2025 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/config.hpp" +#if openPMD_HAVE_ADIOS2 + +#include +#include +#include +#include + +#include "openPMD/Datatype.hpp" +#include "openPMD/auxiliary/Variant.hpp" + +namespace openPMD::detail +{ +/** + * @brief Pointer to an attribute's data along with its shape. + * + * @tparam T Underlying attribute data type. + */ +template +struct AttributeWithShape +{ + size_t len; + T const *data; +}; + +/** + * Class that is responsible for buffering loaded openPMD attributes from + * ADIOS2. + * + * This is used for reading variable-encoded files in random-access mode. + * Random-access mode in ADIOS2 has the restriction that modifiable attributes + * can only be recovered in normal non-random-access read mode, so we open the + * file first in that mode, store all attribute metadata in this class and only + * then continue in random-access mode. + * + */ +class PreloadAdiosAttributes +{ +public: + /** + * Meta information on a buffered attribute. + */ + struct AttributeLocation + { + size_t len; + size_t offset; + Datatype dt; + char *destroy = nullptr; + + AttributeLocation() = delete; + AttributeLocation(size_t len, size_t offset, Datatype dt); + + AttributeLocation(AttributeLocation const &other) = delete; + AttributeLocation &operator=(AttributeLocation const &other) = delete; + + AttributeLocation(AttributeLocation &&other) noexcept; + AttributeLocation &operator=(AttributeLocation &&other) noexcept; + + ~AttributeLocation(); + }; + +private: + /* + * Allocate one large buffer instead of hundreds of single heap + * allocations. + * This will comply with alignment requirements, since + * std::allocator::allocate() will call the untyped new operator + * ::operator new(std::size_t) + * https://en.cppreference.com/w/cpp/memory/allocator/allocate + */ + std::vector m_rawBuffer; + std::map m_offsets; + +public: + explicit PreloadAdiosAttributes() = default; + PreloadAdiosAttributes(PreloadAdiosAttributes const &other) = delete; + PreloadAdiosAttributes & + operator=(PreloadAdiosAttributes const &other) = delete; + + PreloadAdiosAttributes(PreloadAdiosAttributes &&other) = default; + PreloadAdiosAttributes &operator=(PreloadAdiosAttributes &&other) = default; + + /** + * @brief Load attributes from the current step into the buffer. + * + * @param IO + */ + void preloadAttributes(adios2::IO &IO); + + /** + * @brief Get an attribute that has been buffered previously. + * + * @tparam T The underlying primitive datatype of the attribute. + * Will fail if the type found in ADIOS does not match. + * @param name The full name of the attribute. + * @return Pointer to the buffered attribute along with information on + * the attribute's shape. Valid only until any non-const member + * of PreloadAdiosAttributes is called. + */ + template + AttributeWithShape getAttribute(std::string const &name) const; + + Datatype attributeType(std::string const &name) const; + + std::map const &availableAttributes() const; +}; + +template +struct AttributeWithShapeAndResource : AttributeWithShape +{ + AttributeWithShapeAndResource(AttributeWithShape parent); + AttributeWithShapeAndResource( + size_t len_in, + T const *data_in, + std::optional> resource_in); + AttributeWithShapeAndResource(adios2::Attribute attr); + operator bool() const; + +private: + /* + * Users should still use the API of AttributeWithShape (parent type), we + * just need somewhere to store the std::vector returned by + * Attribute::Data(). This field will not be used when using preparsing, + * because the data pointer will go right into the preparse buffer. + */ + std::optional> resource; +}; + +struct AdiosAttributes +{ + using RandomAccess_t = std::vector; + struct StreamAccess_t + { + /* + * These are only buffered for performance reasons. + * IO::AvailableAttributes() returns by value, so we should avoid + * calling it too often. Instead, store the returned value along with + * the step, so we know when we need to update again (i.e. when the + * current step changes). + */ + size_t m_currentStep = 0; + std::optional> m_attributes; + }; + + /* + * Variant RandomAcces_t has to be initialized explicitly by + * ADIOS2IOHandlerImpl::readAttributeAllsteps(), so we use StreamAccess_t by + * default. + */ + std::variant m_data = StreamAccess_t{}; + + /* + * Needs to be this somewhat ugly API since the AvailableAttributes map has + * a different type depending if we use preparsing or not. If we don't use + * preparsing, we just use the returned std::map from IO::AvailableAttributes(). Otherwise, we use the + * std::map map from the + * PreloadAdiosAttributes class. The functor f will be called either with + * the one or the other, so needs to be written such that the mapped-to type + * does not matter. + */ + template + auto withAvailableAttributes(size_t step, adios2::IO &IO, Functor &&f) + -> decltype(std::forward(f)( + std::declval &>())) + { + using ret_t = decltype(std::forward(f)( + std::declval &>())); + return std::visit( + auxiliary::overloaded{ + [step, &f](RandomAccess_t &ra) -> ret_t { + auto &attribute_data = ra.at(step); + return std::forward(f)( + attribute_data.availableAttributes()); + }, + [step, &f, &IO](StreamAccess_t &sa) -> ret_t { + if (!sa.m_attributes.has_value() || + sa.m_currentStep != step) + { + sa = StreamAccess_t{step, IO.AvailableAttributes()}; + } + return std::forward(f)(*sa.m_attributes); + }}, + m_data); + } + + template + AttributeWithShapeAndResource + getAttribute(size_t step, adios2::IO &IO, std::string const &name) const; +}; +} // namespace openPMD::detail + +#endif // openPMD_HAVE_ADIOS2 diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 71800bdfc6..27edec51a8 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -344,7 +344,11 @@ namespace size_t ADIOS2File::currentStep() { - if (nonpersistentEngine(m_impl->m_engineType)) + if (auto step_selection = stepSelection(); step_selection.has_value()) + { + return *step_selection; + } + else if (nonpersistentEngine(m_impl->m_engineType)) { return m_currentStep; } @@ -759,19 +763,21 @@ void ADIOS2File::configure_IO() auto ADIOS2File::detectGroupTable() -> UseGroupTable { - auto const &attributes = availableAttributes(); - auto lower_bound = - attributes.lower_bound(adios_defaults::str_activeTablePrefix); - if (lower_bound != attributes.end() && - auxiliary::starts_with( - lower_bound->first, adios_defaults::str_activeTablePrefix)) - { - return UseGroupTable::Yes; - } - else - { - return UseGroupTable::No; - } + return m_attributes.withAvailableAttributes( + currentStep(), m_IO, [&](auto const &attributes) { + auto lower_bound = + attributes.lower_bound(adios_defaults::str_activeTablePrefix); + if (lower_bound != attributes.end() && + auxiliary::starts_with( + lower_bound->first, adios_defaults::str_activeTablePrefix)) + { + return UseGroupTable::Yes; + } + else + { + return UseGroupTable::No; + } + }); } adios2::Engine &ADIOS2File::getEngine() @@ -1264,7 +1270,6 @@ AdvanceStatus ADIOS2File::advance(AdvanceMode mode) case adios2::StepStatus::OtherError: throw std::runtime_error("[ADIOS2] Unexpected step status."); } - invalidateAttributesMap(); invalidateVariablesMap(); m_pathsMarkedAsActive.clear(); return res; @@ -1280,13 +1285,11 @@ void ADIOS2File::drop() assert(m_buffer.empty()); } +template static std::vector availableAttributesOrVariablesPrefixed( - std::string const &prefix, - ADIOS2File::AttributeMap_t const &(ADIOS2File::*getBasicMap)(), - ADIOS2File &ba) + std::string const &prefix, AttributesMap const &attributes) { std::string var = auxiliary::ends_with(prefix, '/') ? prefix : prefix + '/'; - ADIOS2File::AttributeMap_t const &attributes = (ba.*getBasicMap)(); std::vector ret; for (auto it = attributes.lower_bound(prefix); it != attributes.end(); ++it) { @@ -1305,33 +1308,17 @@ static std::vector availableAttributesOrVariablesPrefixed( std::vector ADIOS2File::availableAttributesPrefixed(std::string const &prefix) { - return availableAttributesOrVariablesPrefixed( - prefix, &ADIOS2File::availableAttributes, *this); + return m_attributes.withAvailableAttributes( + currentStep(), m_IO, [&](auto const &attributes) { + return availableAttributesOrVariablesPrefixed(prefix, attributes); + }); } std::vector ADIOS2File::availableVariablesPrefixed(std::string const &prefix) { return availableAttributesOrVariablesPrefixed( - prefix, &ADIOS2File::availableVariables, *this); -} - -void ADIOS2File::invalidateAttributesMap() -{ - m_availableAttributes = std::optional(); -} - -ADIOS2File::AttributeMap_t const &ADIOS2File::availableAttributes() -{ - if (m_availableAttributes) - { - return m_availableAttributes.value(); - } - else - { - m_availableAttributes = std::make_optional(m_IO.AvailableAttributes()); - return m_availableAttributes.value(); - } + prefix, ADIOS2File::availableVariables()); } void ADIOS2File::invalidateVariablesMap() diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index c91ff6b272..647c31d856 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -27,6 +27,7 @@ #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp" #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/Streaming.hpp" @@ -51,6 +52,7 @@ #include #include #include +#include namespace openPMD { @@ -1236,7 +1238,12 @@ void ADIOS2IOHandlerImpl::readAttribute( } Datatype ret = switchType( - type, ba.m_IO, name, *parameters.resource); + type, + ba.currentStep(), + ba.m_IO, + name, + *parameters.resource, + ba.attributes()); *parameters.dtype = ret; } @@ -1245,10 +1252,16 @@ namespace /* Used by both readAttribute() and readAttributeAllsteps() tasks. Functor fun will be called with the value of the retrieved attribute; both functions use different logic for processing the retrieved values. + Functor getAttribute is called for retrieving an attribute (Different + wrappers around IO::InquireAttribute<>()::Data(), together with + buffering). */ - template - Datatype - genericReadAttribute(Functor &&fun, adios2::IO &IO, std::string const &name) + template + Datatype genericReadAttribute( + Functor &&fun, + adios2::IO &IO, + std::string const &name, + GetAttribute const &getAttribute) { /* * If we store an attribute of boolean type, we store an additional @@ -1259,7 +1272,7 @@ namespace if constexpr (std::is_same::value) { - auto attr = IO.InquireAttribute(name); + auto attr = getAttribute.template call(name); if (!attr) { throw std::runtime_error( @@ -1282,15 +1295,15 @@ namespace if (type == determineDatatype()) { - auto meta = IO.InquireAttribute(metaAttr); + auto meta = IO.template InquireAttribute(metaAttr); if (meta.Data().size() == 1 && meta.Data()[0] == 1) { std::forward(fun)( - detail::bool_repr::fromRep(attr.Data()[0])); + detail::bool_repr::fromRep(attr.data[0])); return determineDatatype(); } } - std::forward(fun)(attr.Data()[0]); + std::forward(fun)(attr.data[0]); } else if constexpr (detail::IsUnsupportedComplex_v) { @@ -1300,27 +1313,30 @@ namespace } else if constexpr (auxiliary::IsVector_v) { - auto attr = IO.InquireAttribute(name); + auto attr = + getAttribute.template call(name); if (!attr) { throw std::runtime_error( "[ADIOS2] Internal error: Failed reading attribute '" + name + "'."); } - std::forward(fun)(attr.Data()); + std::forward(fun)(std::vector( + attr.data, attr.data + attr.len)); } else if constexpr (auxiliary::IsArray_v) { - auto attr = IO.InquireAttribute(name); + auto attr = + getAttribute.template call(name); if (!attr) { throw std::runtime_error( "[ADIOS2] Internal error: Failed reading attribute '" + name + "'."); } - auto data = attr.Data(); + auto data = attr.data; T res; - for (size_t i = 0; i < data.size(); i++) + for (size_t i = 0; i < attr.len; i++) { res[i] = data[i]; } @@ -1333,14 +1349,14 @@ namespace } else { - auto attr = IO.InquireAttribute(name); + auto attr = getAttribute.template call(name); if (!attr) { throw std::runtime_error( "[ADIOS2] Internal error: Failed reading attribute '" + name + "'."); } - std::forward(fun)(attr.Data()[0]); + std::forward(fun)(attr.data[0]); } return determineDatatype(); @@ -1348,18 +1364,29 @@ namespace struct ReadAttributeAllsteps { + struct GetAttribute + { + detail::PreloadAdiosAttributes const &p; + + template + [[nodiscard]] auto call(std::string const &name) const + -> detail::AttributeWithShapeAndResource + { + return p.getAttribute(name); + } + }; + template static void call( + std::vector const &preload, adios2::IO &IO, - adios2::Engine &engine, std::string const &name, - adios2::StepStatus status, Parameter::result_type &put_result_here) { - std::vector res; - res.reserve(engine.Steps()); - while (status == adios2::StepStatus::OK) + auto &res = put_result_here.emplace>(); + res.reserve(preload.size()); + for (auto const &p : preload) { genericReadAttribute( [&res](auto &&val) { @@ -1379,162 +1406,13 @@ namespace } }, IO, - name); - engine.EndStep(); - status = engine.BeginStep(); + name, + GetAttribute{p}); } - switch (status) - { - case adios2::StepStatus::OK: - throw error::Internal("Control flow error."); - case adios2::StepStatus::NotReady: - case adios2::StepStatus::OtherError: - throw error::ReadError( - error::AffectedObject::File, - error::Reason::CannotRead, - "ADIOS2", - "Unexpected step status while preparsing snapshots."); - case adios2::StepStatus::EndOfStream: - break; - } - put_result_here = std::move(res); } static constexpr char const *errorMsg = "ReadAttributeAllsteps"; }; - -#if openPMD_HAVE_MPI - struct DistributeToAllRanks - { - template - static void call( - Parameter::result_type - &put_result_here_in, - MPI_Comm comm, - int rank) - { - if (rank != 0) - { - put_result_here_in = std::vector{}; - } - std::vector &put_result_here = - std::get>(put_result_here_in); - size_t num_items = put_result_here.size(); - MPI_CHECK(MPI_Bcast( - &num_items, 1, auxiliary::openPMD_MPI_type(), 0, comm)); - if constexpr ( - std::is_same_v || - std::is_same_v> || - std::is_same_v || - std::is_same_v> || - auxiliary::IsArray_v || isComplexFloatingPoint()) - { - throw error::OperationUnsupportedInBackend( - "ADIOS2", - "[readAttributeAllsteps] No support for attributes of type " - "std::string, bool, std::complex or std::array in " - "parallel."); - } - else if constexpr ( - // auxiliary::IsArray_v || - auxiliary::IsVector_v) - { - std::vector sizes; - sizes.reserve(num_items); - if (rank == 0) - { - std::transform( - put_result_here.begin(), - put_result_here.end(), - std::back_inserter(sizes), - [](T const &arr) { return arr.size(); }); - } - sizes.resize(num_items); - MPI_CHECK(MPI_Bcast( - sizes.data(), - num_items, - auxiliary::openPMD_MPI_type(), - 0, - comm)); - size_t total_flat_size = - std::accumulate(sizes.begin(), sizes.end(), size_t(0)); - using flat_type = typename T::value_type; - std::vector flat_vector; - flat_vector.reserve(total_flat_size); - if (rank == 0) - { - for (auto const &arr : put_result_here) - { - for (auto val : arr) - { - flat_vector.push_back(val); - } - } - } - flat_vector.resize(total_flat_size); - MPI_CHECK(MPI_Bcast( - flat_vector.data(), - total_flat_size, - auxiliary::openPMD_MPI_type(), - 0, - comm)); - if (rank != 0) - { - size_t offset = 0; - put_result_here.reserve(num_items); - for (size_t current_extent : sizes) - { - put_result_here.emplace_back( - flat_vector.begin() + offset, - flat_vector.begin() + offset + current_extent); - offset += current_extent; - } - } - } - else - { - std::vector receive; - if (rank != 0) - { - receive.resize(num_items); - } - MPI_CHECK(MPI_Bcast( - rank == 0 ? put_result_here.data() : receive.data(), - num_items, - auxiliary::openPMD_MPI_type(), - 0, - comm)); - if (rank != 0) - { - put_result_here = std::move(receive); - } - } - } - static constexpr char const *errorMsg = "DistributeToAllRanks"; - }; -#endif - - void warn_ignored_modifiable_attributes(adios2::IO &IO) - { - auto modifiable_flag = IO.InquireAttribute( - adios_defaults::str_useModifiableAttributes); - auto print_warning = [](std::string const ¬e) { - std::cerr << "Warning: " << note << R"( -Random-access for variable-encoding in ADIOS2 is currently -experimental. Support for modifiable attributes is currently not implemented -yet, meaning that attributes such as /data/time will show useless values. -Use Access::READ_LINEAR to retrieve those values if needed. -)"; - }; - if (!modifiable_flag) - { - print_warning("File might be using modifiable attributes."); - } - else if (modifiable_flag.Data().at(0) != 0) - { - print_warning("File uses modifiable attributes."); - } - } } // namespace void ADIOS2IOHandlerImpl::readAttributeAllsteps( @@ -1543,42 +1421,48 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); auto pos = setAndGetFilePosition(writable); auto name = nameOfAttribute(writable, param.name); + detail::ADIOS2File &ba = getFileData(file, IfFileNotOpen::ThrowError); - auto read_from_file_in_serial = [&]() { - adios2::ADIOS adios; - auto IO = adios.DeclareIO("PreparseSnapshots"); - // @todo check engine type - IO.SetEngine(realEngineType()); - IO.SetParameter("StreamReader", "ON"); // this be for BP4 - auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); - auto status = engine.BeginStep(); - warn_ignored_modifiable_attributes(IO); - auto type = detail::attributeInfo(IO, name, /* verbose = */ true); - switchType( - type, IO, engine, name, status, *param.resource); - engine.Close(); - return type; - }; + auto type = detail::attributeInfo(ba.m_IO, name, /* verbose = */ true); #if openPMD_HAVE_MPI - if (!m_communicator.has_value()) + auto adios = [&]() { + if (m_communicator.has_value()) + { + return adios2::ADIOS(*m_communicator); + } + else + { + return adios2::ADIOS{}; + } + }(); +#else + adios2::ADIOS adios; +#endif + auto IO = adios.DeclareIO("PreparseSnapshots"); + IO.SetEngine(ba.m_IO.EngineType()); + IO.SetParameters(ba.m_IO.Parameters()); + IO.SetParameter("StreamReader", "ON"); // this be for BP4 + auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); + + std::vector preload; + preload.reserve(engine.Steps()); + adios2::StepStatus status; + while ((status = engine.BeginStep()) == adios2::StepStatus::OK) { - read_from_file_in_serial(); - return; + auto &new_entry = preload.emplace_back(); + new_entry.preloadAttributes(IO); + engine.EndStep(); } - int rank, size; - MPI_Comm_rank(*m_communicator, &rank); - MPI_Comm_size(*m_communicator, &size); - Datatype type; - if (rank == 0) + if (status != adios2::StepStatus::EndOfStream) { - type = read_from_file_in_serial(); + throw std::runtime_error( + "[ADIOS2IOHandlerImpl::readAttributeAllsteps] Unexpected step " + "status while beginning a step."); } - MPI_CHECK(MPI_Bcast(&type, 1, MPI_INT, 0, *m_communicator)); - switchType( - type, *param.resource, *m_communicator, rank); -#else - read_from_file_in_serial(); -#endif + engine.Close(); + auto &attributes = ba.attributes(); + switchType(type, preload, IO, name, *param.resource); + attributes.m_data = std::move(preload); } void ADIOS2IOHandlerImpl::listPaths( @@ -1653,15 +1537,26 @@ void ADIOS2IOHandlerImpl::listPaths( auto tablePrefix = adios_defaults::str_activeTablePrefix + myName; std::vector attrs = fileData.availableAttributesPrefixed(tablePrefix); - if (fileData.streamStatus == - detail::ADIOS2File::StreamStatus::DuringStep) + if ( + // either a step is currently active... + fileData.streamStatus == + detail::ADIOS2File::StreamStatus::DuringStep || + // ...or a step selection is currently active + (fileData.stepSelection().has_value() && + // This check is currently redundant, but may be relevant if we + // (re-)introduce a RandomAccessMode lite that does no + // preparsing of attributes. + std::holds_alternative< + detail::AdiosAttributes::RandomAccess_t>( + fileData.attributes().m_data))) { auto currentStep = fileData.currentStep(); + auto &IO = fileData.m_IO; for (auto const &attrName : attrs) { using table_t = unsigned long long; - auto attr = fileData.m_IO.InquireAttribute( - tablePrefix + attrName); + auto attr = fileData.attributes().getAttribute( + currentStep, IO, tablePrefix + attrName); if (!attr) { std::cerr << "[ADIOS2 backend] Unable to inquire group " @@ -1671,7 +1566,7 @@ void ADIOS2IOHandlerImpl::listPaths( << std::endl; continue; } - if (attr.Data()[0] != currentStep) + if (attr.data[0] != currentStep) { // group wasn't defined in current step continue; @@ -2113,14 +2008,19 @@ namespace detail { template Datatype AttributeReader::call( - adios2::IO &IO, std::string name, Attribute::resource &resource) + size_t step, + adios2::IO &IO, + std::string name, + Attribute::resource &resource, + detail::AdiosAttributes const &attributes) { return genericReadAttribute( [&resource](auto &&value) { resource = static_cast(value); }, IO, - name); + name, + GetAttribute{step, IO, attributes}); } template @@ -2148,7 +2048,6 @@ namespace detail auto &filedata = impl->getFileData( file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError); - filedata.invalidateAttributesMap(); adios2::IO IO = filedata.m_IO; impl->m_dirty.emplace(std::move(file)); diff --git a/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp new file mode 100644 index 0000000000..c3c2fbfbd7 --- /dev/null +++ b/src/IO/ADIOS/ADIOS2PreloadAttributes.cpp @@ -0,0 +1,353 @@ +/* Copyright 2020-2025 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/config.hpp" +#if openPMD_HAVE_ADIOS2 + +#include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp" + +#include "openPMD/Datatype.hpp" +#include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp" + +#include +#include +#include +#include + +namespace openPMD::detail +{ +namespace +{ + struct GetAlignment + { + template + static constexpr size_t call() + { + return alignof(T); + } + + template + static constexpr size_t call(Args &&...) + { + return alignof(std::max_align_t); + } + }; + + struct GetSize + { + template + static constexpr size_t call() + { + return sizeof(T); + } + + template + static constexpr size_t call(Args &&...) + { + return 0; + } + }; + + struct ScheduleLoad + { + template + static void call( + adios2::IO &IO, + std::string const &name, + char *buffer, + PreloadAdiosAttributes::AttributeLocation &location) + { + adios2::Attribute attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Variable not found: " + name); + } + /* + * MSVC does not like placement new of arrays, so we do it + * in a loop instead. + * https://developercommunity.visualstudio.com/t/c-placement-new-is-incorrectly-compiled/206439 + */ + T *dest = reinterpret_cast(buffer); + for (size_t i = 0; i < location.len; ++i) + { + new (dest + i) T(); + } + location.destroy = buffer; + auto data = attr.Data(); + std::copy_n(data.begin(), data.size(), dest); + } + + static constexpr char const *errorMsg = "ADIOS2"; + }; + + struct AttributeLen + { + template + static size_t call(adios2::IO &IO, std::string const &name) + { + auto attr = IO.InquireAttribute(name); + if (!attr) + { + throw std::runtime_error( + "[ADIOS2] Variable not found: " + name); + } + return attr.Data().size(); + } + + template + static size_t call(Args &&...) + { + return {}; + } + }; + + struct AttributeLocationDestroy + { + template + static void call(char *ptr, size_t numItems) + { + T *destroy = reinterpret_cast(ptr); + for (size_t i = 0; i < numItems; ++i) + { + destroy[i].~T(); + } + } + + template + static void call(Args &&...) + {} + }; +} // namespace + +using AttributeLocation = PreloadAdiosAttributes::AttributeLocation; + +AttributeLocation::AttributeLocation( + size_t len_in, size_t offset_in, Datatype dt_in) + : len(len_in), offset(offset_in), dt(dt_in) +{} + +AttributeLocation::AttributeLocation(AttributeLocation &&other) noexcept + : len{other.len}, offset{other.offset}, dt{other.dt}, destroy{other.destroy} +{ + other.destroy = nullptr; +} + +AttributeLocation & +AttributeLocation::operator=(AttributeLocation &&other) noexcept +{ + this->len = other.len; + this->offset = other.offset; + this->dt = other.dt; + this->destroy = other.destroy; + other.destroy = nullptr; + return *this; +} + +PreloadAdiosAttributes::AttributeLocation::~AttributeLocation() +{ + /* + * If the object has been moved from, this may be empty. + * Or else, if no custom destructor has been emplaced. + */ + if (destroy) + { + switchAdios2AttributeType(dt, destroy, len); + } +} + +void PreloadAdiosAttributes::preloadAttributes(adios2::IO &IO) +{ + m_offsets.clear(); + std::map> attributesByType; + auto addAttribute = [&attributesByType](Datatype dt, std::string name) { + constexpr size_t reserve = 10; + auto it = attributesByType.find(dt); + if (it == attributesByType.end()) + { + it = attributesByType.emplace_hint( + it, dt, std::vector()); + it->second.reserve(reserve); + } + it->second.push_back(std::move(name)); + }; + // PHASE 1: collect names of available attributes by ADIOS datatype + for (auto &attribute : IO.AvailableAttributes()) + { + // this will give us basic types only, no fancy vectors or similar + Datatype dt = fromADIOS2Type(IO.AttributeType(attribute.first)); + addAttribute(dt, attribute.first); + } + + // PHASE 2: get offsets for attributes in buffer + std::map offsets; + size_t currentOffset = 0; + for (auto &pair : attributesByType) + { + size_t alignment = switchAdios2AttributeType(pair.first); + size_t size = switchAdios2AttributeType(pair.first); + // go to next offset with valid alignment + size_t modulus = currentOffset % alignment; + if (modulus > 0) + { + currentOffset += alignment - modulus; + } + for (std::string &name : pair.second) + { + size_t elements = + switchAdios2AttributeType(pair.first, IO, name); + + m_offsets.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(name)), + std::forward_as_tuple(elements, currentOffset, pair.first)); + currentOffset += elements * size; + } + } + // now, currentOffset is the number of bytes that we need to allocate + // PHASE 3: allocate new buffer and schedule loads + m_rawBuffer.resize(currentOffset); + for (auto &pair : m_offsets) + { + switchAdios2AttributeType( + pair.second.dt, + IO, + pair.first, + &m_rawBuffer[pair.second.offset], + pair.second); + } +} + +template +AttributeWithShape +PreloadAdiosAttributes::getAttribute(std::string const &name) const +{ + auto it = m_offsets.find(name); + if (it == m_offsets.end()) + { + throw std::runtime_error( + "[ADIOS2] Requested attribute not found: " + name); + } + AttributeLocation const &location = it->second; + Datatype determinedDatatype = determineDatatype(); + if (location.dt != determinedDatatype) + { + std::stringstream errorMsg; + errorMsg << "[ADIOS2] Wrong datatype for attribute: " << name + << "(location.dt=" << location.dt + << ", T=" << determineDatatype() << ")"; + throw std::runtime_error(errorMsg.str()); + } + AttributeWithShape res; + res.len = location.len; + res.data = reinterpret_cast(&m_rawBuffer[location.offset]); + return res; +} + +Datatype PreloadAdiosAttributes::attributeType(std::string const &name) const +{ + auto it = m_offsets.find(name); + if (it == m_offsets.end()) + { + return Datatype::UNDEFINED; + } + return it->second.dt; +} + +std::map const & +PreloadAdiosAttributes::availableAttributes() const +{ + return m_offsets; +} + +template +auto AdiosAttributes::getAttribute( + size_t step, adios2::IO &IO, std::string const &name) const + -> AttributeWithShapeAndResource +{ + return std::visit( + auxiliary::overloaded{ + [step, &name]( + RandomAccess_t const &ra) -> AttributeWithShapeAndResource { + auto &attribute_data = ra.at(step); + return attribute_data.getAttribute(name); + }, + [&name, + &IO](StreamAccess_t const &) -> AttributeWithShapeAndResource { + auto attr = IO.InquireAttribute(name); + return {std::move(attr)}; + }}, + m_data); +} + +template +AttributeWithShapeAndResource::AttributeWithShapeAndResource( + AttributeWithShape parent) + : AttributeWithShape(std::move(parent)) +{} +template +AttributeWithShapeAndResource::AttributeWithShapeAndResource( + size_t len_in, T const *data_in, std::optional> resource_in) + : AttributeWithShape{len_in, data_in}, resource{std::move(resource_in)} +{} +template +AttributeWithShapeAndResource::AttributeWithShapeAndResource( + adios2::Attribute attr) +{ + if (!attr) + { + this->data = nullptr; + this->len = 0; + return; + } + auto vec = attr.Data(); + this->len = vec.size(); + this->data = vec.data(); + this->resource = std::move(vec); +} +template +AttributeWithShapeAndResource::operator bool() const +{ + return this->data; +} + +#define OPENPMD_INSTANTIATE_GETATTRIBUTE(type) \ + template AttributeWithShape PreloadAdiosAttributes::getAttribute( \ + std::string const &name) const; \ + template auto AdiosAttributes::getAttribute( \ + size_t step, adios2::IO &IO, std::string const &name) const \ + -> AttributeWithShapeAndResource; \ + template AttributeWithShapeAndResource< \ + type>::AttributeWithShapeAndResource(AttributeWithShape parent); \ + template AttributeWithShapeAndResource:: \ + AttributeWithShapeAndResource( \ + size_t len_in, \ + type const \ + *data_in, /* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \ + std::optional> resource_in); \ + template AttributeWithShapeAndResource< \ + type>::AttributeWithShapeAndResource(adios2::Attribute attr); \ + template AttributeWithShapeAndResource::operator bool() const; +ADIOS2_FOREACH_TYPE_1ARG(OPENPMD_INSTANTIATE_GETATTRIBUTE) +#undef OPENPMD_INSTANTIATE_GETATTRIBUTE +} // namespace openPMD::detail + +#endif // openPMD_HAVE_ADIOS2 diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 40aba2c8c1..14e46f90c8 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5950,11 +5950,8 @@ void variableBasedSeries(std::string const &file) { REQUIRE( iteration.getAttribute("changing_value").get() == - (supportsModifiableAttributes - ? (access == Access::READ_LINEAR - ? iteration.iterationIndex - : 9) - : 0)); + (supportsModifiableAttributes ? iteration.iterationIndex + : 0)); } auto E_x = iteration.meshes["E"]["x"]; REQUIRE(E_x.getDimensionality() == 1); @@ -5974,13 +5971,12 @@ void variableBasedSeries(std::string const &file) last_iteration_index = iteration.iterationIndex; - if (access == Access::READ_RANDOM_ACCESS) - { - continue; - } - // this loop ensures that only the recordcomponent ["E"]["i"] is // present where i == iteration.iterationIndex + // Note that this works for ReadRandomAccess as well since constant + // components contain no datasets. The ADIOS2 backend is however not + // (yet) able to deal with array datasets that are present only in a + // subselection of steps. for (uint64_t otherIteration = 0; otherIteration < 10; ++otherIteration) {