From a5074ef3fb7083281fe0c0864e8557d78059cf12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 13 Nov 2019 16:24:01 +0100 Subject: [PATCH 1/2] Buffer attributes map Use the fact that std::map sorts string lexicographically This allows faster lookup by prefix. Dito for variables --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 33 +++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 179 +++++++++++++------ 2 files changed, 156 insertions(+), 56 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 5927212af1..9188e57eee 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -562,6 +562,7 @@ namespace detail detail::AttributeReader m_attributeReader; ADIOS2IOHandlerImpl & m_impl; + using AttributeMap_t = std::map< std::string, adios2::Params >; BufferedActions( ADIOS2IOHandlerImpl & impl, InvalidatableFile file ); @@ -579,8 +580,36 @@ namespace detail */ void drop( ); - std::map< std::string, adios2::Params > - availableAttributesTemporary( std::string const & variable ); + AttributeMap_t const & + availableAttributes(); + + std::vector< std::string > + availableAttributesPrefixed( std::string const & prefix ); + + void + invalidateAttributesMap(); + + AttributeMap_t const & + availableVariables(); + + std::vector< std::string > + availableVariablesPrefixed( std::string const & prefix ); + + void + invalidateVariablesMap(); + + private: + /* + * Revisit once https://github.com/openPMD/openPMD-api/issues/563 has + * been resolved + * If false, the buffered map has been invalidated and needs to be + * queried from ADIOS2 again. + */ + bool m_availableAttributesValid = false; + AttributeMap_t m_availableAttributes; + + bool m_availableVariablesValid = false; + AttributeMap_t m_availableVariables; }; diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2cfa531cf8..12d3a9211e 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -204,9 +204,11 @@ void ADIOS2IOHandlerImpl::createDataset( // cast from openPMD::Extent to adios2::Dims adios2::Dims const shape( parameters.extent.begin(), parameters.extent.end() ); + auto & fileData = getFileData( file ); switchType( parameters.dtype, detail::VariableDefiner( ), - getFileData( file ).m_IO, varName, + fileData.m_IO, varName, std::move( compression ), shape ); + fileData.invalidateVariablesMap(); writable->written = true; m_dirty.emplace( file ); } @@ -288,15 +290,19 @@ void ADIOS2IOHandlerImpl::deletePath( throw std::runtime_error( "ADIOS2 backend does not support deletion." ); } -void ADIOS2IOHandlerImpl::deleteDataset( - Writable *, const Parameter< Operation::DELETE_DATASET > & ) +void +ADIOS2IOHandlerImpl::deleteDataset( + Writable *, + const Parameter< Operation::DELETE_DATASET > & ) { + // call filedata.invalidateVariablesMap throw std::runtime_error( "ADIOS2 backend does not support deletion." ); } void ADIOS2IOHandlerImpl::deleteAttribute( Writable *, const Parameter< Operation::DELETE_ATT > & ) { + // call filedata.invalidateAttributesMap throw std::runtime_error( "ADIOS2 backend does not support deletion." ); } @@ -358,7 +364,6 @@ void ADIOS2IOHandlerImpl::listPaths( "Internal error: Writable not marked written during path listing" ); auto file = refreshFileFromParent( writable ); auto pos = setAndGetFilePosition( writable ); - // adios2::Engine & engine = getEngine( file ); std::string myName = filePositionToString( pos ); if ( !auxiliary::ends_with( myName, '/' ) ) { @@ -368,10 +373,9 @@ void ADIOS2IOHandlerImpl::listPaths( /* * since ADIOS does not have a concept of paths, restore them * from variables and attributes. - * yes. */ - - auto & IO = getFileData( file ).m_IO; + auto & fileData = getFileData( file ); + fileData.getEngine( ); // make sure that the attributes are present std::unordered_set< std::string > subdirs; /* @@ -389,34 +393,30 @@ void ADIOS2IOHandlerImpl::listPaths( std::vector< std::string > delete_me; auto f = [myName, &subdirs, &delete_me]( std::vector< std::string > & varsOrAttrs, bool variables ) { - for ( auto var : varsOrAttrs ) + for( auto var : varsOrAttrs ) { - if ( auxiliary::starts_with( var, myName ) ) + auto firstSlash = var.find_first_of( '/' ); + if( firstSlash != std::string::npos ) { - var = auxiliary::replace_first( var, myName, "" ); - auto firstSlash = var.find_first_of( '/' ); - if ( firstSlash != std::string::npos ) - { - var = var.substr( 0, firstSlash ); - subdirs.emplace( std::move( var ) ); - } - else if ( variables ) - { // var is a dataset at the current level - delete_me.push_back( std::move( var ) ); - } + var = var.substr( 0, firstSlash ); + subdirs.emplace( std::move( var ) ); + } + else if( variables ) + { // var is a dataset at the current level + delete_me.push_back( std::move( var ) ); } } }; std::vector< std::string > vars; - for ( auto const & p : IO.AvailableVariables( ) ) + for( auto const & p : fileData.availableVariablesPrefixed( myName ) ) { - vars.emplace_back( p.first ); + vars.emplace_back( std::move( p ) ); } std::vector< std::string > attrs; - for ( auto const & p : IO.AvailableAttributes( ) ) + for( auto const & p : fileData.availableAttributesPrefixed( myName ) ) { - attrs.emplace_back( p.first ); + attrs.emplace_back( std::move( p ) ); } f( vars, true ); f( attrs, false ); @@ -448,28 +448,25 @@ void ADIOS2IOHandlerImpl::listDatasets( /* * since ADIOS does not have a concept of paths, restore them * from variables and attributes. - * yes. */ + auto & fileData = getFileData( file ); + fileData.getEngine( ); // make sure that the attributes are present + std::map< std::string, adios2::Params > vars = - getFileData( file ).m_IO.AvailableVariables( ); + fileData.availableVariables(); std::unordered_set< std::string > subdirs; - for ( auto & pair : vars ) + for( auto & var : fileData.availableVariablesPrefixed( myName ) ) { - std::string var = pair.first; - if ( auxiliary::starts_with( var, myName ) ) + auto firstSlash = var.find_first_of( '/' ); + if( firstSlash == std::string::npos ) { - var = auxiliary::replace_first( var, myName, "" ); - auto firstSlash = var.find_first_of( '/' ); - if ( firstSlash == std::string::npos ) - { - subdirs.emplace( std::move( var ) ); - } // else: var is a path or a dataset in a group below the current - // group - } + subdirs.emplace( std::move( var ) ); + } // else: var is a path or a dataset in a group below the current + // group } - for ( auto & dataset : subdirs ) + for( auto & dataset : subdirs ) { parameters.datasets->emplace_back( std::move( dataset ) ); } @@ -490,10 +487,10 @@ void ADIOS2IOHandlerImpl::listAttributes( } auto & ba = getFileData( file ); ba.getEngine( ); // make sure that the attributes are present - auto const & attrs = ba.availableAttributesTemporary( attributePrefix ); - for( auto & pair : attrs ) + auto const & attrs = ba.availableAttributesPrefixed( attributePrefix ); + for( auto & rawAttr : attrs ) { - auto attr = auxiliary::removeSlashes( pair.first ); + auto attr = auxiliary::removeSlashes( rawAttr ); if( attr.find_last_of( '/' ) == std::string::npos ) { // std::cout << "ATTRIBUTE at " << attributePrefix << ": " << attr @@ -504,7 +501,8 @@ void ADIOS2IOHandlerImpl::listAttributes( } } -adios2::Mode ADIOS2IOHandlerImpl::adios2Accesstype( ) +adios2::Mode +ADIOS2IOHandlerImpl::adios2Accesstype() { switch ( m_handler->accessTypeBackend ) { @@ -741,7 +739,9 @@ namespace detail auto fullName = impl->nameOfAttribute( writable, parameters.name ); auto prefix = impl->filePositionToString( pos ); - adios2::IO IO = impl->getFileData( file ).m_IO; + auto & filedata = impl->getFileData( file ); + filedata.invalidateAttributesMap(); + adios2::IO IO = filedata.m_IO; impl->m_dirty.emplace( std::move( file ) ); std::string t = IO.AttributeType( fullName ); @@ -1223,25 +1223,96 @@ namespace detail m_buffer.clear(); } - std::map< std::string, adios2::Params > - BufferedActions::availableAttributesTemporary( std::string const & variable ) + void + BufferedActions::invalidateAttributesMap() + { + m_availableAttributesValid = false; + m_availableAttributes.clear( ); + } + + BufferedActions::AttributeMap_t const & + BufferedActions::availableAttributes() + { + if( m_availableAttributesValid ) + { + return m_availableAttributes; + } + else + { + m_availableAttributes = m_IO.AvailableAttributes(); + m_availableAttributesValid = true; + return m_availableAttributes; + } + } + + void + BufferedActions::invalidateVariablesMap() + { + m_availableVariablesValid = false; + m_availableVariables.clear(); + } + + BufferedActions::AttributeMap_t const & + BufferedActions::availableVariables() { - std::string var = - auxiliary::ends_with( variable, '/' ) ? variable : variable + '/'; - auto attributes = m_IO.AvailableAttributes( "" ); - decltype( attributes ) ret; - for( auto & pair : attributes ) + if( m_availableVariablesValid ) + { + return m_availableVariables; + } + else + { + m_availableVariables = m_IO.AvailableVariables(); + m_availableVariablesValid = true; + return m_availableVariables; + } + } + + static std::vector< std::string > + availableAttributesOrVariablesPrefixed( + std::string const & prefix, + BufferedActions::AttributeMap_t const & ( + BufferedActions::*getBasicMap )(), + BufferedActions & ba ) + { + std::string var = auxiliary::ends_with( prefix, '/' ) ? prefix + : prefix + '/'; + BufferedActions::AttributeMap_t const & attributes = + ( ba.*getBasicMap )(); + std::vector< std::string > ret; + for( auto it = attributes.lower_bound( prefix ); it != attributes.end(); + ++it ) { - if( auxiliary::starts_with( pair.first, var ) ) + if( auxiliary::starts_with( it->first, var ) ) + { + ret.emplace_back( + auxiliary::replace_first( it->first, var, "" ) ); + } + else { - ret.emplace( - auxiliary::replace_first( pair.first, var, "" ), - std::move( pair.second ) ); + break; } } return ret; } + std::vector< std::string > + BufferedActions::availableAttributesPrefixed( std::string const & prefix ) + { + return availableAttributesOrVariablesPrefixed( + prefix, + &BufferedActions::availableAttributes, + *this ); + } + + std::vector< std::string > + BufferedActions::availableVariablesPrefixed( std::string const & prefix ) + { + return availableAttributesOrVariablesPrefixed( + prefix, + &BufferedActions::availableVariables, + *this ); + } + } // namespace detail #if openPMD_HAVE_MPI From 29dbe31142c57607c7f035f8ca0c66e54bf03680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 29 Nov 2019 11:10:58 +0100 Subject: [PATCH 2/2] Add documentation for the nature of this PR --- include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index 9188e57eee..92c23f9ecf 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -586,6 +586,9 @@ namespace detail std::vector< std::string > availableAttributesPrefixed( std::string const & prefix ); + /* + * See description below. + */ void invalidateAttributesMap(); @@ -595,15 +598,26 @@ namespace detail std::vector< std::string > availableVariablesPrefixed( std::string const & prefix ); + /* + * See description below. + */ void invalidateVariablesMap(); private: /* + * ADIOS2 does not give direct access to its internal attribute and + * variable maps, but will instead give access to copies of them. + * In order to avoid unnecessary copies, we buffer the returned map. + * The downside of this is that we need to pay attention to invalidate + * the map whenever an attribute/variable is altered. In that case, we + * fetch the map anew. * Revisit once https://github.com/openPMD/openPMD-api/issues/563 has * been resolved * If false, the buffered map has been invalidated and needs to be - * queried from ADIOS2 again. + * queried from ADIOS2 again. If true, the buffered map is equivalent to + * the map that would be returned by a call to + * IO::Available(Attributes|Variables). */ bool m_availableAttributesValid = false; AttributeMap_t m_availableAttributes;