Skip to content

Improve attribute read time in ADIOS2 backend #613

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ax3l merged 2 commits into from
Dec 6, 2019
Merged

Improve attribute read time in ADIOS2 backend #613

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a5074ef
Buffer attributes map
franzpoeschel Nov 13, 2019
29dbe31
Add documentation for the nature of this PR
franzpoeschel Nov 29, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 45 additions & 2 deletions include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,7 @@ namespace detail
detail::AttributeReader m_attributeReader;
ADIOS2IOHandlerImpl & m_impl;

using AttributeMap_t = std::map< std::string, adios2::Params >;

BufferedActions( ADIOS2IOHandlerImpl & impl, InvalidatableFile file );

Expand All @@ -579,8 +580,50 @@ namespace detail
*/
void drop( );

std::map< std::string, adios2::Params >
availableAttributesTemporary( std::string const & variable );
AttributeMap_t const &
availableAttributes();

std::vector< std::string >
availableAttributesPrefixed( std::string const & prefix );

/*
* See description below.
*/
void
invalidateAttributesMap();

AttributeMap_t const &
availableVariables();

std::vector< std::string >
availableVariablesPrefixed( std::string const & prefix );

/*
* See description below.
*/
void
invalidateVariablesMap();

private:
/*
* ADIOS2 does not give direct access to its internal attribute and
* variable maps, but will instead give access to copies of them.
* In order to avoid unnecessary copies, we buffer the returned map.
* The downside of this is that we need to pay attention to invalidate
* the map whenever an attribute/variable is altered. In that case, we
* fetch the map anew.
* Revisit once https://github.com/openPMD/openPMD-api/issues/563 has
* been resolved
* If false, the buffered map has been invalidated and needs to be
* queried from ADIOS2 again. If true, the buffered map is equivalent to
* the map that would be returned by a call to
* IO::Available(Attributes|Variables).
*/
bool m_availableAttributesValid = false;
AttributeMap_t m_availableAttributes;

bool m_availableVariablesValid = false;
AttributeMap_t m_availableVariables;
};


Expand Down
179 changes: 125 additions & 54 deletions src/IO/ADIOS/ADIOS2IOHandler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,11 @@ void ADIOS2IOHandlerImpl::createDataset(
// cast from openPMD::Extent to adios2::Dims
adios2::Dims const shape( parameters.extent.begin(), parameters.extent.end() );

auto & fileData = getFileData( file );
switchType( parameters.dtype, detail::VariableDefiner( ),
getFileData( file ).m_IO, varName,
fileData.m_IO, varName,
std::move( compression ), shape );
fileData.invalidateVariablesMap();
writable->written = true;
m_dirty.emplace( file );
}
Expand Down Expand Up @@ -288,15 +290,19 @@ void ADIOS2IOHandlerImpl::deletePath(
throw std::runtime_error( "ADIOS2 backend does not support deletion." );
}

void ADIOS2IOHandlerImpl::deleteDataset(
Writable *, const Parameter< Operation::DELETE_DATASET > & )
void
ADIOS2IOHandlerImpl::deleteDataset(
Writable *,
const Parameter< Operation::DELETE_DATASET > & )
{
// call filedata.invalidateVariablesMap
throw std::runtime_error( "ADIOS2 backend does not support deletion." );
}

void ADIOS2IOHandlerImpl::deleteAttribute(
Writable *, const Parameter< Operation::DELETE_ATT > & )
{
// call filedata.invalidateAttributesMap
throw std::runtime_error( "ADIOS2 backend does not support deletion." );
}

Expand Down Expand Up @@ -358,7 +364,6 @@ void ADIOS2IOHandlerImpl::listPaths(
"Internal error: Writable not marked written during path listing" );
auto file = refreshFileFromParent( writable );
auto pos = setAndGetFilePosition( writable );
// adios2::Engine & engine = getEngine( file );
std::string myName = filePositionToString( pos );
if ( !auxiliary::ends_with( myName, '/' ) )
{
Expand All @@ -368,10 +373,9 @@ void ADIOS2IOHandlerImpl::listPaths(
/*
* since ADIOS does not have a concept of paths, restore them
* from variables and attributes.
* yes.
*/

auto & IO = getFileData( file ).m_IO;
auto & fileData = getFileData( file );
fileData.getEngine( ); // make sure that the attributes are present

std::unordered_set< std::string > subdirs;
/*
Expand All @@ -389,34 +393,30 @@ void ADIOS2IOHandlerImpl::listPaths(
std::vector< std::string > delete_me;
auto f = [myName, &subdirs, &delete_me](
std::vector< std::string > & varsOrAttrs, bool variables ) {
for ( auto var : varsOrAttrs )
for( auto var : varsOrAttrs )
{
if ( auxiliary::starts_with( var, myName ) )
auto firstSlash = var.find_first_of( '/' );
if( firstSlash != std::string::npos )
{
var = auxiliary::replace_first( var, myName, "" );
auto firstSlash = var.find_first_of( '/' );
if ( firstSlash != std::string::npos )
{
var = var.substr( 0, firstSlash );
subdirs.emplace( std::move( var ) );
}
else if ( variables )
{ // var is a dataset at the current level
delete_me.push_back( std::move( var ) );
}
var = var.substr( 0, firstSlash );
subdirs.emplace( std::move( var ) );
}
else if( variables )
{ // var is a dataset at the current level
delete_me.push_back( std::move( var ) );
}
}
};
std::vector< std::string > vars;
for ( auto const & p : IO.AvailableVariables( ) )
for( auto const & p : fileData.availableVariablesPrefixed( myName ) )
{
vars.emplace_back( p.first );
vars.emplace_back( std::move( p ) );
}

std::vector< std::string > attrs;
for ( auto const & p : IO.AvailableAttributes( ) )
for( auto const & p : fileData.availableAttributesPrefixed( myName ) )
{
attrs.emplace_back( p.first );
attrs.emplace_back( std::move( p ) );
}
f( vars, true );
f( attrs, false );
Expand Down Expand Up @@ -448,28 +448,25 @@ void ADIOS2IOHandlerImpl::listDatasets(
/*
* since ADIOS does not have a concept of paths, restore them
* from variables and attributes.
* yes.
*/

auto & fileData = getFileData( file );
fileData.getEngine( ); // make sure that the attributes are present

std::map< std::string, adios2::Params > vars =
getFileData( file ).m_IO.AvailableVariables( );
fileData.availableVariables();

std::unordered_set< std::string > subdirs;
for ( auto & pair : vars )
for( auto & var : fileData.availableVariablesPrefixed( myName ) )
{
std::string var = pair.first;
if ( auxiliary::starts_with( var, myName ) )
auto firstSlash = var.find_first_of( '/' );
if( firstSlash == std::string::npos )
{
var = auxiliary::replace_first( var, myName, "" );
auto firstSlash = var.find_first_of( '/' );
if ( firstSlash == std::string::npos )
{
subdirs.emplace( std::move( var ) );
} // else: var is a path or a dataset in a group below the current
// group
}
subdirs.emplace( std::move( var ) );
} // else: var is a path or a dataset in a group below the current
// group
}
for ( auto & dataset : subdirs )
for( auto & dataset : subdirs )
{
parameters.datasets->emplace_back( std::move( dataset ) );
}
Expand All @@ -490,10 +487,10 @@ void ADIOS2IOHandlerImpl::listAttributes(
}
auto & ba = getFileData( file );
ba.getEngine( ); // make sure that the attributes are present
auto const & attrs = ba.availableAttributesTemporary( attributePrefix );
for( auto & pair : attrs )
auto const & attrs = ba.availableAttributesPrefixed( attributePrefix );
for( auto & rawAttr : attrs )
{
auto attr = auxiliary::removeSlashes( pair.first );
auto attr = auxiliary::removeSlashes( rawAttr );
if( attr.find_last_of( '/' ) == std::string::npos )
{
// std::cout << "ATTRIBUTE at " << attributePrefix << ": " << attr
Expand All @@ -504,7 +501,8 @@ void ADIOS2IOHandlerImpl::listAttributes(
}
}

adios2::Mode ADIOS2IOHandlerImpl::adios2Accesstype( )
adios2::Mode
ADIOS2IOHandlerImpl::adios2Accesstype()
{
switch ( m_handler->accessTypeBackend )
{
Expand Down Expand Up @@ -741,7 +739,9 @@ namespace detail
auto fullName = impl->nameOfAttribute( writable, parameters.name );
auto prefix = impl->filePositionToString( pos );

adios2::IO IO = impl->getFileData( file ).m_IO;
auto & filedata = impl->getFileData( file );
filedata.invalidateAttributesMap();
adios2::IO IO = filedata.m_IO;
impl->m_dirty.emplace( std::move( file ) );

std::string t = IO.AttributeType( fullName );
Expand Down Expand Up @@ -1223,25 +1223,96 @@ namespace detail
m_buffer.clear();
}

std::map< std::string, adios2::Params >
BufferedActions::availableAttributesTemporary( std::string const & variable )
void
BufferedActions::invalidateAttributesMap()
{
m_availableAttributesValid = false;
m_availableAttributes.clear( );
}

BufferedActions::AttributeMap_t const &
BufferedActions::availableAttributes()
{
if( m_availableAttributesValid )
{
return m_availableAttributes;
}
else
{
m_availableAttributes = m_IO.AvailableAttributes();
m_availableAttributesValid = true;
return m_availableAttributes;
}
}

void
BufferedActions::invalidateVariablesMap()
{
m_availableVariablesValid = false;
m_availableVariables.clear();
}

BufferedActions::AttributeMap_t const &
BufferedActions::availableVariables()
{
std::string var =
auxiliary::ends_with( variable, '/' ) ? variable : variable + '/';
auto attributes = m_IO.AvailableAttributes( "" );
decltype( attributes ) ret;
for( auto & pair : attributes )
if( m_availableVariablesValid )
{
return m_availableVariables;
}
else
{
m_availableVariables = m_IO.AvailableVariables();
m_availableVariablesValid = true;
return m_availableVariables;
}
}

static std::vector< std::string >
availableAttributesOrVariablesPrefixed(
std::string const & prefix,
BufferedActions::AttributeMap_t const & (
BufferedActions::*getBasicMap )(),
BufferedActions & ba )
{
std::string var = auxiliary::ends_with( prefix, '/' ) ? prefix
: prefix + '/';
BufferedActions::AttributeMap_t const & attributes =
( ba.*getBasicMap )();
std::vector< std::string > ret;
for( auto it = attributes.lower_bound( prefix ); it != attributes.end();
++it )
{
if( auxiliary::starts_with( pair.first, var ) )
if( auxiliary::starts_with( it->first, var ) )
{
ret.emplace_back(
auxiliary::replace_first( it->first, var, "" ) );
}
else
{
ret.emplace(
auxiliary::replace_first( pair.first, var, "" ),
std::move( pair.second ) );
break;
}
}
return ret;
}

std::vector< std::string >
BufferedActions::availableAttributesPrefixed( std::string const & prefix )
{
return availableAttributesOrVariablesPrefixed(
prefix,
&BufferedActions::availableAttributes,
*this );
}

std::vector< std::string >
BufferedActions::availableVariablesPrefixed( std::string const & prefix )
{
return availableAttributesOrVariablesPrefixed(
prefix,
&BufferedActions::availableVariables,
*this );
}

} // namespace detail

#if openPMD_HAVE_MPI
Expand Down