From 09996005614367c8e8cdf9ab763248fc27bcb84b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 20 Aug 2018 12:48:23 +0200 Subject: [PATCH 1/2] JSON backend Implement JSON backend to dummy state Misses the actual implementation of AbstractIOHandlerImpl Declare IOHandlerImpl for JSON and integrate it with other places Misses the implementation. Undebugged minimum implementation for JSON writing First basically runnable version of JSON writing To address: No reading or deleting yet. Datatypes are currently ignored and the data is assumed to be int64_t. Attribute values are ignored and replaced with a dummy value. If a subgroup name can be parsed as a nonnegative string, the JSON API will create a JSON array rather than a JSON object (associative array) as intended. Correctly handle groups that can be parsed as int See last commit's description. Fix index calculation with offsets in WriteData Fix some mistakes in JSON writing Correctly handle overwriting files: -> overwritten files should not be possible to access any longer and be clearly distinguished from the newly-created file Make some verifications execute independent of compiler options. Full implementation of JSON writing Respects all datatypes now. Format code according to Clion Stylesheet https://github.com/ComputationalRadiationPhysics/contributing/blob/master/IDESettings/CLion/CRP_CLion2016_1.xml Add generic branching over an openPMD datatype First runnable version of JSON Reading Cleanup and implementation of dataset extension Undebugged version of JSON deletion Properly (de)serialize datatypes Instead of casting the Datatype enum to and from int (which is likely to break when altering the enum), serialize to and from String values. Fix a number of mistakes in JSON reading and writing Cleanup Add JSON tests and fix bugs found thusly Add further tests and fix a further bug The JSON library does not understand long double values (i.e. 128bit floats), represent them as a char array. Handle floating point special values JSON represents +/-Infinity and NaN values as null. The JSON library will correctly serialize those values *to* JSON, implement (semi)-correct handly for deserialization. As it is unclear which exact value a null represents, deserialize it to NaN. Take notice that large floating point values (128 bit) might be serialized to null as well. Use std::is_floating_point to distinguish them from other types Additionally write the byte width of the underlying type Not yet used in reading Mark the writable written after successfully extending a dataset Remove support for absolute paths from openPath Fix some rough edges from rebasing Add documentation for the JSON backend Integrate the JSON backend with the build system Make platform bytewidth information global per JSON file Was previously annotated for every single Dataset and Attribute. Fixes -> shadowed variables -> return file streams by shared pointer so that Clang won't be mad about missing copy constructors -> handle inconsistencies between GCC, Clang and MSVC in switchType function Add licensing information and cleanup imports Cleanup Remove unused code from Filesystem.(h|c)pp Unify replace_all and replace_all_nonrecursively in StringManip.hpp Move implementation of JSONFilePosition class to source file Formatting Refactor tests Move tests into functions of the backend to be used. Remove /subgroups/ and /datasets/ indirection Make optional_paths_110_test HDF5 specific again Eliminate "extent" field from datasets Remove object_type key --- CMakeLists.txt | 5 +- README.md | 4 +- docs/source/backends/json.rst | 83 + docs/source/backends/json_example.json | 142 + docs/source/dev/buildoptions.rst | 2 +- docs/source/index.rst | 12 + docs/source/usage/firststeps.rst | 2 +- docs/source/usage/parallel.rst | 2 +- docs/source/usage/serial.rst | 2 +- include/openPMD/Datatype.hpp | 11 +- include/openPMD/IO/Format.hpp | 1 + include/openPMD/IO/JSON/JSONFilePosition.hpp | 49 + include/openPMD/IO/JSON/JSONIOHandler.hpp | 47 + include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 576 +++ include/openPMD/auxiliary/StringManip.hpp | 21 +- include/openPMD/backend/Writable.hpp | 1 + src/IO/AbstractIOHandlerHelper.cpp | 3 + src/IO/JSON/JSONFilePosition.cpp | 11 + src/IO/JSON/JSONIOHandler.cpp | 47 + src/IO/JSON/JSONIOHandlerImpl.cpp | 1562 +++++++++ src/Series.cpp | 15 + src/auxiliary/Filesystem.cpp | 3 +- test/SerialIOTest.cpp | 3116 ++++++++--------- 23 files changed, 4049 insertions(+), 1668 deletions(-) create mode 100644 docs/source/backends/json.rst create mode 100644 docs/source/backends/json_example.json create mode 100644 include/openPMD/IO/JSON/JSONFilePosition.hpp create mode 100644 include/openPMD/IO/JSON/JSONIOHandler.hpp create mode 100644 include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp create mode 100644 src/IO/JSON/JSONFilePosition.cpp create mode 100644 src/IO/JSON/JSONIOHandler.cpp create mode 100644 src/IO/JSON/JSONIOHandlerImpl.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c73e2dbe1..417a507607 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -304,7 +304,10 @@ set(IO_SOURCE src/IO/AbstractIOHandlerHelper.cpp src/IO/IOTask.cpp src/IO/HDF5/HDF5IOHandler.cpp - src/IO/HDF5/ParallelHDF5IOHandler.cpp) + src/IO/HDF5/ParallelHDF5IOHandler.cpp + src/IO/JSON/JSONIOHandler.cpp + src/IO/JSON/JSONIOHandlerImpl.cpp + src/IO/JSON/JSONFilePosition.cpp) set(IO_ADIOS1_SEQUENTIAL_SOURCE src/IO/AbstractIOHandler.cpp src/IO/AbstractIOHandlerImpl.cpp diff --git a/README.md b/README.md index 2a1cfbdbb0..b5fcb0f562 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Shipped internally in `share/openPMD/thirdParty/`: * [NLohmann-JSON](https://github.com/nlohmann/json) 3.4.0+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) Optional I/O backends: -* [JSON](https://en.wikipedia.org/wiki/JSON) (*not yet implemented*) +* [JSON](https://en.wikipedia.org/wiki/JSON) * [HDF5](https://support.hdfgroup.org/HDF5) 1.8.13+ * [ADIOS1](https://www.olcf.ornl.gov/center-projects/adios) 1.13.1+ * [ADIOS2](https://github.com/ornladios/ADIOS2) 2.1+ (*not yet implemented*) @@ -175,7 +175,7 @@ CMake controls options with prefixed `-D`, e.g. `-DopenPMD_USE_MPI=OFF`: | CMake Option | Values | Description | |------------------------------|------------------|------------------------------------------------------------------------------| | `openPMD_USE_MPI` | **AUTO**/ON/OFF | Enable MPI support | -| `openPMD_USE_JSON` | **AUTO**/ON/OFF | Enable support for JSON 1 | +| `openPMD_USE_JSON` | **AUTO**/ON/OFF | Enable support for JSON | | `openPMD_USE_HDF5` | **AUTO**/ON/OFF | Enable support for HDF5 | | `openPMD_USE_ADIOS1` | **AUTO**/ON/OFF | Enable support for ADIOS1 | | `openPMD_USE_ADIOS2` | AUTO/ON/**OFF** | Enable support for ADIOS2 1 | diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst new file mode 100644 index 0000000000..42b55844b7 --- /dev/null +++ b/docs/source/backends/json.rst @@ -0,0 +1,83 @@ +.. _backends-json: + +JSON Backend +============ + +openPMD supports writing to and reading from JSON files. +For this, the installed copy of openPMD must have been built with support for the JSON backend. +To build openPMD with support for JSON, use the CMake option ``-DopenPMD_USE_JSON=ON``. +For further information, check out the :ref:`installation guide `, +:ref:`build dependencies ` and the :ref:`build options `. + + +JSON File Format +---------------- +A JSON file uses the file ending ``.json``. The JSON backend is chosen by creating +a ``Series`` object with a filename that has this file ending. + +The top-level JSON object is a group representing the openPMD root group ``"/"``. +Any **openPMD group** is represented in JSON as a JSON object with two reserved keys: + + * ``attributes``: Attributes associated with the group. This key may be null or not be present + at all, thus indicating a group without attributes. + * ``platform_byte_widths`` (root group only): Byte widths specific to the writing platform. + Will be overwritten every time that a JSON value is stored to disk, hence this information + is only available about the last platform writing the JSON value. + +All datasets and subgroups contained in this group are represented as a further key of +the group object. ``attributes`` and ``platform_byte_widths`` have +hence the character of reserved keywords and cannot be used for group and dataset names +when working with the JSON backend. +Datasets and groups have the same namespace, meaning that there may not be a subgroup +and a dataset with the same name contained in one group. + +Any **openPMD dataset** is a JSON object with three keys: + + * ``attributes``: Attributes associated with the dataset. May be ``null`` or not present if no attributes are associated with the dataset. + * ``datatype``: A string describing the type of the stored data. + * ``data`` A nested array storing the actual data in row-major manner. + The data needs to be consistent with the fields ``datatype`` and ``extent``. + Checking whether this key points to an array can be (and is internally) used to distinguish groups from datasets. + +**Attributes** are stored as a JSON object with a key for each attribute. +Every such attribute is itself a JSON object with two keys: + + * ``datatype``: A string describing the type of the value. + * ``value``: The actual value of type ``datatype``. + +Restrictions +------------ +For creation of JSON serializations (i.e. writing), the restrictions of the JSON backend are +equivalent to those of the `JSON library by Niels Lohmann `_ +used by the openPMD backend. + +Numerical values, integral as well as floating point, are supported up to a length of +64 bits. +Since JSON does not support special floating point values (i.e. NaN, Infinity, -Infinity), +those values are rendered as ``null``. + +Instructing openPMD to write values of a datatype that is too wide for the JSON +backend does *not* result in an error: + * If casting the value to the widest supported datatype of the same category (integer or floating point) + is possible without data loss, the cast is performed and the value is written. + As an example, on a platform with ``sizeof(double) == 8``, writing the value + ``static_cast(std::numeric_limits::max())`` will work as expected + since it can be cast back to ``double``. + * Otherwise, a ``null`` value is written. + +Upon reading ``null`` when expecting a floating point number, a NaN value will be +returned. Take notice that a NaN value returned from the deserialization process +may have originally been +/-Infinity or beyond the supported value range. + +Upon reading ``null`` when expecting any other datatype, the JSON backend will +propagate the exception thrown by Niels Lohmann's library. + +A parallel (i.e. MPI) implementation is *not* available. + +Example +------- +The example code in the :ref:`usage section ` will produce the following JSON serialization +when picking the JSON backend: + +.. literalinclude:: json_example.json + diff --git a/docs/source/backends/json_example.json b/docs/source/backends/json_example.json new file mode 100644 index 0000000000..2fad4fc451 --- /dev/null +++ b/docs/source/backends/json_example.json @@ -0,0 +1,142 @@ +{ + "attributes": { + "basePath": { + "datatype": "STRING", + "value": "/data/%T/" + }, + "iterationEncoding": { + "datatype": "STRING", + "value": "groupBased" + }, + "iterationFormat": { + "datatype": "STRING", + "value": "/data/%T/" + }, + "meshesPath": { + "datatype": "STRING", + "value": "meshes/" + }, + "openPMD": { + "datatype": "STRING", + "value": "1.1.0" + }, + "openPMDextension": { + "datatype": "UINT", + "value": 0 + } + }, + "data": { + "1": { + "attributes": { + "dt": { + "datatype": "DOUBLE", + "value": 1 + }, + "time": { + "datatype": "DOUBLE", + "value": 0 + }, + "timeUnitSI": { + "datatype": "DOUBLE", + "value": 1 + } + }, + "meshes": { + "rho": { + "attributes": { + "axisLabels": { + "datatype": "VEC_STRING", + "value": [ + "x" + ] + }, + "dataOrder": { + "datatype": "STRING", + "value": "C" + }, + "geometry": { + "datatype": "STRING", + "value": "cartesian" + }, + "gridGlobalOffset": { + "datatype": "VEC_DOUBLE", + "value": [ + 0 + ] + }, + "gridSpacing": { + "datatype": "VEC_DOUBLE", + "value": [ + 1 + ] + }, + "gridUnitSI": { + "datatype": "DOUBLE", + "value": 1 + }, + "position": { + "datatype": "VEC_DOUBLE", + "value": [ + 0 + ] + }, + "timeOffset": { + "datatype": "FLOAT", + "value": 0 + }, + "unitDimension": { + "datatype": "ARR_DBL_7", + "value": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + "unitSI": { + "datatype": "DOUBLE", + "value": 1 + } + }, + "data": [ + [ + 0, + 1, + 2 + ], + [ + 3, + 4, + 5 + ], + [ + 6, + 7, + 8 + ] + ], + "datatype": "DOUBLE" + } + } + } + }, + "platform_byte_widths": { + "BOOL": 1, + "CHAR": 1, + "DOUBLE": 8, + "FLOAT": 4, + "INT": 4, + "LONG": 8, + "LONGLONG": 8, + "LONG_DOUBLE": 16, + "SHORT": 2, + "UCHAR": 1, + "UINT": 4, + "ULONG": 8, + "ULONGLONG": 8, + "USHORT": 2 + } +} diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index bcae829343..e016a38ef4 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -15,7 +15,7 @@ CMake controls options with prefixed ``-D``, e.g. ``-DopenPMD_USE_MPI=OFF``: CMake Option Values Description ============================== =============== ======================================================================== ``openPMD_USE_MPI`` **AUTO**/ON/OFF Enable MPI support -``openPMD_USE_JSON`` **AUTO**/ON/OFF Enable support for JSON :sup:`1` +``openPMD_USE_JSON`` **AUTO**/ON/OFF Enable support for JSON ``openPMD_USE_HDF5`` **AUTO**/ON/OFF Enable support for HDF5 ``openPMD_USE_ADIOS1`` **AUTO**/ON/OFF Enable support for ADIOS1 ``openPMD_USE_ADIOS2`` AUTO/ON/**OFF** Enable support for ADIOS2 :sup:`1` diff --git a/docs/source/index.rst b/docs/source/index.rst index 2c9f0a743a..6d8fecb00e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -84,6 +84,17 @@ Development dev/doxygen dev/release +******** +Backends +******** +.. toctree:: + :caption: BACKENDS + :maxdepth: 1 + :hidden: + + backends/json + + ********* Utilities ********* @@ -93,3 +104,4 @@ Utilities :hidden: utilities/benchmark.rst + diff --git a/docs/source/usage/firststeps.rst b/docs/source/usage/firststeps.rst index be53d92d60..a799ca56ae 100644 --- a/docs/source/usage/firststeps.rst +++ b/docs/source/usage/firststeps.rst @@ -1,4 +1,4 @@ -.. usage-firststeps: +.. _usage-firststeps: First Steps =========== diff --git a/docs/source/usage/parallel.rst b/docs/source/usage/parallel.rst index 17c2c881bf..c98e362929 100644 --- a/docs/source/usage/parallel.rst +++ b/docs/source/usage/parallel.rst @@ -1,4 +1,4 @@ -.. usage-parallel: +.. _usage-parallel: Parallel API ============ diff --git a/docs/source/usage/serial.rst b/docs/source/usage/serial.rst index fc92894d5d..fe77c2d446 100644 --- a/docs/source/usage/serial.rst +++ b/docs/source/usage/serial.rst @@ -1,4 +1,4 @@ -.. usage-serial: +.. _usage-serial: Serial API ========== diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 2c38077271..0635f1c951 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -519,6 +519,11 @@ isSame( openPMD::Datatype const d, openPMD::Datatype const e ) return false; } +#if _MSC_VER && !__INTEL_COMPILER +#define OPENPMD_TEMPLATE_OPERATOR operator +#else +#define OPENPMD_TEMPLATE_OPERATOR template operator +#endif /** * Generalizes switching over an openPMD datatype. * @@ -536,12 +541,6 @@ isSame( openPMD::Datatype const d, openPMD::Datatype const e ) * the passed arguments and the template parameter type corresponding to the * openPMD type. */ - -#if _MSC_VER && !__INTEL_COMPILER -#define OPENPMD_TEMPLATE_OPERATOR operator -#else -#define OPENPMD_TEMPLATE_OPERATOR template operator -#endif template< typename ReturnType = void, typename Action, diff --git a/include/openPMD/IO/Format.hpp b/include/openPMD/IO/Format.hpp index 4d48e315da..3b80a85327 100644 --- a/include/openPMD/IO/Format.hpp +++ b/include/openPMD/IO/Format.hpp @@ -30,6 +30,7 @@ enum class Format HDF5, ADIOS1, ADIOS2, + JSON, DUMMY }; //Format } // openPMD diff --git a/include/openPMD/IO/JSON/JSONFilePosition.hpp b/include/openPMD/IO/JSON/JSONFilePosition.hpp new file mode 100644 index 0000000000..206ec34a10 --- /dev/null +++ b/include/openPMD/IO/JSON/JSONFilePosition.hpp @@ -0,0 +1,49 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + + +#include "openPMD/IO/AbstractFilePosition.hpp" + + +#if openPMD_HAVE_JSON +#include +#endif + +namespace openPMD +{ + + + struct JSONFilePosition : + public AbstractFilePosition +#if openPMD_HAVE_JSON + { + using json = nlohmann::json; + json::json_pointer id; + + JSONFilePosition( json::json_pointer ptr = json::json_pointer( ) ); + }; +#else + {}; +#endif + +} // openPMD diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp new file mode 100644 index 0000000000..a6d0ba3ffd --- /dev/null +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -0,0 +1,47 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" + + +namespace openPMD +{ + class JSONIOHandler : + public AbstractIOHandler + { + public: + JSONIOHandler( + std::string path, + AccessType at + ); + + virtual ~JSONIOHandler( ); + + std::future< void > flush( ) override; + + private: + JSONIOHandlerImpl m_impl; + }; +} // openPMD diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp new file mode 100644 index 0000000000..ef4e1a2154 --- /dev/null +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -0,0 +1,576 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + + +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/AccessType.hpp" +#include "openPMD/IO/JSON/JSONFilePosition.hpp" + +#include +#include +#include +#include +#include +#include + + +#if openPMD_HAVE_JSON + + +#include + + +#endif + + +namespace openPMD +{ + // Wrapper around a shared pointer to: + // * a filename + // * and a boolean indicating whether the file still exists + // The wrapper adds no extra information, but some commodity functions. + // Invariant for JSONIOHandlerImpl: + // For any valid filename, there is at any time at most one + // such shared pointer (wrapper) in the HandlerImpl's data structures + // (counting by pointer equality) + // This means, that a file can be invalidated (i.e. deleted or overwritten) + // by simply searching for one instance of the file e.g. in m_files and + // invalidating this instance + // A new instance may hence only be created after making sure that there are + // no valid instances in the data structures. + struct File + { + explicit File( std::string s ) : + fileState { std::make_shared< FileState >( s ) } + {} + + + File( ) = default; + + + struct FileState + { + explicit FileState( std::string s ) : + name { std::move( s ) } + {} + + + std::string name; + bool valid = true; + }; + + std::shared_ptr< FileState > fileState; + + + void invalidate( ) + { + fileState->valid = false; + } + + + bool valid( ) const + { + return fileState->valid; + } + + + File & operator=( std::string s ) + { + if( fileState ) + { + fileState->name = s; + } + else + { + fileState = std::make_shared< FileState >( s ); + } + return *this; + } + + + bool operator==( + File const & f + ) const + { + return this->fileState == f.fileState; + } + + + std::string & operator*( ) const + { + return fileState->name; + } + + + std::string * operator->( ) const + { + return &fileState->name; + } + + + explicit operator bool( ) const + { + return fileState.operator bool( ); + } + }; +} + +namespace std +{ + template< > + struct hash< openPMD::File > + { + typedef openPMD::File argument_type; + typedef std::size_t result_type; + + + result_type operator()( argument_type const & s ) const noexcept + { + return std::hash< shared_ptr< openPMD::File::FileState>> {}( s.fileState ); + } + }; +} + +namespace openPMD +{ +#if openPMD_HAVE_JSON + + class JSONIOHandlerImpl : + public AbstractIOHandlerImpl + { + using json = nlohmann::json; + + public: + explicit JSONIOHandlerImpl( AbstractIOHandler * ); + + ~JSONIOHandlerImpl( ) override; + + void createFile( + Writable *, + Parameter< Operation::CREATE_FILE > const & + ) override; + + void createPath( + Writable *, + Parameter< Operation::CREATE_PATH > const & + ) override; + + void createDataset( + Writable *, + Parameter< Operation::CREATE_DATASET > const & + ) override; + + void extendDataset( + Writable *, + Parameter< Operation::EXTEND_DATASET > const & + ) override; + + void openFile( + Writable *, + Parameter< Operation::OPEN_FILE > const & + ) override; + + void openPath( + Writable *, + Parameter< Operation::OPEN_PATH > const & + ) override; + + void openDataset( + Writable *, + Parameter< Operation::OPEN_DATASET > & + ) override; + + void deleteFile( + Writable *, + Parameter< Operation::DELETE_FILE > const & + ) override; + + void deletePath( + Writable *, + Parameter< Operation::DELETE_PATH > const & + ) override; + + void deleteDataset( + Writable *, + Parameter< Operation::DELETE_DATASET > const & + ) override; + + void deleteAttribute( + Writable *, + Parameter< Operation::DELETE_ATT > const & + ) override; + + void writeDataset( + Writable *, + Parameter< Operation::WRITE_DATASET > const & + ) override; + + void writeAttribute( + Writable *, + Parameter< Operation::WRITE_ATT > const & + ) override; + + void readDataset( + Writable *, + Parameter< Operation::READ_DATASET > & + ) override; + + void readAttribute( + Writable *, + Parameter< Operation::READ_ATT > & + ) override; + + void listPaths( + Writable *, + Parameter< Operation::LIST_PATHS > & + ) override; + + void listDatasets( + Writable *, + Parameter< Operation::LIST_DATASETS > & + ) override; + + void listAttributes( + Writable *, + Parameter< Operation::LIST_ATTS > & + ) override; + + std::future< void > flush( ) override; + + + private: + + using FILEHANDLE = std::fstream; + + // map each Writable to its associated file + // contains only the filename, without the OS path + std::unordered_map< + Writable *, + File + > m_files; + + std::unordered_map< + File, + std::shared_ptr< nlohmann::json >> m_jsonVals; + + // files that have logically, but not physically been written to + std::unordered_set< File > m_dirty; + + + // HELPER FUNCTIONS + + + // will use the IOHandler to retrieve the correct directory + // shared pointer to circumvent the fact that c++ pre 17 does + // not enforce (only allow) copy elision in return statements + std::shared_ptr< FILEHANDLE > getFilehandle( + File, + AccessType accessType + ); //, AccessType accessType=this->m_handler->accessType); + + // full operating system path of the given file + std::string fullPath( File ); + + std::string fullPath( std::string ); + + // from a path specification /a/b/c, remove the last + // "folder" (i.e. modify the string to equal /a/b) + static void parentDir( std::string & ); + + // Fileposition is assumed to have already been set, + // get it in string form + static std::string filepositionOf( Writable * w ); + + // Execute visitor on each pair of positions in the json value + // and the flattened multidimensional array. + // Used for writing from the data to JSON and for reading back into + // the array from JSON + template< + typename T, + typename Visitor + > + static void syncMultidimensionalJson( + nlohmann::json & j, + Offset const & offset, + Extent const & extent, + Extent const & multiplicator, + Visitor visitor, + T * data, + size_t currentdim = 0 + ); + + // multiplicators: an array [m_0,...,m_n] s.t. + // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n] + // (m_n = 1) + // essentially: m_i = \prod_{j=0}^{i-1} extent_j + static Extent getMultiplicators( Extent const & extent ); + + static nlohmann::json initializeNDArray( Extent const & extent ); + + static Extent getExtent( nlohmann::json & j ); + + + // remove single '/' in the beginning and end of a string + static std::string removeSlashes( std::string ); + + template< typename KeyT > + static bool hasKey( + nlohmann::json &, + KeyT && key + ); + + // make sure that the given path exists in proper form in + // the passed json value + static void ensurePath( + nlohmann::json * json, + std::string path + ); + + // In order not to insert the same file name into the data structures + // with a new pointer (e.g. when reopening), search for a possibly + // existing old pointer. Construct a new pointer only upon failure. + // The bool is true iff the pointer has been newly-created. + // The iterator is an iterator for m_files + std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + > getPossiblyExisting( + std::string file + ); + + // get the json value representing the whole file, possibly reading + // from disk + std::shared_ptr< nlohmann::json > obtainJsonContents( File ); + + // get the json value at the writable's fileposition + nlohmann::json & obtainJsonContents( Writable * writable ); + + // write to disk the json contents associated with the file + // remove from m_dirty if unsetDirty == true + void putJsonContents( + File, + bool unsetDirty = true + ); + + // figure out the file position of the writable + // (preferring the parent's file position) and extend it + // by extend. return the modified file position. + std::shared_ptr< JSONFilePosition > setAndGetFilePosition( + Writable *, + std::string extend + ); + + // figure out the file position of the writable + // (preferring the parent's file position) + // only modify the writable's fileposition when specified + std::shared_ptr< JSONFilePosition > setAndGetFilePosition( + Writable *, + bool write = true + ); + + // get the writable's containing file + // if the parent is associated with another file, + // associate the writable with that file and return it + File refreshFileFromParent( Writable * writable ); + + void associateWithFile( + Writable * writable, + File + ); + + // need to check the name too in order to exclude "attributes" key + static bool isGroup( nlohmann::json::const_iterator it ); + + static bool isDataset( nlohmann::json const & j ); + + + // check whether the json reference contains a valid dataset + template< typename Param > + void verifyDataset( + Param const & parameters, + nlohmann::json & + ); + + static nlohmann::json platformSpecifics( ); + + struct DatasetWriter + { + template< typename T > + void operator()( + nlohmann::json & json, + const Parameter< Operation::WRITE_DATASET > & parameters + ); + + template< int n > + void operator()( + nlohmann::json & json, + const Parameter< Operation::WRITE_DATASET > & parameters + ); + + }; + + struct DatasetReader + { + template< typename T > + void operator()( + nlohmann::json & json, + Parameter< Operation::READ_DATASET > & parameters + ); + + template< int n > + void operator()( + nlohmann::json & json, + Parameter< Operation::READ_DATASET > & parameters + ); + }; + + struct AttributeWriter + { + template< typename T > + void operator()( + nlohmann::json &, + Attribute::resource const & + ); + + template< int n > + void operator()( + nlohmann::json &, + Attribute::resource const & + ); + + }; + + struct AttributeReader + { + template< typename T > + void operator()( + nlohmann::json &, + Parameter< Operation::READ_ATT > & + ); + + template< int n > + void operator()( + nlohmann::json &, + Parameter< Operation::READ_ATT > & + ); + + }; + + template< typename T > + struct CppToJSON + { + nlohmann::json operator()( T const & ); + }; + + template< typename T > + struct CppToJSON< std::vector< T>> + { + nlohmann::json operator()( std::vector< T > const & ); + }; + + template< typename T, int n > + struct CppToJSON< + std::array< + T, + n>> + { + nlohmann::json operator()( + std::array< + T, + n + > const & + ); + }; + + template< + typename T, + typename Enable = T + > + struct JsonToCpp + { + T operator()( nlohmann::json const & ); + }; + + template< typename T > + struct JsonToCpp< std::vector< T > > + { + std::vector< T > operator()( nlohmann::json const & ); + }; + + template< typename T, int n > + struct JsonToCpp< + std::array< + T, + n + > + > + { + std::array< + T, + n + > operator()( nlohmann::json const & ); + }; + + template< typename T > + struct JsonToCpp< + T, + typename std::enable_if< + std::is_floating_point< + T + >::value + >::type + > + { + T operator()( nlohmann::json const & ); + }; + }; + +#else + + class JSONIOHandlerImpl + { + public: + JSONIOHandlerImpl( openPMD::AbstractIOHandler * ) + {}; + + + ~JSONIOHandlerImpl( ) + {}; + + + std::future< void > flush( ) + { + return std::future< void >( ); + } + }; + +#endif + + +} // openPMD diff --git a/include/openPMD/auxiliary/StringManip.hpp b/include/openPMD/auxiliary/StringManip.hpp index 78c7db6ac8..58f8cbf919 100644 --- a/include/openPMD/auxiliary/StringManip.hpp +++ b/include/openPMD/auxiliary/StringManip.hpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace openPMD @@ -109,11 +110,23 @@ replace_all(std::string s, std::string const& target, std::string const& replacement) { - std::string::size_type pos; - while( (pos = s.find(target)) != std::string::npos ) - s.replace(pos, target.size(), replacement); + std::string::size_type pos = 0; + auto tsize = target.size(); + assert(tsize > 0); + auto rsize = replacement.size(); + while (true) + { + pos = s.find(target, pos); + if (pos == std::string::npos) + break; + s.replace(pos, tsize, replacement); + // Allow replacing recursively, but only if + // the next replaced substring overlaps with + // some parts of the original word. + // This avoids loops. + pos += rsize - std::min(tsize - 1, rsize); + } s.shrink_to_fit(); - return s; } diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index af905d31d8..9c1b2627c9 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -69,6 +69,7 @@ class Writable friend class ADIOS2IOHandlerImpl; friend class HDF5IOHandlerImpl; friend class ParallelHDF5IOHandlerImpl; + friend class JSONIOHandlerImpl; friend struct test::TestHelper; friend std::string concrete_h5_file_position(Writable*); friend std::string concrete_bp1_file_position(Writable*); diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 926bafd448..ab358c2bfa 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -24,6 +24,7 @@ #include "openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp" #include "openPMD/IO/HDF5/HDF5IOHandler.hpp" #include "openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp" +#include "openPMD/IO/JSON/JSONIOHandler.hpp" namespace openPMD @@ -65,6 +66,8 @@ namespace openPMD return std::make_shared< ADIOS1IOHandler >(path, accessType); case Format::ADIOS2: throw std::runtime_error("ADIOS2 backend not yet implemented"); + case Format::JSON: + return std::make_shared< JSONIOHandler >(path, accessType); default: return std::make_shared< DummyIOHandler >(path, accessType); } diff --git a/src/IO/JSON/JSONFilePosition.cpp b/src/IO/JSON/JSONFilePosition.cpp new file mode 100644 index 0000000000..f5019fd260 --- /dev/null +++ b/src/IO/JSON/JSONFilePosition.cpp @@ -0,0 +1,11 @@ +#include "openPMD/IO/JSON/JSONFilePosition.hpp" + + +namespace openPMD { + +#if openPMD_HAVE_JSON + JSONFilePosition::JSONFilePosition( json::json_pointer ptr): + id( ptr ) + {} +#endif +} diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp new file mode 100644 index 0000000000..dde4b2df3b --- /dev/null +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -0,0 +1,47 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/IO/JSON/JSONIOHandler.hpp" + + +namespace openPMD +{ + JSONIOHandler::~JSONIOHandler( ) + {} + + + JSONIOHandler::JSONIOHandler( + std::string path, + AccessType at + ) : + AbstractIOHandler { + path, + at + }, + m_impl { JSONIOHandlerImpl { this } } + {} + + + std::future< void > JSONIOHandler::flush( ) + { + return m_impl.flush( ); + } +} // openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp new file mode 100644 index 0000000000..cb26694d26 --- /dev/null +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -0,0 +1,1562 @@ +/* Copyright 2017-2018 Franz Pöschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Memory.hpp" +#include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Writable.hpp" +#include "openPMD/Datatype.hpp" +#include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" + + +namespace openPMD +{ +#if openPMD_USE_VERIFY +# define VERIFY( CONDITION, TEXT ) { if(!(CONDITION)) throw std::runtime_error((TEXT)); } +#else +# define VERIFY( CONDITION, TEXT ) do{ (void)sizeof(CONDITION); } while( 0 ); +#endif + +#define VERIFY_ALWAYS( CONDITION, TEXT ) { if(!(CONDITION)) throw std::runtime_error((TEXT)); } + +#if openPMD_HAVE_JSON + + + JSONIOHandlerImpl::JSONIOHandlerImpl( AbstractIOHandler * handler ) : + AbstractIOHandlerImpl( handler ) + {} + + + JSONIOHandlerImpl::~JSONIOHandlerImpl( ) + { + flush( ); + } + + + std::future< void > JSONIOHandlerImpl::flush( ) + { + + AbstractIOHandlerImpl::flush( ); + for( auto const & file: m_dirty ) + { + putJsonContents( + file, + false + ); + } + m_dirty.clear( ); + return std::future< void >( ); + } + + + void JSONIOHandlerImpl::createFile( + Writable * writable, + Parameter< Operation::CREATE_FILE > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Creating a file in read-only mode is not possible." ); + + if( !writable->written ) + { + std::string name = parameters.name; + if( !auxiliary::ends_with( + name, + ".json" + ) ) + { + name += ".json"; + } + + auto res_pair = getPossiblyExisting( name ); + File shared_name = File( name ); + VERIFY_ALWAYS( !( m_handler->accessType == AccessType::READ_WRITE && + ( !std::get< 2 >( res_pair ) || + auxiliary::file_exists( fullPath( std::get< 0 >( res_pair ) ) ) ) ), + "Can only overwrite existing file in CREATE mode." ); + + if( !std::get< 2 >( res_pair ) ) + { + auto file = std::get< 0 >( res_pair ); + m_dirty.erase( file ); + m_jsonVals.erase( file ); + file.invalidate( ); + } + + std::string const dir( m_handler->directory ); + if( !auxiliary::directory_exists( dir ) ) + { + auto success = auxiliary::create_directories( dir ); + VERIFY( success, + "Could not create directory." ); + } + + associateWithFile( + writable, + shared_name + ); + this->m_dirty + .emplace( shared_name ); + // make sure to overwrite! + this->m_jsonVals[shared_name] = + std::make_shared< nlohmann::json >( ); + + + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( ); + } + } + + + void JSONIOHandlerImpl::createPath( + Writable * writable, + Parameter< Operation::CREATE_PATH > const & parameter + ) + { + std::string path = parameter.path; + /* Sanitize: + * The JSON API does not like to have slashes in the end. + */ + if( auxiliary::ends_with( + path, + "/" + ) ) + { + path = auxiliary::replace_last( + path, + "/", + "" + ); + } + + auto file = refreshFileFromParent( writable ); + + auto * jsonVal = &*obtainJsonContents( file ); + if( !auxiliary::starts_with( + path, + "/" + ) ) + { // path is relative + auto filepos = setAndGetFilePosition( + writable, + false + ); + + jsonVal = &( *jsonVal )[filepos->id]; + ensurePath( + jsonVal, + path + ); + path = + filepos->id + .to_string( ) + "/" + path; + } + else + { + + ensurePath( + jsonVal, + path + ); + } + + m_dirty.emplace( file ); + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( nlohmann::json::json_pointer( path ) ); + } + + + void JSONIOHandlerImpl::createDataset( + Writable * writable, + Parameter< Operation::CREATE_DATASET > const & parameter + ) + { + if( m_handler->accessType == AccessType::READ_ONLY ) + { + throw std::runtime_error( "Creating a dataset in a file opened as read only is not possible." ); + } + if( !writable->written ) + { + /* Sanitize name */ + std::string name = removeSlashes( parameter.name ); + + auto file = refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto & jsonVal = obtainJsonContents( writable ); + // be sure to have a JSON object, not a list + if( jsonVal.empty( ) ) + { + jsonVal = nlohmann::json::object( ); + } + setAndGetFilePosition( + writable, + name + ); + auto & dset = jsonVal[name]; + dset["datatype"] = datatypeToString( parameter.dtype ); + dset["data"] = initializeNDArray( parameter.extent ); + writable->written = true; + m_dirty.emplace( file ); + } + } + + + void JSONIOHandlerImpl::extendDataset( + Writable * writable, + Parameter< Operation::EXTEND_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot extend a dataset in read-only mode." ) + refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto name = removeSlashes( parameters.name ); + auto & j = obtainJsonContents( writable )[name]; + + try + { + auto datasetExtent = getExtent( j["data"] ); + VERIFY_ALWAYS( datasetExtent.size( ) == + parameters.extent + .size( ), + "Cannot change dimensionality of a dataset" ) + for( size_t currentdim = 0; + currentdim < + parameters.extent + .size( ); + currentdim++ ) + { + VERIFY_ALWAYS( datasetExtent[currentdim] <= + parameters.extent[currentdim], + "Cannot shrink the extent of a dataset" ) + } + } catch( json::basic_json::type_error & e ) + { + throw std::runtime_error( "The specified location contains no valid dataset" ); + } + j["data"] = initializeNDArray( parameters.extent ); + writable->written = true; + + } + + + void JSONIOHandlerImpl::openFile( + Writable * writable, + Parameter< Operation::OPEN_FILE > const & parameter + ) + { + if( !auxiliary::directory_exists( m_handler->directory ) ) + { + throw no_such_file_error( + "Supplied directory is not valid: " + m_handler->directory + ); + } + + std::string name = parameter.name; + if( !auxiliary::ends_with( + name, + ".json" + ) ) + { + name += ".json"; + } + + auto file = std::get< 0 >( getPossiblyExisting( name ) ); + + associateWithFile( + writable, + file + ); + + writable->written = true; + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( ); + } + + + void JSONIOHandlerImpl::openPath( + Writable * writable, + Parameter< Operation::OPEN_PATH > const & parameters + ) + { + auto file = refreshFileFromParent( writable ); + + nlohmann::json * j = &obtainJsonContents( writable->parent ); + auto path = removeSlashes( parameters.path ); + path = + path.empty( ) + ? filepositionOf( writable->parent ) + : filepositionOf( writable->parent ) + "/" + path; + + if( writable->abstractFilePosition ) + { + *setAndGetFilePosition( + writable, + false + ) = JSONFilePosition( json::json_pointer( path ) ); + } + else + { + writable->abstractFilePosition = + std::make_shared< JSONFilePosition >( json::json_pointer( path ) ); + } + + ensurePath( + j, + removeSlashes( parameters.path ) + ); + + writable->written = true; + } + + + void JSONIOHandlerImpl::openDataset( + Writable * writable, + Parameter< Operation::OPEN_DATASET > & parameters + ) + { + refreshFileFromParent( writable ); + auto name = removeSlashes( parameters.name ); + auto & datasetJson = obtainJsonContents( writable->parent )[name]; + setAndGetFilePosition( + writable, + name + ); + + *parameters.dtype = + Datatype( stringToDatatype( datasetJson["datatype"].get< std::string >( ) ) ); + *parameters.extent = getExtent( datasetJson["data"] ); + writable->written = true; + + } + + + void JSONIOHandlerImpl::deleteFile( + Writable * writable, + Parameter< Operation::DELETE_FILE > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete files in read-only mode" ) + + if( !writable->written ) + { + return; + } + + auto filename = auxiliary::ends_with( + parameters.name, + ".json" + ) ? parameters.name : parameters.name + ".json"; + + auto tuple = getPossiblyExisting( filename ); + if( !std::get< 2 >( tuple ) ) + { + // file is already in the system + auto file = std::get< 0 >( tuple ); + m_dirty.erase( file ); + m_jsonVals.erase( file ); + file.invalidate( ); + } + + std::remove( fullPath( filename ).c_str( ) ); + + writable->written = false; + } + + + void JSONIOHandlerImpl::deletePath( + Writable * writable, + Parameter< Operation::DELETE_PATH > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete paths in read-only mode" ) + + if( !writable->written ) + { + return; + } + + VERIFY_ALWAYS( !auxiliary::starts_with( + parameters.path, + '/' + ), + "Paths passed for deletion should be relative, the given path is absolute (starts with '/')" ) + auto file = refreshFileFromParent( writable ); + auto filepos = setAndGetFilePosition( + writable, + false + ); + auto path = removeSlashes( parameters.path ); + VERIFY( !path.empty( ), + "No path passed for deletion." ) + nlohmann::json * j; + if( path == "." ) + { + auto + s = + filepos->id + .to_string( ); + if( s == "/" ) + { + throw std::runtime_error( "Cannot delete the root group" ); + } + + auto i = s.rfind( '/' ); + path = s; + path.replace( + 0, + i + 1, + "" + ); + // path should now be equal to the name of the current group + // go up one group + + // go to parent directory + // parent exists since we have verified that the current + // directory is != root + parentDir( s ); + j = + &( *obtainJsonContents( file ) )[nlohmann::json::json_pointer( s )]; + } + else + { + if( auxiliary::starts_with( + path, + "./" + ) ) + { + path = auxiliary::replace_first( + path, + "./", + "" + ); + } + j = &obtainJsonContents( writable ); + } + nlohmann::json * lastPointer = j; + bool needToDelete = true; + auto splitPath = auxiliary::split( + path, + "/" + ); + // be careful not to create the group by accident + // the loop will execute at least once + for( auto folder: splitPath ) + { + auto it = j->find( folder ); + if( it == j->end( ) ) + { + needToDelete = false; + break; + } + else + { + lastPointer = j; + j = &it.value( ); + } + } + if( needToDelete ) + { + lastPointer->erase( + splitPath[splitPath.size( ) - 1] + ); + } + + putJsonContents( file ); + writable->abstractFilePosition + .reset( ); + writable->written = false; + } + + + void JSONIOHandlerImpl::deleteDataset( + Writable * writable, + Parameter< Operation::DELETE_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete datasets in read-only mode" ) + + if( !writable->written ) + { + return; + } + + auto filepos = setAndGetFilePosition( + writable, + false + ); + + auto file = refreshFileFromParent( writable ); + auto dataset = removeSlashes( parameters.name ); + nlohmann::json * parent; + if( dataset == "." ) + { + auto + s = + filepos->id + .to_string( ); + if( s.empty( ) ) + { + throw std::runtime_error( "Invalid position for a dataset in the JSON file." ); + } + dataset = s; + auto i = dataset.rfind( '/' ); + dataset.replace( + 0, + i + 1, + "" + ); + + parentDir( s ); + parent = + &( *obtainJsonContents( file ) )[nlohmann::json::json_pointer( s )]; + } + else + { + parent = &obtainJsonContents( writable ); + } + parent->erase( dataset ); + putJsonContents( file ); + writable->written = false; + writable->abstractFilePosition + .reset( ); + } + + + void JSONIOHandlerImpl::deleteAttribute( + Writable * writable, + Parameter< Operation::DELETE_ATT > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot delete attributes in read-only mode" ) + if( !writable->written ) + { + return; + } + setAndGetFilePosition( writable ); + auto file = refreshFileFromParent( writable ); + auto & j = obtainJsonContents( writable ); + j.erase( parameters.name ); + putJsonContents( file ); + } + + + void JSONIOHandlerImpl::writeDataset( + Writable * writable, + Parameter< Operation::WRITE_DATASET > const & parameters + ) + { + VERIFY_ALWAYS( m_handler->accessType != AccessType::READ_ONLY, + "Cannot write data in read-only mode." ); + + auto pos = setAndGetFilePosition( writable ); + auto file = refreshFileFromParent( writable ); + auto & j = obtainJsonContents( writable ); + + verifyDataset( + parameters, + j + ); + + + DatasetWriter dw; + switchType( + parameters.dtype, + dw, + j, + parameters + ); + + writable->written = true; + putJsonContents( file ); + } + + + void JSONIOHandlerImpl::writeAttribute( + Writable * writable, + Parameter< Operation::WRITE_ATT > const & parameter + ) + { + if( m_handler->accessType == AccessType::READ_ONLY ) + { + throw std::runtime_error( "Creating a dataset in a file opened as read only is not possible." ); + } + + /* Sanitize name */ + std::string name = removeSlashes( parameter.name ); + + auto file = refreshFileFromParent( writable ); + auto jsonVal = obtainJsonContents( file ); + auto filePosition = setAndGetFilePosition( writable ); + if( ( *jsonVal )[filePosition->id]["attributes"].empty( ) ) + { + ( *jsonVal )[filePosition->id]["attributes"] = + nlohmann::json::object( ); + } + nlohmann::json value; + AttributeWriter aw; + switchType( + parameter.dtype, + aw, + value, + parameter.resource + ); + ( *jsonVal )[filePosition->id]["attributes"][parameter.name] = { + { + "datatype", + datatypeToString( parameter.dtype ) + }, + { + "value", + value + } + }; + writable->written = true; + m_dirty.emplace( file ); + } + + + void JSONIOHandlerImpl::readDataset( + Writable * writable, + Parameter< Operation::READ_DATASET > & parameters + ) + { + refreshFileFromParent( writable ); + setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable ); + verifyDataset( + parameters, + j + ); + + try + { + DatasetReader dr; + switchType( + parameters.dtype, + dr, + j["data"], + parameters + ); + } catch( json::basic_json::type_error & ) + { + throw std::runtime_error( "The given path does not contain a valid dataset." ); + } + } + + + void JSONIOHandlerImpl::readAttribute( + Writable * writable, + Parameter< Operation::READ_ATT > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Attributes have to be written before reading." ) + refreshFileFromParent( writable ); + auto name = removeSlashes( parameters.name ); + auto & jsonLoc = obtainJsonContents( writable )["attributes"]; + setAndGetFilePosition( writable ); + VERIFY_ALWAYS( hasKey( + jsonLoc, + name + ), + "No such attribute in the given location." ) + auto & j = jsonLoc[name]; + try + { + *parameters.dtype = + Datatype( stringToDatatype( j["datatype"].get< std::string >( ) ) ); + AttributeReader ar; + switchType( + *parameters.dtype, + ar, + j["value"], + parameters + ); + } catch( json::type_error & ) + { + throw std::runtime_error( "The given location does not contain a properly formatted attribute" ); + } + } + + + void JSONIOHandlerImpl::listPaths( + Writable * writable, + Parameter< Operation::LIST_PATHS > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Values have to be written before reading a directory" ); + auto & j = obtainJsonContents( writable ); + setAndGetFilePosition( writable ); + refreshFileFromParent( writable ); + parameters.paths + ->clear( ); + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + if( isGroup( it ) ) + { + parameters.paths + ->push_back( it.key( ) ); + } + } + } + + + void JSONIOHandlerImpl::listDatasets( + Writable * writable, + Parameter< Operation::LIST_DATASETS > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Datasets have to be written before reading." ) + refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable ); + parameters.datasets + ->clear( ); + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + if( isDataset( it.value() ) ) + { + parameters.datasets + ->push_back( it.key( ) ); + } + } + } + + + void JSONIOHandlerImpl::listAttributes( + Writable * writable, + Parameter< Operation::LIST_ATTS > & parameters + ) + { + VERIFY_ALWAYS( writable->written, + "Attributes have to be written before reading." ) + refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( writable ); + auto & j = obtainJsonContents( writable )["attributes"]; + for( auto it = j.begin( ); it != j.end( ); it++ ) + { + parameters.attributes + ->push_back( it.key( ) ); + } + } + + + std::shared_ptr< JSONIOHandlerImpl::FILEHANDLE > + JSONIOHandlerImpl::getFilehandle( + File fileName, + AccessType accessType + ) + { + VERIFY_ALWAYS( fileName.valid( ), + "Tried opening a file that has been overwritten or deleted." ) + auto path = fullPath( std::move( fileName ) ); + auto fs = std::make_shared< std::fstream >( ); + switch( accessType ) + { + case AccessType::CREATE: + case AccessType::READ_WRITE: + fs->open( + path, + std::ios_base::out | std::ios_base::trunc + ); + break; + case AccessType::READ_ONLY: + fs->open( + path, + std::ios_base::in + ); + break; + } + VERIFY( fs->good( ), + "Failed opening a file" ); + return fs; + } + + + std::string JSONIOHandlerImpl::fullPath( File fileName ) + { + return fullPath( *fileName ); + } + + + std::string JSONIOHandlerImpl::fullPath( std::string fileName ) + { + if( auxiliary::ends_with( + m_handler->directory, + "/" + ) ) + { + return m_handler->directory + fileName; + } + else + { + return m_handler->directory + "/" + fileName; + } + } + + + void JSONIOHandlerImpl::parentDir( std::string & s ) + { + auto i = s.rfind( '/' ); + if( i != std::string::npos ) + { + s.replace( + i, + s.size( ) - i, + "" + ); + s.shrink_to_fit( ); + } + } + + + std::string JSONIOHandlerImpl::filepositionOf( Writable * writable ) + { + return std::dynamic_pointer_cast< JSONFilePosition >( writable->abstractFilePosition )->id + .to_string( ); + } + + + template< + typename T, + typename Visitor + > + void JSONIOHandlerImpl::syncMultidimensionalJson( + nlohmann::json & j, + Offset const & offset, + Extent const & extent, + Extent const & multiplicator, + Visitor visitor, + T * data, + size_t currentdim + ) + { + // Offset only relevant for JSON, the array data is contiguous + auto off = offset[currentdim]; + // maybe rewrite iteratively, using a stack that stores for each level the + // current iteration value i + + if( currentdim == offset.size( ) - 1 ) + { + for( std::size_t i = 0; i < extent[currentdim]; ++i ) + { + visitor( + j[i + off], + data[i] + ); + } + } + else + { + for( std::size_t i = 0; i < extent[currentdim]; ++i ) + { + syncMultidimensionalJson< + T, + Visitor + >( + j[i + off], + offset, + extent, + multiplicator, + visitor, + data + i * multiplicator[currentdim], + currentdim + 1 + ); + } + } + } + + + // multiplicators: an array [m_0,...,m_n] s.t. + // data[i_0]...[i_n] = data[m_0*i_0+...+m_n*i_n] + // (m_n = 1) + Extent JSONIOHandlerImpl::getMultiplicators( Extent const & extent ) + { + Extent res( extent ); + Extent::value_type n = 1; + size_t i = extent.size( ); + do + { + --i; + res[i] = n; + n *= extent[i]; + } + while( i > 0 ); + return res; + } + + + nlohmann::json JSONIOHandlerImpl::initializeNDArray( Extent const & extent ) + { + // idea: begin from the innermost shale and copy the result into the + // outer shales + nlohmann::json accum; + nlohmann::json old; + auto * accum_ptr = & accum; + auto * old_ptr = & old; + for( auto it = extent.rbegin( ); it != extent.rend( ); it++ ) + { + std::swap(old_ptr, accum_ptr); + *accum_ptr = nlohmann::json {}; + for( Extent::value_type i = 0; i < *it; i++ ) + { + (*accum_ptr)[i] = *old_ptr; // copy boi + } + } + return *accum_ptr; + } + + + Extent JSONIOHandlerImpl::getExtent( nlohmann::json & j ) + { + Extent res; + nlohmann::json * ptr = &j; + while( ptr->is_array( ) ) + { + res.push_back( ptr->size( ) ); + ptr = &( *ptr )[0]; + } + return res; + } + + + std::string JSONIOHandlerImpl::removeSlashes( std::string s ) + { + if( auxiliary::starts_with( + s, + '/' + ) ) + { + s = auxiliary::replace_first( + s, + "/", + "" + ); + } + if( auxiliary::ends_with( + s, + '/' + ) ) + { + s = auxiliary::replace_last( + s, + "/", + "" + ); + } + return s; + } + + + template< typename KeyT > + bool JSONIOHandlerImpl::hasKey( + nlohmann::json & j, + KeyT && key + ) + { + return j.find( std::forward< KeyT >( key ) ) != j.end( ); + } + + + void JSONIOHandlerImpl::ensurePath( + nlohmann::json * jsonp, + std::string path + ) + { + auto groups = auxiliary::split( + path, + "/" + ); + for( std::string & group: groups ) + { + // Enforce a JSON object + // the library will automatically create a list if the first + // key added to it is parseable as an int + jsonp = &( *jsonp )[group]; + if (jsonp->is_null()) + { + *jsonp = nlohmann::json::object(); + } + } + } + + + std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + > JSONIOHandlerImpl::getPossiblyExisting( std::string file ) + { + + auto it = std::find_if( + m_files.begin( ), + m_files.end( ), + [file]( + std::unordered_map< + Writable *, + File + >::value_type const & entry + ) + { + return *entry.second == file && + entry.second + .valid( ); + } + ); + + bool newlyCreated; + File name; + if( it == m_files.end( ) ) + { + name = file; + newlyCreated = true; + } + else + { + name = it->second; + newlyCreated = false; + } + return std::tuple< + File, + std::unordered_map< + Writable *, + File + >::iterator, + bool + >( + std::move( name ), + it, + newlyCreated + ); + } + + + std::shared_ptr< nlohmann::json > + JSONIOHandlerImpl::obtainJsonContents( File file ) + { + VERIFY_ALWAYS( file.valid( ), + "File has been overwritten or deleted before reading" ); + auto it = m_jsonVals.find( file ); + if( it != m_jsonVals.end( ) ) + { + return it->second; + } + // read from file + auto fh = getFilehandle( + file, + AccessType::READ_ONLY + ); + std::shared_ptr< nlohmann::json > + res = std::make_shared< nlohmann::json >( ); + *fh >> *res; + VERIFY( fh->good( ), + "Failed reading from a file." ); + m_jsonVals.emplace( + file, + res + ); + return res; + } + + + nlohmann::json & + JSONIOHandlerImpl::obtainJsonContents( Writable * writable ) + { + auto file = refreshFileFromParent( writable ); + auto filePosition = setAndGetFilePosition( + writable, + false + ); + return ( *obtainJsonContents( file ) )[filePosition->id]; + } + + + void JSONIOHandlerImpl::putJsonContents( + File filename, + bool unsetDirty // = true + ) + { + VERIFY_ALWAYS( filename.valid( ), + "File has been overwritten/deleted before writing" ); + auto it = m_jsonVals.find( filename ); + if( it != m_jsonVals.end( ) ) + { + auto fh = getFilehandle( + filename, + AccessType::CREATE + ); + ( *it->second )["platform_byte_widths"] = platformSpecifics( ); + *fh << *it->second << std::endl; + VERIFY( fh->good( ), + "Failed writing data to disk." ) + m_jsonVals.erase( it ); + if( unsetDirty ) + { + m_dirty.erase( filename ); + } + } + + } + + + std::shared_ptr< JSONFilePosition > + JSONIOHandlerImpl::setAndGetFilePosition( + Writable * writable, + std::string extend + ) + { + std::string path; + if( writable->abstractFilePosition ) + { + // do NOT reuse the old pointer, we want to change the file position + // only for the writable! + path = filepositionOf( writable ) + "/" + extend; + } + else if( writable->parent ) + { + path = filepositionOf( writable->parent ) + "/" + extend; + } + else + { // we are root + path = extend; + if( !auxiliary::starts_with( + path, + "/" + ) ) + { + path = "/" + path; + } + } + auto + res = + std::make_shared< JSONFilePosition >( json::json_pointer( path ) ); + + writable->abstractFilePosition = res; + + return res; + } + + + std::shared_ptr< JSONFilePosition > + JSONIOHandlerImpl::setAndGetFilePosition( + Writable * writable, + bool write + ) + { + std::shared_ptr< AbstractFilePosition > res; + + + if( writable->abstractFilePosition ) + { + res = writable->abstractFilePosition; + } + else if( writable->parent ) + { + res = + writable->parent + ->abstractFilePosition; + } + else + { // we are root + res = std::make_shared< JSONFilePosition >( ); + } + if( write ) + { + writable->abstractFilePosition = res; + } + return std::dynamic_pointer_cast< JSONFilePosition >( res ); + } + + + File JSONIOHandlerImpl::refreshFileFromParent( Writable * writable ) + { + if( writable->parent ) + { + auto + file = + m_files.find( writable->parent ) + ->second; + associateWithFile( + writable, + file + ); + return file; + } + else + { + return m_files.find( writable ) + ->second; + } + } + + + void JSONIOHandlerImpl::associateWithFile( + Writable * writable, + File file + ) + { + // make sure to overwrite + m_files[writable] = std::move( file ); + } + + + bool JSONIOHandlerImpl::isDataset( nlohmann::json const & j ) + { + if( !j.is_object( ) ) + { + return false; + } + auto i = j.find( "data" ); + return i != j.end( ) && i.value( ).is_array(); + } + + + bool JSONIOHandlerImpl::isGroup( nlohmann::json::const_iterator it ) + { + auto & j = it.value(); + if( it.key() == "attributes" || it.key() == "platform_byte_widths" || !j.is_object( ) ) + { + return false; + } + auto i = j.find( "data" ); + return i == j.end( ) || !i.value( ).is_array(); + } + + + template< typename Param > + void JSONIOHandlerImpl::verifyDataset( + Param const & parameters, + nlohmann::json & j + ) + { + VERIFY_ALWAYS( isDataset(j), + "Specified dataset does not exist or is not a dataset." ); + + try + { + auto datasetExtent = getExtent( j["data"] ); + VERIFY_ALWAYS( datasetExtent.size( ) == + parameters.extent + .size( ), + "Read/Write request does not fit the dataset's dimension" ); + for( unsigned int dimension = 0; + dimension < + parameters.extent + .size( ); + dimension++ ) + { + VERIFY_ALWAYS( parameters.offset[dimension] + + parameters.extent[dimension] <= + datasetExtent[dimension], + "Read/Write request exceeds the dataset's size" ); + } + Datatype + dt = stringToDatatype( j["datatype"].get< std::string >( ) ); + VERIFY_ALWAYS( dt == parameters.dtype, + "Read/Write request does not fit the dataset's type" ); + } catch( json::basic_json::type_error & e ) + { + throw std::runtime_error( "The given path does not contain a valid dataset." ); + } + } + + + nlohmann::json JSONIOHandlerImpl::platformSpecifics( ) + { + nlohmann::json res; + static Datatype datatypes[] = { + Datatype::CHAR, + Datatype::UCHAR, + Datatype::SHORT, + Datatype::INT, + Datatype::LONG, + Datatype::LONGLONG, + Datatype::USHORT, + Datatype::UINT, + Datatype::ULONG, + Datatype::ULONGLONG, + Datatype::FLOAT, + Datatype::DOUBLE, + Datatype::LONG_DOUBLE, + Datatype::BOOL + }; + for( auto it = std::begin( datatypes ); + it != std::end( datatypes ); + it++ ) + { + res[datatypeToString( *it )] = toBytes( *it ); + } + return res; + } + + + template< typename T > + void JSONIOHandlerImpl::DatasetWriter::operator()( + nlohmann::json & json, + const Parameter< Operation::WRITE_DATASET > & parameters + ) + { + CppToJSON< T > ctj; + syncMultidimensionalJson( + json["data"], + parameters.offset, + parameters.extent, + getMultiplicators( parameters.extent ), + [&ctj]( + nlohmann::json & j, + T const & data + ) + { + j = ctj( data ); + }, + static_cast(parameters.data + .get( )) + ); + } + + + template< int n > + void JSONIOHandlerImpl::DatasetWriter::operator()( + nlohmann::json &, + const Parameter< Operation::WRITE_DATASET > & + ) + { + throw std::runtime_error( "Unknown datatype given for writing." ); + } + + + template< typename T > + void JSONIOHandlerImpl::DatasetReader::operator()( + nlohmann::json & json, + Parameter< Operation::READ_DATASET > & parameters + ) + { + JsonToCpp< + T + > jtc; + syncMultidimensionalJson( + json, + parameters.offset, + parameters.extent, + getMultiplicators( parameters.extent ), + [&jtc]( + nlohmann::json & j, + T & data + ) + { + data = jtc( j ); + }, + static_cast(parameters.data + .get( )) + ); + } + + + template< int n > + void JSONIOHandlerImpl::DatasetReader::operator()( + nlohmann::json &, + Parameter< Operation::READ_DATASET > & + ) + { + throw std::runtime_error( "Unknown datatype while reading a dataset." ); + } + + + template< typename T > + void JSONIOHandlerImpl::AttributeWriter::operator()( + nlohmann::json & value, + Attribute::resource const & resource + ) + { + CppToJSON< T > ctj; + value = ctj( variantSrc::get< T >( resource ) ); + } + + + template< int n > + void JSONIOHandlerImpl::AttributeWriter::operator()( + nlohmann::json &, + Attribute::resource const & + ) + { + throw std::runtime_error( "Unknown datatype in attribute writing." ); + } + + + template< int n > + void JSONIOHandlerImpl::AttributeReader::operator()( + nlohmann::json &, + Parameter< Operation::READ_ATT > & + ) + { + throw std::runtime_error( "Unknown datatype while reading attribute." ); + } + + + template< typename T > + void JSONIOHandlerImpl::AttributeReader::operator()( + nlohmann::json & json, + Parameter< Operation::READ_ATT > & parameters + ) + { + JsonToCpp< + T + > jtc; + *parameters.resource = jtc( + json + ); + } + + + template< typename T > + nlohmann::json + JSONIOHandlerImpl::CppToJSON< T >::operator()( const T & val ) + { + return nlohmann::json( val ); + } + + + template< typename T > + nlohmann::json + JSONIOHandlerImpl::CppToJSON< std::vector< T > >::operator()( const std::vector< T > & v ) + { + nlohmann::json j; + CppToJSON< T > ctj; + for( auto a: v ) + { + j.push_back( ctj( a ) ); + } + return j; + } + + + template< typename T, int n > + nlohmann::json JSONIOHandlerImpl::CppToJSON< + std::array< + T, + n + > + >::operator()( + const std::array< + T, + n + > & v + ) + { + nlohmann::json j; + CppToJSON< T > ctj; + for( auto a: v ) + { + j.push_back( ctj( a ) ); + } + return j; + } + + + template< + typename T, + typename Dummy + > + T JSONIOHandlerImpl::JsonToCpp< + T, + Dummy + >::operator()( nlohmann::json const & json ) + { return json.get< T >( ); } + + + template< typename T > + std::vector< T > + JSONIOHandlerImpl::JsonToCpp< std::vector< T > >::operator()( nlohmann::json const & json ) + { + std::vector< T > v; + JsonToCpp< T > jtp; + for( auto & j: json ) + { + v.push_back( jtp( j ) ); + } + return v; + } + + + template< typename T, int n > + std::array< + T, + n + > JSONIOHandlerImpl::JsonToCpp< + std::array< + T, + n + > + >::operator()( nlohmann::json const & json ) + { + std::array< + T, + n + > a; + JsonToCpp< T > jtp; + size_t i = 0; + for( auto & j: json ) + { + a[i] = jtp( j ); + i++; + } + return a; + } + + + template< + typename T + > + T JSONIOHandlerImpl::JsonToCpp< + T, + typename std::enable_if< + std::is_floating_point< + T + >::value + >::type + >::operator()( nlohmann::json const & j ) + { + try + { + return j.get< T >( ); + } catch( ... ) + { + return std::numeric_limits< T >::quiet_NaN( ); + } + } + + +#endif + + +} // openPMD diff --git a/src/Series.cpp b/src/Series.cpp index b84a2560cf..161a5508a8 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -862,6 +862,8 @@ determineFormat(std::string const& filename) return Format::HDF5; if( auxiliary::ends_with(filename, ".bp") ) return Format::ADIOS1; + if( auxiliary::ends_with(filename, ".json") ) + return Format::JSON; if( std::string::npos != filename.find('.') /* extension is provided */ ) throw std::runtime_error("Unknown file format. Did you append a valid filename extension?"); @@ -879,6 +881,8 @@ suffix(Format f) case Format::ADIOS1: case Format::ADIOS2: return ".bp"; + case Format::JSON: + return ".json"; default: return ""; } @@ -892,6 +896,7 @@ cleanFilename(std::string const& filename, Format f) case Format::HDF5: case Format::ADIOS1: case Format::ADIOS2: + case Format::JSON: return auxiliary::replace_last(filename, suffix(f), ""); default: return filename; @@ -952,6 +957,16 @@ matcher(std::string const& prefix, int padding, std::string const& postfix, Form nameReg += + ")" + postfix + ".bp$"; return buildMatcher(nameReg); } + case Format::JSON: + { + std::string nameReg = "^" + prefix + "([[:digit:]]"; + if( padding != 0 ) + nameReg += "{" + std::to_string(padding) + "}"; + else + nameReg += "+"; + nameReg += + ")" + postfix + ".json$"; + return buildMatcher(nameReg); + } default: return [](std::string const&) -> std::tuple< bool, int > { return std::tuple< bool, int >{false, 0}; }; } diff --git a/src/auxiliary/Filesystem.cpp b/src/auxiliary/Filesystem.cpp index 39c5500293..20405be98c 100644 --- a/src/auxiliary/Filesystem.cpp +++ b/src/auxiliary/Filesystem.cpp @@ -160,6 +160,7 @@ remove_file( std::string const& path ) #else return (0 == remove(path.c_str())); #endif -} +} + } // auxiliary } // openPMD diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index fe2c98fba2..64e73dac15 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -19,6 +19,23 @@ using namespace openPMD; +std::vector> getBackends() { + // first component: backend file ending + // second component: whether to test 128 bit values + std::vector> res; +#if openPMD_HAVE_ADIOS1 + res.emplace_back("bp", true); +#endif +#if openPMD_HAVE_HDF5 + res.emplace_back("h5", true); +#endif +#if openPMD_HAVE_JSON + res.emplace_back("json", false); +#endif + return res; +} + +auto const backends = getBackends(); inline void constant_scalar(std::string file_ending) @@ -110,6 +127,15 @@ void constant_scalar(std::string file_ending) } } +TEST_CASE( "constant_scalar", "[serial]" ) +{ + for (auto const & t: backends) + { + constant_scalar(std::get<0>(t)); + } +} + + inline void particle_patches( std::string file_ending ) { @@ -192,1932 +218,1721 @@ void particle_patches( std::string file_ending ) } } -#if openPMD_HAVE_HDF5 -TEST_CASE( "git_hdf5_sample_structure_test", "[serial][hdf5]" ) +TEST_CASE( "particle_patches", "[serial]" ) { -#if openPMD_USE_INVASIVE_TESTS - try + for (auto const & t: backends) { - Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); + particle_patches(std::get<0>(t)); + } +} - REQUIRE(!o.parent); - REQUIRE(o.iterations.parent == getWritable(&o)); - REQUIRE_THROWS_AS(o.iterations[42], std::out_of_range); - REQUIRE(o.iterations[100].parent == getWritable(&o.iterations)); - REQUIRE(o.iterations[100].meshes.parent == getWritable(&o.iterations[100])); - REQUIRE(o.iterations[100].meshes["E"].parent == getWritable(&o.iterations[100].meshes)); - REQUIRE(o.iterations[100].meshes["E"]["x"].parent == getWritable(&o.iterations[100].meshes["E"])); - REQUIRE(o.iterations[100].meshes["E"]["y"].parent == getWritable(&o.iterations[100].meshes["E"])); - REQUIRE(o.iterations[100].meshes["E"]["z"].parent == getWritable(&o.iterations[100].meshes["E"])); - REQUIRE(o.iterations[100].meshes["rho"].parent == getWritable(&o.iterations[100].meshes)); - REQUIRE(o.iterations[100].meshes["rho"][MeshRecordComponent::SCALAR].parent == getWritable(&o.iterations[100].meshes)); - REQUIRE_THROWS_AS(o.iterations[100].meshes["cherries"], std::out_of_range); - REQUIRE(o.iterations[100].particles.parent == getWritable(&o.iterations[100])); - REQUIRE(o.iterations[100].particles["electrons"].parent == getWritable(&o.iterations[100].particles)); - REQUIRE(o.iterations[100].particles["electrons"]["charge"].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["charge"][RecordComponent::SCALAR].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["mass"].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["mass"][RecordComponent::SCALAR].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["momentum"].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["momentum"]["x"].parent == getWritable(&o.iterations[100].particles["electrons"]["momentum"])); - REQUIRE(o.iterations[100].particles["electrons"]["momentum"]["y"].parent == getWritable(&o.iterations[100].particles["electrons"]["momentum"])); - REQUIRE(o.iterations[100].particles["electrons"]["momentum"]["z"].parent == getWritable(&o.iterations[100].particles["electrons"]["momentum"])); - REQUIRE(o.iterations[100].particles["electrons"]["position"].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["position"]["x"].parent == getWritable(&o.iterations[100].particles["electrons"]["position"])); - REQUIRE(o.iterations[100].particles["electrons"]["position"]["y"].parent == getWritable(&o.iterations[100].particles["electrons"]["position"])); - REQUIRE(o.iterations[100].particles["electrons"]["position"]["z"].parent == getWritable(&o.iterations[100].particles["electrons"]["position"])); - REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"]["x"].parent == getWritable(&o.iterations[100].particles["electrons"]["positionOffset"])); - REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"]["y"].parent == getWritable(&o.iterations[100].particles["electrons"]["positionOffset"])); - REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"]["z"].parent == getWritable(&o.iterations[100].particles["electrons"]["positionOffset"])); - REQUIRE(o.iterations[100].particles["electrons"]["weighting"].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE(o.iterations[100].particles["electrons"]["weighting"][RecordComponent::SCALAR].parent == getWritable(&o.iterations[100].particles["electrons"])); - REQUIRE_THROWS_AS(o.iterations[100].particles["electrons"]["numberOfLegs"], std::out_of_range); - REQUIRE_THROWS_AS(o.iterations[100].particles["apples"], std::out_of_range); +inline +void dtype_test(const std::string & backend, bool test_128_bit = true) +{ + bool test_long_double = test_128_bit || sizeof (long double) <= 8; + bool test_long_long = test_128_bit || sizeof (long long) <= 8; + { + Series s = Series("../samples/dtype_test." + backend, AccessType::CREATE); + char c = 'c'; + s.setAttribute("char", c); + unsigned char uc = 'u'; + s.setAttribute("uchar", uc); + int16_t i16 = 16; + s.setAttribute("int16", i16); int32_t i32 = 32; - REQUIRE_THROWS(o.setAttribute("setAttributeFail", i32)); - } catch (no_such_file_error& e) + s.setAttribute("int32", i32); + int64_t i64 = 64; + s.setAttribute("int64", i64); + uint16_t u16 = 16u; + s.setAttribute("uint16", u16); + uint32_t u32 = 32u; + s.setAttribute("uint32", u32); + uint64_t u64 = 64u; + s.setAttribute("uint64", u64); + float f = 16.e10f; + s.setAttribute("float", f); + double d = 1.e64; + s.setAttribute("double", d); + if (test_long_double) + { + long double ld = 1.e80L; + s.setAttribute("longdouble", ld); + } + std::string str = "string"; + s.setAttribute("string", str); + s.setAttribute("vecChar", std::vector< char >({'c', 'h', 'a', 'r'})); + s.setAttribute("vecInt16", std::vector< int16_t >({32766, 32767})); + s.setAttribute("vecInt32", std::vector< int32_t >({2147483646, 2147483647})); + s.setAttribute("vecInt64", std::vector< int64_t >({9223372036854775806, 9223372036854775807})); + s.setAttribute("vecUchar", std::vector< char >({'u', 'c', 'h', 'a', 'r'})); + s.setAttribute("vecUint16", std::vector< uint16_t >({65534u, 65535u})); + s.setAttribute("vecUint32", std::vector< uint32_t >({4294967294u, 4294967295u})); + s.setAttribute("vecUint64", std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); + s.setAttribute("vecFloat", std::vector< float >({0.f, 3.40282e+38f})); + s.setAttribute("vecDouble", std::vector< double >({0., 1.79769e+308})); + if (test_long_double) + { + s.setAttribute("vecLongdouble", std::vector< long double >({0.L, std::numeric_limits::max()})); + } + s.setAttribute("vecString", std::vector< std::string >({"vector", "of", "strings"})); + s.setAttribute("bool", true); + s.setAttribute("boolF", false); + + // non-fixed size integer types + short ss = 16; + s.setAttribute("short", ss); + int si = 32; + s.setAttribute("int", si); + long sl = 64; + s.setAttribute("long", sl); + if (test_long_long) + { + long long sll = 128; + s.setAttribute("longlong", sll); + } + unsigned short us = 16u; + s.setAttribute("ushort", us); + unsigned int ui = 32u; + s.setAttribute("uint", ui); + unsigned long ul = 64u; + s.setAttribute("ulong", ul); + if (test_long_long) + { + unsigned long long ull = 128u; + s.setAttribute("ulonglong", ull); + } + s.setAttribute("vecShort", std::vector< short >({32766, 32767})); + s.setAttribute("vecInt", std::vector< int >({32766, 32767})); + s.setAttribute("vecLong", std::vector< long >({2147483646, 2147483647})); + if (test_long_long) + { + s.setAttribute("vecLongLong", std::vector< long long >({2147483644, 2147483643})); + } + s.setAttribute("vecUShort", std::vector< unsigned short >({65534u, 65535u})); + s.setAttribute("vecUInt", std::vector< unsigned int >({65533u, 65531u})); + s.setAttribute("vecULong", std::vector< unsigned long >({65532u, 65530u})); + if (test_long_long) + { + s.setAttribute("vecULongLong", std::vector< unsigned long long >({65531u, 65529u})); + } + } + + Series s = Series("../samples/dtype_test." + backend, AccessType::READ_ONLY); + + REQUIRE(s.getAttribute("char").get< char >() == 'c'); + REQUIRE(s.getAttribute("uchar").get< unsigned char >() == 'u'); + REQUIRE(s.getAttribute("int16").get< int16_t >() == 16); + REQUIRE(s.getAttribute("int32").get< int32_t >() == 32); + REQUIRE(s.getAttribute("int64").get< int64_t >() == 64); + REQUIRE(s.getAttribute("uint16").get< uint16_t >() == 16u); + REQUIRE(s.getAttribute("uint32").get< uint32_t >() == 32u); + REQUIRE(s.getAttribute("uint64").get< uint64_t >() == 64u); + REQUIRE(s.getAttribute("float").get< float >() == 16.e10f); + REQUIRE(s.getAttribute("double").get< double >() == 1.e64); + if (test_long_double) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + REQUIRE(s.getAttribute("longdouble").get< long double >() == 1.e80L); + } + REQUIRE(s.getAttribute("string").get< std::string >() == "string"); + REQUIRE(s.getAttribute("vecChar").get< std::vector< char > >() == std::vector< char >({'c', 'h', 'a', 'r'})); + REQUIRE(s.getAttribute("vecInt16").get< std::vector< int16_t > >() == std::vector< int16_t >({32766, 32767})); + REQUIRE(s.getAttribute("vecInt32").get< std::vector< int32_t > >() == std::vector< int32_t >({2147483646, 2147483647})); + REQUIRE(s.getAttribute("vecInt64").get< std::vector< int64_t > >() == std::vector< int64_t >({9223372036854775806, 9223372036854775807})); + REQUIRE(s.getAttribute("vecUchar").get< std::vector< char > >() == std::vector< char >({'u', 'c', 'h', 'a', 'r'})); + REQUIRE(s.getAttribute("vecUint16").get< std::vector< uint16_t > >() == std::vector< uint16_t >({65534u, 65535u})); + REQUIRE(s.getAttribute("vecUint32").get< std::vector< uint32_t > >() == std::vector< uint32_t >({4294967294u, 4294967295u})); + REQUIRE(s.getAttribute("vecUint64").get< std::vector< uint64_t > >() == std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); + REQUIRE(s.getAttribute("vecFloat").get< std::vector< float > >() == std::vector< float >({0.f, 3.40282e+38f})); + REQUIRE(s.getAttribute("vecDouble").get< std::vector< double > >() == std::vector< double >({0., 1.79769e+308})); + if (test_long_double) + { + REQUIRE(s.getAttribute("vecLongdouble").get< std::vector< long double > >() == std::vector< long double >({0.L, std::numeric_limits::max()})); + } + REQUIRE(s.getAttribute("vecString").get< std::vector< std::string > >() == std::vector< std::string >({"vector", "of", "strings"})); + REQUIRE(s.getAttribute("bool").get< bool >() == true); + REQUIRE(s.getAttribute("boolF").get< bool >() == false); + + // same implementation types (not necessary aliases) detection +#if !defined(_MSC_VER) + REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); + REQUIRE(s.getAttribute("int").dtype == Datatype::INT); + REQUIRE(s.getAttribute("long").dtype == Datatype::LONG); + REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); + REQUIRE(s.getAttribute("ushort").dtype == Datatype::USHORT); + REQUIRE(s.getAttribute("uint").dtype == Datatype::UINT); + REQUIRE(s.getAttribute("ulong").dtype == Datatype::ULONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("ulonglong").dtype == Datatype::ULONGLONG); + } + + REQUIRE(s.getAttribute("vecShort").dtype == Datatype::VEC_SHORT); + REQUIRE(s.getAttribute("vecInt").dtype == Datatype::VEC_INT); + REQUIRE(s.getAttribute("vecLong").dtype == Datatype::VEC_LONG); + REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); + REQUIRE(s.getAttribute("vecUShort").dtype == Datatype::VEC_USHORT); + REQUIRE(s.getAttribute("vecUInt").dtype == Datatype::VEC_UINT); + REQUIRE(s.getAttribute("vecULong").dtype == Datatype::VEC_ULONG); + if (test_long_long) + { + REQUIRE(s.getAttribute("vecULongLong").dtype == Datatype::VEC_ULONGLONG); } -#else - std::cerr << "Invasive tests not enabled. Hierarchy is not visible.\n"; #endif + REQUIRE(isSame(s.getAttribute("short").dtype, Datatype::SHORT)); + REQUIRE(isSame(s.getAttribute("int").dtype, Datatype::INT)); + REQUIRE(isSame(s.getAttribute("long").dtype, Datatype::LONG)); + if (test_long_long) + { + REQUIRE(isSame(s.getAttribute("longlong").dtype, Datatype::LONGLONG)); + } + REQUIRE(isSame(s.getAttribute("ushort").dtype, Datatype::USHORT)); + REQUIRE(isSame(s.getAttribute("uint").dtype, Datatype::UINT)); + REQUIRE(isSame(s.getAttribute("ulong").dtype, Datatype::ULONG)); + if (test_long_long) + { + REQUIRE(isSame(s.getAttribute("ulonglong").dtype, Datatype::ULONGLONG)); + } + + REQUIRE(isSame(s.getAttribute("vecShort").dtype, Datatype::VEC_SHORT)); + REQUIRE(isSame(s.getAttribute("vecInt").dtype, Datatype::VEC_INT)); + REQUIRE(isSame(s.getAttribute("vecLong").dtype, Datatype::VEC_LONG)); + if (test_long_long) + { + REQUIRE(isSame(s.getAttribute("vecLongLong").dtype, Datatype::VEC_LONGLONG)); + } + REQUIRE(isSame(s.getAttribute("vecUShort").dtype, Datatype::VEC_USHORT)); + REQUIRE(isSame(s.getAttribute("vecUInt").dtype, Datatype::VEC_UINT)); + REQUIRE(isSame(s.getAttribute("vecULong").dtype, Datatype::VEC_ULONG)); + if (test_long_long) + { + REQUIRE(isSame(s.getAttribute("vecULongLong").dtype, Datatype::VEC_ULONGLONG)); + } } -TEST_CASE( "git_hdf5_sample_attribute_test", "[serial][hdf5]" ) +TEST_CASE( "dtype_test", "[serial]" ) { - try + std::string backend; + bool test_128_bit; + for (auto const & t: backends) { - Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); - - REQUIRE(o.openPMD() == "1.1.0"); - REQUIRE(o.openPMDextension() == 1); - REQUIRE(o.basePath() == "/data/%T/"); - REQUIRE(o.meshesPath() == "fields/"); - REQUIRE(o.particlesPath() == "particles/"); - REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); - REQUIRE(o.iterationFormat() == "data%T.h5"); - REQUIRE(o.name() == "data%T"); + std::tie(backend, test_128_bit) = t; + dtype_test(backend, test_128_bit); + } +} - REQUIRE(o.iterations.size() == 5); - REQUIRE(o.iterations.count(100) == 1); +inline +void write_test(const std::string & backend) +{ + Series o = Series("../samples/serial_write." + backend, AccessType::CREATE); - Iteration& iteration_100 = o.iterations[100]; - REQUIRE(iteration_100.time< double >() == 3.2847121452090077e-14); - REQUIRE(iteration_100.dt< double >() == 3.2847121452090093e-16); - REQUIRE(iteration_100.timeUnitSI() == 1.0); + ParticleSpecies& e_1 = o.iterations[1].particles["e"]; - REQUIRE(iteration_100.meshes.size() == 2); - REQUIRE(iteration_100.meshes.count("E") == 1); - REQUIRE(iteration_100.meshes.count("rho") == 1); + std::vector< double > position_global(4); + double pos{0.}; + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_1(new double); + e_1["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_1), {4})); - std::vector< std::string > al{"x", "y", "z"}; - std::vector< double > gs{8.0000000000000007e-07, - 8.0000000000000007e-07, - 1.0000000000000001e-07}; - std::vector< double > ggo{-1.0000000000000001e-05, - -1.0000000000000001e-05, - -5.1999999999999993e-06}; - std::array< double, 7 > ud{{1., 1., -3., -1., 0., 0., 0.}}; - Mesh& E = iteration_100.meshes["E"]; - REQUIRE(E.geometry() == Mesh::Geometry::cartesian); - REQUIRE(E.dataOrder() == Mesh::DataOrder::C); - REQUIRE(E.axisLabels() == al); - REQUIRE(E.gridSpacing< double >() == gs); - REQUIRE(E.gridGlobalOffset() == ggo); - REQUIRE(E.gridUnitSI() == 1.0); - REQUIRE(E.unitDimension() == ud); - REQUIRE(E.timeOffset< double >() == 0.0); + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_1 = position_global[i]; + e_1["position"]["x"].storeChunk(position_local_1, {i}, {1}); + } - REQUIRE(E.size() == 3); - REQUIRE(E.count("x") == 1); - REQUIRE(E.count("y") == 1); - REQUIRE(E.count("z") == 1); + std::vector< uint64_t > positionOffset_global(4); + uint64_t posOff{0}; + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); + e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); - std::vector< double > p{0.5, 0., 0.}; - Extent e{26, 26, 201}; - MeshRecordComponent& E_x = E["x"]; - REQUIRE(E_x.unitSI() == 1.0); - REQUIRE(E_x.position< double >() == p); - REQUIRE(E_x.getDatatype() == Datatype::DOUBLE); - REQUIRE(E_x.getExtent() == e); - REQUIRE(E_x.getDimensionality() == 3); + for( uint64_t i = 0; i < 4; ++i ) + { + *positionOffset_local_1 = positionOffset_global[i]; + e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); + } - p = {0., 0.5, 0.}; - MeshRecordComponent& E_y = E["y"]; - REQUIRE(E_y.unitSI() == 1.0); - REQUIRE(E_y.position< double >() == p); - REQUIRE(E_y.getDatatype() == Datatype::DOUBLE); - REQUIRE(E_y.getExtent() == e); - REQUIRE(E_y.getDimensionality() == 3); + ParticleSpecies& e_2 = o.iterations[2].particles["e"]; - p = {0., 0., 0.5}; - MeshRecordComponent& E_z = E["z"]; - REQUIRE(E_z.unitSI() == 1.0); - REQUIRE(E_z.position< double >() == p); - REQUIRE(E_z.getDatatype() == Datatype::DOUBLE); - REQUIRE(E_z.getExtent() == e); - REQUIRE(E_z.getDimensionality() == 3); + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_2(new double); + e_2["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_2), {4})); - gs = {8.0000000000000007e-07, - 8.0000000000000007e-07, - 1.0000000000000001e-07}; - ggo = {-1.0000000000000001e-05, - -1.0000000000000001e-05, - -5.1999999999999993e-06}; - ud = {{-3., 0., 1., 1., 0., 0., 0.}}; - Mesh& rho = iteration_100.meshes["rho"]; - REQUIRE(rho.geometry() == Mesh::Geometry::cartesian); - REQUIRE(rho.dataOrder() == Mesh::DataOrder::C); - REQUIRE(rho.axisLabels() == al); - REQUIRE(rho.gridSpacing< double >() == gs); - REQUIRE(rho.gridGlobalOffset() == ggo); - REQUIRE(rho.gridUnitSI() == 1.0); - REQUIRE(rho.unitDimension() == ud); - REQUIRE(rho.timeOffset< double >() == 0.0); + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_2 = position_global[i]; + e_2["position"]["x"].storeChunk(position_local_2, {i}, {1}); + } - REQUIRE(rho.size() == 1); - REQUIRE(rho.count(MeshRecordComponent::SCALAR) == 1); + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_2(new uint64_t); + e_2["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_2), {4})); - p = {0., 0., 0.}; - e = {26, 26, 201}; - MeshRecordComponent& rho_scalar = rho[MeshRecordComponent::SCALAR]; - REQUIRE(rho_scalar.unitSI() == 1.0); - REQUIRE(rho_scalar.position< double >() == p); - REQUIRE(rho_scalar.getDatatype() == Datatype::DOUBLE); - REQUIRE(rho_scalar.getExtent() == e); - REQUIRE(rho_scalar.getDimensionality() == 3); + for( uint64_t i = 0; i < 4; ++i ) + { + *positionOffset_local_2 = positionOffset_global[i]; + e_2["positionOffset"]["x"].storeChunk(positionOffset_local_2, {i}, {1}); + } - REQUIRE(iteration_100.particles.size() == 1); - REQUIRE(iteration_100.particles.count("electrons") == 1); + o.flush(); - ParticleSpecies& electrons = iteration_100.particles["electrons"]; + ParticleSpecies& e_3 = o.iterations[3].particles["e"]; - REQUIRE(electrons.size() == 6); - REQUIRE(electrons.count("charge") == 1); - REQUIRE(electrons.count("mass") == 1); - REQUIRE(electrons.count("momentum") == 1); - REQUIRE(electrons.count("position") == 1); - REQUIRE(electrons.count("positionOffset") == 1); - REQUIRE(electrons.count("weighting") == 1); + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_3(new double); + e_3["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_3), {4})); - ud = {{0., 0., 1., 1., 0., 0., 0.}}; - Record& charge = electrons["charge"]; - REQUIRE(charge.unitDimension() == ud); - REQUIRE(charge.timeOffset< double >() == 0.0); + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_3 = position_global[i]; + e_3["position"]["x"].storeChunk(position_local_3, {i}, {1}); + } - REQUIRE(charge.size() == 1); - REQUIRE(charge.count(RecordComponent::SCALAR) == 1); + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_3(new uint64_t); + e_3["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_3), {4})); - e = {85625}; - RecordComponent& charge_scalar = charge[RecordComponent::SCALAR]; - REQUIRE(charge_scalar.unitSI() == 1.0); - REQUIRE(charge_scalar.getDatatype() == Datatype::DOUBLE); - REQUIRE(charge_scalar.getDimensionality() == 1); - REQUIRE(charge_scalar.getExtent() == e); + for( uint64_t i = 0; i < 4; ++i ) + { + *positionOffset_local_3 = positionOffset_global[i]; + e_3["positionOffset"]["x"].storeChunk(positionOffset_local_3, {i}, {1}); + } - ud = {{1., 0., 0., 0., 0., 0., 0.}}; - Record& mass = electrons["mass"]; - REQUIRE(mass.unitDimension() == ud); - REQUIRE(mass.timeOffset< double >() == 0.0); + o.flush(); +} - REQUIRE(mass.size() == 1); - REQUIRE(mass.count(RecordComponent::SCALAR) == 1); +TEST_CASE( "write_test", "[serial]" ) +{ + for (auto const & t: backends) + { + write_test(std::get<0>(t)); + } +} - RecordComponent& mass_scalar = mass[RecordComponent::SCALAR]; - REQUIRE(mass_scalar.unitSI() == 1.0); - REQUIRE(mass_scalar.getDatatype() == Datatype::DOUBLE); - REQUIRE(mass_scalar.getDimensionality() == 1); - REQUIRE(mass_scalar.getExtent() == e); +inline +void fileBased_write_empty_test(const std::string & backend) +{ + if( auxiliary::directory_exists("../samples/subdir") ) + auxiliary::remove_directory("../samples/subdir"); - ud = {{1., 1., -1., 0., 0., 0., 0.}}; - Record& momentum = electrons["momentum"]; - REQUIRE(momentum.unitDimension() == ud); - REQUIRE(momentum.timeOffset< double >() == 0.0); + Dataset dset = Dataset(Datatype::DOUBLE, {2}); + { + Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::CREATE); - REQUIRE(momentum.size() == 3); - REQUIRE(momentum.count("x") == 1); - REQUIRE(momentum.count("y") == 1); - REQUIRE(momentum.count("z") == 1); + ParticleSpecies& e_1 = o.iterations[1].particles["e"]; + e_1["position"][RecordComponent::SCALAR].resetDataset(dset); + e_1["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[1].setTime(1.f); + ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + e_2["position"][RecordComponent::SCALAR].resetDataset(dset); + e_2["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[2].setTime(2.f); + ParticleSpecies& e_3 = o.iterations[3].particles["e"]; + e_3["position"][RecordComponent::SCALAR].resetDataset(dset); + e_3["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[3].setTime(3.f); + } - RecordComponent& momentum_x = momentum["x"]; - REQUIRE(momentum_x.unitSI() == 1.0); - REQUIRE(momentum_x.getDatatype() == Datatype::DOUBLE); - REQUIRE(momentum_x.getDimensionality() == 1); - REQUIRE(momentum_x.getExtent() == e); + REQUIRE(auxiliary::directory_exists("../samples/subdir")); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write1." + backend)); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write2." + backend)); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write3." + backend)); - RecordComponent& momentum_y = momentum["y"]; - REQUIRE(momentum_y.unitSI() == 1.0); - REQUIRE(momentum_y.getDatatype() == Datatype::DOUBLE); - REQUIRE(momentum_y.getDimensionality() == 1); - REQUIRE(momentum_y.getExtent() == e); + { + Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY); - RecordComponent& momentum_z = momentum["z"]; - REQUIRE(momentum_z.unitSI() == 1.0); - REQUIRE(momentum_z.getDatatype() == Datatype::DOUBLE); - REQUIRE(momentum_z.getDimensionality() == 1); - REQUIRE(momentum_z.getExtent() == e); + REQUIRE(o.iterations.size() == 3); + REQUIRE(o.iterations.count(1) == 1); + REQUIRE(o.iterations.count(2) == 1); + REQUIRE(o.iterations.count(3) == 1); - ud = {{1., 0., 0., 0., 0., 0., 0.}}; - Record& position = electrons["position"]; - REQUIRE(position.unitDimension() == ud); - REQUIRE(position.timeOffset< double >() == 0.0); + REQUIRE(o.iterations[1].time< float >() == 1.f); + REQUIRE(o.iterations[2].time< float >() == 2.f); + REQUIRE(o.iterations[3].time< float >() == 3.f); - REQUIRE(position.size() == 3); - REQUIRE(position.count("x") == 1); - REQUIRE(position.count("y") == 1); - REQUIRE(position.count("z") == 1); + REQUIRE(o.iterations[1].particles.size() == 1); + REQUIRE(o.iterations[1].particles.count("e") == 1); + REQUIRE(o.iterations[2].particles.size() == 1); + REQUIRE(o.iterations[2].particles.count("e") == 1); + REQUIRE(o.iterations[3].particles.size() == 1); + REQUIRE(o.iterations[3].particles.count("e") == 1); - RecordComponent& position_x = position["x"]; - REQUIRE(position_x.unitSI() == 1.0); - REQUIRE(position_x.getDatatype() == Datatype::DOUBLE); - REQUIRE(position_x.getDimensionality() == 1); - REQUIRE(position_x.getExtent() == e); + REQUIRE(o.iterations[1].particles["e"].size() == 2); + REQUIRE(o.iterations[1].particles["e"].count("position") == 1); + REQUIRE(o.iterations[1].particles["e"].count("positionOffset") == 1); + REQUIRE(o.iterations[2].particles["e"].size() == 2); + REQUIRE(o.iterations[2].particles["e"].count("position") == 1); + REQUIRE(o.iterations[2].particles["e"].count("positionOffset") == 1); + REQUIRE(o.iterations[3].particles["e"].size() == 2); + REQUIRE(o.iterations[3].particles["e"].count("position") == 1); + REQUIRE(o.iterations[3].particles["e"].count("positionOffset") == 1); - RecordComponent& position_y = position["y"]; - REQUIRE(position_y.unitSI() == 1.0); - REQUIRE(position_y.getDatatype() == Datatype::DOUBLE); - REQUIRE(position_y.getDimensionality() == 1); - REQUIRE(position_y.getExtent() == e); + REQUIRE(o.iterations[1].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[1].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[1].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[1].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + REQUIRE(o.iterations[2].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[2].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[2].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[2].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + REQUIRE(o.iterations[3].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[3].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[3].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[3].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + } - RecordComponent& position_z = position["z"]; - REQUIRE(position_z.unitSI() == 1.0); - REQUIRE(position_z.getDatatype() == Datatype::DOUBLE); - REQUIRE(position_z.getDimensionality() == 1); - REQUIRE(position_z.getExtent() == e); + { + Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_WRITE); + ParticleSpecies& e_4 = o.iterations[4].particles["e"]; + e_4["position"][RecordComponent::SCALAR].resetDataset(dset); + e_4["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[4].setTime(4.f); + } - Record& positionOffset = electrons["positionOffset"]; - REQUIRE(positionOffset.unitDimension() == ud); - REQUIRE(positionOffset.timeOffset< double >() == 0.0); + { + Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY); - REQUIRE(positionOffset.size() == 3); - REQUIRE(positionOffset.count("x") == 1); - REQUIRE(positionOffset.count("y") == 1); - REQUIRE(positionOffset.count("z") == 1); - - RecordComponent& positionOffset_x = positionOffset["x"]; - REQUIRE(positionOffset_x.unitSI() == 1.0); - REQUIRE(positionOffset_x.getDatatype() == Datatype::DOUBLE); - REQUIRE(positionOffset_x.getDimensionality() == 1); - REQUIRE(positionOffset_x.getExtent() == e); + REQUIRE(o.iterations.size() == 4); + REQUIRE(o.iterations.count(4) == 1); - RecordComponent& positionOffset_y = positionOffset["y"]; - REQUIRE(positionOffset_y.unitSI() == 1.0); - REQUIRE(positionOffset_y.getDatatype() == Datatype::DOUBLE); - REQUIRE(positionOffset_y.getDimensionality() == 1); - REQUIRE(positionOffset_y.getExtent() == e); + REQUIRE(o.iterations[4].time< float >() == 4.f); - RecordComponent& positionOffset_z = positionOffset["z"]; - REQUIRE(positionOffset_z.unitSI() == 1.0); - REQUIRE(positionOffset_z.getDatatype() == Datatype::DOUBLE); - REQUIRE(positionOffset_z.getDimensionality() == 1); - REQUIRE(positionOffset_z.getExtent() == e); + REQUIRE(o.iterations[4].particles.size() == 1); + REQUIRE(o.iterations[4].particles.count("e") == 1); - ud = {{0., 0., 0., 0., 0., 0., 0.}}; - Record& weighting = electrons["weighting"]; - REQUIRE(weighting.unitDimension() == ud); - REQUIRE(weighting.timeOffset< double >() == 0.0); + REQUIRE(o.iterations[4].particles["e"].size() == 2); + REQUIRE(o.iterations[4].particles["e"].count("position") == 1); + REQUIRE(o.iterations[4].particles["e"].count("positionOffset") == 1); - REQUIRE(weighting.size() == 1); - REQUIRE(weighting.count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[4].particles["e"]["position"].size() == 1); + REQUIRE(o.iterations[4].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[4].particles["e"]["positionOffset"].size() == 1); + REQUIRE(o.iterations[4].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); + REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); + REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); + REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); + } +} - RecordComponent& weighting_scalar = weighting[RecordComponent::SCALAR]; - REQUIRE(weighting_scalar.unitSI() == 1.0); - REQUIRE(weighting_scalar.getDatatype() == Datatype::DOUBLE); - REQUIRE(weighting_scalar.getDimensionality() == 1); - REQUIRE(weighting_scalar.getExtent() == e); - } catch (no_such_file_error& e) +TEST_CASE( "fileBased_write_empty_test", "[serial]" ) +{ + for (auto const & t: backends) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + fileBased_write_empty_test(std::get<0>(t)); } } -TEST_CASE( "git_hdf5_sample_content_test", "[serial][hdf5]" ) +inline +void fileBased_write_test(const std::string & backend) { - try + if( auxiliary::directory_exists("../samples/subdir") ) + auxiliary::remove_directory("../samples/subdir"); + { - Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); + Series o = Series("../samples/subdir/serial_fileBased_write%08T." + backend, AccessType::CREATE); - { - double actual[3][3][3] = {{{-1.9080703683727052e-09, -1.5632650729457964e-10, 1.1497536256399599e-09}, - {-1.9979540244463578e-09, -2.5512036927466397e-10, 1.0402234629225404e-09}, - {-1.7353589676361025e-09, -8.0899198451334087e-10, -1.6443779671249104e-10}}, + ParticleSpecies& e_1 = o.iterations[1].particles["e"]; - {{-2.0029988778702545e-09, -1.9543477947081556e-10, 1.0916454407094989e-09}, - {-2.3890367462087170e-09, -4.7158010829662089e-10, 9.0026075483251589e-10}, - {-1.9033881137886510e-09, -7.5192119197708962e-10, 5.0038861942880430e-10}}, + std::vector< double > position_global(4); + double pos{0.}; + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_1(new double); + e_1["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_1), {4})); + std::vector< uint64_t > positionOffset_global(4); + uint64_t posOff{0}; + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); + e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); - {{-1.3271805876513554e-09, -5.9243276950837753e-10, -2.2445734160214670e-10}, - {-7.4578609954301101e-10, -1.1995737736469891e-10, 2.5611823772919706e-10}, - {-9.4806251738077663e-10, -1.5472800818372434e-10, -3.6461900165818406e-10}}}; - MeshRecordComponent& rho = o.iterations[100].meshes["rho"][MeshRecordComponent::SCALAR]; - Offset offset{20, 20, 190}; - Extent extent{3, 3, 3}; - auto data = rho.loadChunk(offset, extent); + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_1 = position_global[i]; + e_1["position"]["x"].storeChunk(position_local_1, {i}, {1}); + *positionOffset_local_1 = positionOffset_global[i]; + e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); o.flush(); - double* raw_ptr = data.get(); - - for( int i = 0; i < 3; ++i ) - for( int j = 0; j < 3; ++j ) - for( int k = 0; k < 3; ++k ) - REQUIRE(raw_ptr[((i*3) + j)*3 + k] == actual[i][j][k]); } + o.iterations[1].setTime(static_cast< double >(1)); + + ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + e_2["position"]["x"].resetDataset(Dataset(determineDatatype(), {4})); + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_2(new uint64_t); + e_2["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_2), {4})); + + for( uint64_t i = 0; i < 4; ++i ) { - double constant_value = 9.1093829099999999e-31; - RecordComponent& electrons_mass = o.iterations[100].particles["electrons"]["mass"][RecordComponent::SCALAR]; - Offset offset{15}; - Extent extent{3}; - auto data = electrons_mass.loadChunk(offset, extent); + double const position_local_2 = position_global.at(i); + e_2["position"]["x"].storeChunk(shareRaw(&position_local_2), {i}, {1}); + *positionOffset_local_2 = positionOffset_global[i]; + e_2["positionOffset"]["x"].storeChunk(positionOffset_local_2, {i}, {1}); o.flush(); - double* raw_ptr = data.get(); - - for( int i = 0; i < 3; ++i ) - REQUIRE(raw_ptr[i] == constant_value); } - } catch (no_such_file_error& e) - { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; - } -} -TEST_CASE( "git_hdf5_sample_fileBased_read_test", "[serial][hdf5]" ) -{ - try - { - Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); + o.iterations[2].setTime(static_cast< double >(2)); - REQUIRE(o.iterations.size() == 5); - REQUIRE(o.iterations.count(100) == 1); - REQUIRE(o.iterations.count(200) == 1); - REQUIRE(o.iterations.count(300) == 1); - REQUIRE(o.iterations.count(400) == 1); - REQUIRE(o.iterations.count(500) == 1); + ParticleSpecies& e_3 = o.iterations[3].particles["e"]; -#if openPMD_USE_INVASIVE_TESTS - REQUIRE(*o.m_filenamePadding == 8); -#endif - } catch (no_such_file_error& e) - { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); + std::shared_ptr< double > position_local_3(new double); + e_3["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_3), {4})); + std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); + std::shared_ptr< uint64_t > positionOffset_local_3(new uint64_t); + e_3["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_3), {4})); + + for( uint64_t i = 0; i < 4; ++i ) + { + *position_local_3 = position_global[i]; + e_3["position"]["x"].storeChunk(position_local_3, {i}, {1}); + *positionOffset_local_3 = positionOffset_global[i]; + e_3["positionOffset"]["x"].storeChunk(positionOffset_local_3, {i}, {1}); + o.flush(); + } + + o.setOpenPMDextension(1); + o.iterations[3].setTime(static_cast< double >(3)); } + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000001." + backend)); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000002." + backend)); + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000003." + backend)); - try { - Series o = Series("../samples/git-sample/data%08T.h5", AccessType::READ_ONLY); + Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY); - REQUIRE(o.iterations.size() == 5); - REQUIRE(o.iterations.count(100) == 1); - REQUIRE(o.iterations.count(200) == 1); - REQUIRE(o.iterations.count(300) == 1); - REQUIRE(o.iterations.count(400) == 1); - REQUIRE(o.iterations.count(500) == 1); + REQUIRE(o.iterations.size() == 3); + REQUIRE(o.iterations.count(1) == 1); + REQUIRE(o.iterations.count(2) == 1); + REQUIRE(o.iterations.count(3) == 1); #if openPMD_USE_INVASIVE_TESTS REQUIRE(*o.m_filenamePadding == 8); #endif - } catch (no_such_file_error& e) - { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; - } - - REQUIRE_THROWS_WITH(Series("../samples/git-sample/data%07T.h5", AccessType::READ_ONLY), - Catch::Equals("No matching iterations found: data%07T")); - - try - { - std::vector< std::string > newFiles{"../samples/git-sample/data00000001.h5", - "../samples/git-sample/data00000010.h5", - "../samples/git-sample/data00001000.h5", - "../samples/git-sample/data00010000.h5", - "../samples/git-sample/data00100000.h5"}; - - for( auto const& file : newFiles ) - if( auxiliary::file_exists(file) ) - auxiliary::remove_file(file); + REQUIRE(o.basePath() == "/data/%T/"); + REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); + REQUIRE(o.iterationFormat() == "serial_fileBased_write%08T"); + REQUIRE(o.openPMD() == "1.1.0"); + REQUIRE(o.openPMDextension() == 1u); + REQUIRE(o.particlesPath() == "particles/"); + REQUIRE_FALSE(o.containsAttribute("meshesPath")); + REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error); + std::array< double, 7 > udim{{1, 0, 0, 0, 0, 0, 0}}; + Extent ext{4}; + for( auto& entry : o.iterations ) { - Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_WRITE); - -#if openPMD_USE_INVASIVE_TESTS - REQUIRE(*o.m_filenamePadding == 8); + auto& it = entry.second; + REQUIRE(it.dt< double >() == 1.); + REQUIRE(it.time< double >() == static_cast< double >(entry.first)); + REQUIRE(it.timeUnitSI() == 1.); + auto& pos = it.particles.at("e").at("position"); + REQUIRE(pos.timeOffset< float >() == 0.f); + REQUIRE(pos.unitDimension() == udim); + auto& pos_x = pos.at("x"); + REQUIRE(pos_x.unitSI() == 1.); + REQUIRE(pos_x.getExtent() == ext); + REQUIRE(pos_x.getDatatype() == Datatype::DOUBLE); + auto& posOff = it.particles.at("e").at("positionOffset"); + REQUIRE(posOff.timeOffset< float >() == 0.f); + REQUIRE(posOff.unitDimension() == udim); + auto& posOff_x = posOff.at("x"); + REQUIRE(posOff_x.unitSI() == 1.); + REQUIRE(posOff_x.getExtent() == ext); +#if !defined(_MSC_VER) + REQUIRE(posOff_x.getDatatype() == determineDatatype< uint64_t >()); #endif + REQUIRE(isSame(posOff_x.getDatatype(), determineDatatype< uint64_t >())); - o.iterations[1]; - o.iterations[10]; - o.iterations[1000]; - o.iterations[10000]; - o.iterations[100000]; + auto position = pos_x.loadChunk< double >({0}, {4}); + auto position_raw = position.get(); + auto positionOffset = posOff_x.loadChunk< uint64_t >({0}, {4}); + auto positionOffset_raw = positionOffset.get(); o.flush(); + for( uint64_t j = 0; j < 4; ++j ) + { + REQUIRE(position_raw[j] == static_cast< double >(j + (entry.first-1)*4)); + REQUIRE(positionOffset_raw[j] == j + (entry.first-1)*4); + } } + } - for( auto const& file : newFiles ) - { - REQUIRE(auxiliary::file_exists(file)); - auxiliary::remove_file(file); - } - } catch (no_such_file_error& e) + // extend existing series with new step and auto-detection of iteration padding { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; - } -} + Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_WRITE); -TEST_CASE( "hzdr_hdf5_sample_content_test", "[serial][hdf5]" ) -{ - // since this file might not be publicly available, gracefully handle errors - try - { - /* HZDR: /bigdata/hplsim/development/huebl/lwfa-openPMD-062-smallLWFA-h5 - * DOI:10.14278/rodare.57 */ - Series o = Series("../samples/hzdr-sample/h5/simData_%T.h5", AccessType::READ_ONLY); + REQUIRE(o.iterations.size() == 3); + o.iterations[4]; + REQUIRE(o.iterations.size() == 4); + } + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000004." + backend)); - REQUIRE(o.openPMD() == "1.0.0"); - REQUIRE(o.openPMDextension() == 1); - REQUIRE(o.basePath() == "/data/%T/"); - REQUIRE(o.meshesPath() == "fields/"); - REQUIRE(o.particlesPath() == "particles/"); - REQUIRE(o.author() == "Axel Huebl "); - REQUIRE(o.software() == "PIConGPU"); - REQUIRE(o.softwareVersion() == "0.2.0"); - REQUIRE(o.date() == "2016-11-04 00:59:14 +0100"); - REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); - REQUIRE(o.iterationFormat() == "h5/simData_%T.h5"); - REQUIRE(o.name() == "simData_%T"); + // additional iteration with different iteration padding but similar content + { + Series o = Series("../samples/subdir/serial_fileBased_write%01T." + backend, AccessType::READ_WRITE); - REQUIRE(o.iterations.size() >= 1); - REQUIRE(o.iterations.count(0) == 1); + REQUIRE(o.iterations.empty()); - Iteration& i = o.iterations[0]; - REQUIRE(i.time< float >() == static_cast< float >(0.0f)); - REQUIRE(i.dt< float >() == static_cast< float >(1.0f)); - REQUIRE(i.timeUnitSI() == 1.3899999999999999e-16); + auto& it = o.iterations[10]; + ParticleSpecies& e = it.particles["e"]; + e["position"]["x"].resetDataset(Dataset(Datatype::DOUBLE, {42})); + e["positionOffset"]["x"].resetDataset(Dataset(Datatype::DOUBLE, {42})); + e["position"]["x"].makeConstant(1.23); + e["positionOffset"]["x"].makeConstant(1.23); - REQUIRE(i.meshes.size() == 4); - REQUIRE(i.meshes.count("B") == 1); - REQUIRE(i.meshes.count("E") == 1); - REQUIRE(i.meshes.count("e_chargeDensity") == 1); - REQUIRE(i.meshes.count("e_energyDensity") == 1); + REQUIRE(o.iterations.size() == 1); + } + REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write10." + backend)); - std::vector< std::string > al{"z", "y", "x"}; - std::vector< float > gs{static_cast< float >(6.2393283843994141f), - static_cast< float >(1.0630855560302734f), - static_cast< float >(6.2393283843994141f)}; - std::vector< double > ggo{0., 0., 0.}; - std::array< double, 7 > ud{{0., 1., -2., -1., 0., 0., 0.}}; - Mesh& B = i.meshes["B"]; - REQUIRE(B.geometry() == Mesh::Geometry::cartesian); - REQUIRE(B.dataOrder() == Mesh::DataOrder::C); - REQUIRE(B.axisLabels() == al); - REQUIRE(B.gridSpacing< float >() == gs); - REQUIRE(B.gridGlobalOffset() == ggo); - REQUIRE(B.gridUnitSI() == 4.1671151661999998e-08); - REQUIRE(B.unitDimension() == ud); - REQUIRE(B.timeOffset< float >() == static_cast< float >(0.0f)); + // read back with auto-detection and non-fixed padding + { + Series s = Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_ONLY); + REQUIRE(s.iterations.size() == 5); + } - REQUIRE(B.size() == 3); - REQUIRE(B.count("x") == 1); - REQUIRE(B.count("y") == 1); - REQUIRE(B.count("z") == 1); + // write with auto-detection and in-consistent padding + { + REQUIRE_THROWS_WITH(Series("../samples/subdir/serial_fileBased_write%T." + backend, AccessType::READ_WRITE), + Catch::Equals("Cannot write to a series with inconsistent iteration padding. Please specify '%0T' or open as read-only.")); + } - std::vector< float > p{static_cast< float >(0.0f), - static_cast< float >(0.5f), - static_cast< float >(0.5f)}; - Extent e{80, 384, 80}; - MeshRecordComponent& B_x = B["x"]; - REQUIRE(B_x.unitSI() == 40903.822240601701); - REQUIRE(B_x.position< float >() == p); - REQUIRE(B_x.getDatatype() == Datatype::FLOAT); - REQUIRE(B_x.getExtent() == e); - REQUIRE(B_x.getDimensionality() == 3); + // read back with auto-detection and fixed padding + { + Series s = Series("../samples/subdir/serial_fileBased_write%08T." + backend, AccessType::READ_ONLY); + REQUIRE(s.iterations.size() == 4); + } +} - p = {static_cast< float >(0.5f), - static_cast< float >(0.0f), - static_cast< float >(0.5f)}; - MeshRecordComponent& B_y = B["y"]; - REQUIRE(B_y.unitSI() == 40903.822240601701); - REQUIRE(B_y.position< float >() == p); - REQUIRE(B_y.getDatatype() == Datatype::FLOAT); - REQUIRE(B_y.getExtent() == e); - REQUIRE(B_y.getDimensionality() == 3); +TEST_CASE( "fileBased_write_test", "[serial]" ) +{ + for (auto const & t: backends) + { + fileBased_write_test(std::get<0>(t)); + } +} - p = {static_cast< float >(0.5f), - static_cast< float >(0.5f), - static_cast< float >(0.0f)}; - MeshRecordComponent& B_z = B["z"]; - REQUIRE(B_z.unitSI() == 40903.822240601701); - REQUIRE(B_z.position< float >() == p); - REQUIRE(B_z.getDatatype() == Datatype::FLOAT); - REQUIRE(B_z.getExtent() == e); - REQUIRE(B_z.getDimensionality() == 3); +inline +void bool_test(const std::string & backend) +{ + { + Series o = Series("../samples/serial_bool." + backend, AccessType::CREATE); - ud = {{1., 1., -3., -1., 0., 0., 0.}}; - Mesh& E = i.meshes["E"]; - REQUIRE(E.geometry() == Mesh::Geometry::cartesian); - REQUIRE(E.dataOrder() == Mesh::DataOrder::C); - REQUIRE(E.axisLabels() == al); - REQUIRE(E.gridSpacing< float >() == gs); - REQUIRE(E.gridGlobalOffset() == ggo); - REQUIRE(E.gridUnitSI() == 4.1671151661999998e-08); - REQUIRE(E.unitDimension() == ud); - REQUIRE(E.timeOffset< float >() == static_cast< float >(0.0f)); + o.setAttribute("Bool attribute (true)", true); + o.setAttribute("Bool attribute (false)", false); + } + { + Series o = Series("../samples/serial_bool." + backend, AccessType::READ_ONLY); - REQUIRE(E.size() == 3); - REQUIRE(E.count("x") == 1); - REQUIRE(E.count("y") == 1); - REQUIRE(E.count("z") == 1); + auto attrs = o.attributes(); + REQUIRE(std::count(attrs.begin(), attrs.end(), "Bool attribute (true)") == 1); + REQUIRE(std::count(attrs.begin(), attrs.end(), "Bool attribute (false)") == 1); + REQUIRE(o.getAttribute("Bool attribute (true)").get< bool >() == true); + REQUIRE(o.getAttribute("Bool attribute (false)").get< bool >() == false); + } +} - p = {static_cast< float >(0.5f), - static_cast< float >(0.0f), - static_cast< float >(0.0f)}; - e = {80, 384, 80}; - MeshRecordComponent& E_x = E["x"]; - REQUIRE(E_x.unitSI() == 12262657411105.049); - REQUIRE(E_x.position< float >() == p); - REQUIRE(E_x.getDatatype() == Datatype::FLOAT); - REQUIRE(E_x.getExtent() == e); - REQUIRE(E_x.getDimensionality() == 3); +TEST_CASE( "bool_test", "[serial]" ) +{ + for (auto const & t: backends) + { + bool_test(std::get<0>(t)); + } +} - p = {static_cast< float >(0.0f), - static_cast< float >(0.5f), - static_cast< float >(0.0f)}; - MeshRecordComponent& E_y = E["y"]; - REQUIRE(E_y.unitSI() == 12262657411105.049); - REQUIRE(E_y.position< float >() == p); - REQUIRE(E_y.getDatatype() == Datatype::FLOAT); - REQUIRE(E_y.getExtent() == e); - REQUIRE(E_y.getDimensionality() == 3); +inline +void patch_test(const std::string & backend) +{ + Series o = Series("../samples/serial_patch." + backend, AccessType::CREATE); - p = {static_cast< float >(0.0f), - static_cast< float >(0.0f), - static_cast< float >(0.5f)}; - MeshRecordComponent& E_z = E["z"]; - REQUIRE(E_z.unitSI() == 12262657411105.049); - REQUIRE(E_z.position< float >() == p); - REQUIRE(E_z.getDatatype() == Datatype::FLOAT); - REQUIRE(E_z.getExtent() == e); - REQUIRE(E_z.getDimensionality() == 3); + auto dset = Dataset(Datatype::DOUBLE, {1}); + o.iterations[1].particles["e"].particlePatches["offset"]["x"].resetDataset(dset); + o.iterations[1].particles["e"].particlePatches["offset"]["x"].setUnitSI(42); + o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].resetDataset(dset); + o.iterations[1].particles["e"]["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); +} - ud = {{-3., 0., 1., 1., 0., 0., 0.}}; - Mesh& e_chargeDensity = i.meshes["e_chargeDensity"]; - REQUIRE(e_chargeDensity.geometry() == Mesh::Geometry::cartesian); - REQUIRE(e_chargeDensity.dataOrder() == Mesh::DataOrder::C); - REQUIRE(e_chargeDensity.axisLabels() == al); - REQUIRE(e_chargeDensity.gridSpacing< float >() == gs); - REQUIRE(e_chargeDensity.gridGlobalOffset() == ggo); - REQUIRE(e_chargeDensity.gridUnitSI() == 4.1671151661999998e-08); - REQUIRE(e_chargeDensity.unitDimension() == ud); - REQUIRE(e_chargeDensity.timeOffset< float >() == static_cast< float >(0.0f)); +TEST_CASE( "patch_test", "[serial]" ) +{ + for (auto const & t: backends) + { + patch_test(std::get<0>(t)); + } +} - REQUIRE(e_chargeDensity.size() == 1); - REQUIRE(e_chargeDensity.count(MeshRecordComponent::SCALAR) == 1); +inline +void deletion_test(const std::string & backend) +{ + Series o = Series("../samples/serial_deletion." + backend, AccessType::CREATE); - p = {static_cast< float >(0.f), - static_cast< float >(0.f), - static_cast< float >(0.f)}; - MeshRecordComponent& e_chargeDensity_scalar = e_chargeDensity[MeshRecordComponent::SCALAR]; - REQUIRE(e_chargeDensity_scalar.unitSI() == 66306201.002331272); - REQUIRE(e_chargeDensity_scalar.position< float >() == p); - REQUIRE(e_chargeDensity_scalar.getDatatype() == Datatype::FLOAT); - REQUIRE(e_chargeDensity_scalar.getExtent() == e); - REQUIRE(e_chargeDensity_scalar.getDimensionality() == 3); - ud = {{-1., 1., -2., 0., 0., 0., 0.}}; - Mesh& e_energyDensity = i.meshes["e_energyDensity"]; - REQUIRE(e_energyDensity.geometry() == Mesh::Geometry::cartesian); - REQUIRE(e_energyDensity.dataOrder() == Mesh::DataOrder::C); - REQUIRE(e_energyDensity.axisLabels() == al); - REQUIRE(e_energyDensity.gridSpacing< float >() == gs); - REQUIRE(e_energyDensity.gridGlobalOffset() == ggo); - REQUIRE(e_energyDensity.gridUnitSI() == 4.1671151661999998e-08); - REQUIRE(e_energyDensity.unitDimension() == ud); - REQUIRE(e_energyDensity.timeOffset< float >() == static_cast< float >(0.0f)); + o.setAttribute("removed", + "this attribute will be removed after being written to disk"); + o.flush(); - REQUIRE(e_energyDensity.size() == 1); - REQUIRE(e_energyDensity.count(MeshRecordComponent::SCALAR) == 1); + o.deleteAttribute("removed"); + o.flush(); - MeshRecordComponent& e_energyDensity_scalar = e_energyDensity[MeshRecordComponent::SCALAR]; - REQUIRE(e_energyDensity_scalar.unitSI() == 1.0146696675429705e+18); - REQUIRE(e_energyDensity_scalar.position< float >() == p); - REQUIRE(e_energyDensity_scalar.getDatatype() == Datatype::FLOAT); - REQUIRE(e_energyDensity_scalar.getExtent() == e); - REQUIRE(e_energyDensity_scalar.getDimensionality() == 3); + ParticleSpecies& e = o.iterations[1].particles["e"]; + auto dset = Dataset(Datatype::DOUBLE, {1}); + e["position"][RecordComponent::SCALAR].resetDataset(dset); + e["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + e.erase("deletion"); + o.flush(); - REQUIRE(i.particles.size() == 1); - REQUIRE(i.particles.count("e") == 1); + e["deletion_scalar"][RecordComponent::SCALAR].resetDataset(dset); + o.flush(); - ParticleSpecies& species_e = i.particles["e"]; + e["deletion_scalar"].erase(RecordComponent::SCALAR); + e.erase("deletion_scalar"); + o.flush(); - REQUIRE(species_e.size() == 6); - REQUIRE(species_e.count("charge") == 1); - REQUIRE(species_e.count("mass") == 1); - REQUIRE(species_e.count("momentum") == 1); - REQUIRE(species_e.count("particlePatches") == 0); - REQUIRE(species_e.count("position") == 1); - REQUIRE(species_e.count("positionOffset") == 1); - REQUIRE(species_e.count("weighting") == 1); + double value = 0.; + e["deletion_scalar_constant"][RecordComponent::SCALAR].resetDataset(dset); + e["deletion_scalar_constant"][RecordComponent::SCALAR].makeConstant(value); + o.flush(); - ud = {{0., 0., 1., 1., 0., 0., 0.}}; - Record& e_charge = species_e["charge"]; - REQUIRE(e_charge.unitDimension() == ud); - REQUIRE(e_charge.timeOffset< float >() == static_cast< float >(0.0f)); + e["deletion_scalar_constant"].erase(RecordComponent::SCALAR); + e.erase("deletion_scalar_constant"); + o.flush(); +} - REQUIRE(e_charge.size() == 1); - REQUIRE(e_charge.count(RecordComponent::SCALAR) == 1); +TEST_CASE( "deletion_test", "[serial]" ) +{ + for (auto const & t: backends) + { + if (std::get<0>(t) == "bp") + { + continue; // deletion not implemented in ADIOS1 backend + } + deletion_test(std::get<0>(t)); + } +} - e = {2150400}; - RecordComponent& e_charge_scalar = e_charge[RecordComponent::SCALAR]; - REQUIRE(e_charge_scalar.unitSI() == 4.7980045488500004e-15); - REQUIRE(e_charge_scalar.getDatatype() == Datatype::DOUBLE); - REQUIRE(e_charge_scalar.getExtent() == e); - REQUIRE(e_charge_scalar.getDimensionality() == 1); +inline +void optional_paths_110_test(const std::string & backend) +{ + try + { + { + Series s = Series("../samples/issue-sample/no_fields/data%T." + backend, AccessType::READ_ONLY); + auto attrs = s.attributes(); + REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 1); + REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 1); + REQUIRE(s.iterations[400].meshes.empty()); + REQUIRE(s.iterations[400].particles.size() == 1); + } - ud = {{0., 1., 0., 0., 0., 0., 0.}}; - Record& e_mass = species_e["mass"]; - REQUIRE(e_mass.unitDimension() == ud); - REQUIRE(e_mass.timeOffset< float >() == static_cast< float >(0.0f)); + { + Series s = Series("../samples/issue-sample/no_particles/data%T." + backend, AccessType::READ_ONLY); + auto attrs = s.attributes(); + REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 1); + REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 1); + REQUIRE(s.iterations[400].meshes.size() == 2); + REQUIRE(s.iterations[400].particles.empty()); + } + } catch (no_such_file_error& e) + { + std::cerr << "issue sample not accessible. (" << e.what() << ")\n"; + } - REQUIRE(e_mass.size() == 1); - REQUIRE(e_mass.count(RecordComponent::SCALAR) == 1); + { + Series s = Series("../samples/no_meshes_1.1.0_compliant." + backend, AccessType::CREATE); + auto foo = s.iterations[1].particles["foo"]; + Dataset dset = Dataset(Datatype::DOUBLE, {1}); + foo["position"][RecordComponent::SCALAR].resetDataset(dset); + foo["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); + } - RecordComponent& e_mass_scalar = e_mass[RecordComponent::SCALAR]; - REQUIRE(e_mass_scalar.unitSI() == 2.7279684799430467e-26); - REQUIRE(e_mass_scalar.getDatatype() == Datatype::DOUBLE); - REQUIRE(e_mass_scalar.getExtent() == e); - REQUIRE(e_mass_scalar.getDimensionality() == 1); + { + Series s = Series("../samples/no_particles_1.1.0_compliant." + backend, AccessType::CREATE); + auto foo = s.iterations[1].meshes["foo"]; + Dataset dset = Dataset(Datatype::DOUBLE, {1}); + foo[RecordComponent::SCALAR].resetDataset(dset); + } - ud = {{1., 1., -1., 0., 0., 0., 0.}}; - Record& e_momentum = species_e["momentum"]; - REQUIRE(e_momentum.unitDimension() == ud); - REQUIRE(e_momentum.timeOffset< float >() == static_cast< float >(0.0f)); + { + Series s = Series("../samples/no_meshes_1.1.0_compliant." + backend, AccessType::READ_ONLY); + auto attrs = s.attributes(); + REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 0); + REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 1); + REQUIRE(s.iterations[1].meshes.empty()); + REQUIRE(s.iterations[1].particles.size() == 1); + } - REQUIRE(e_momentum.size() == 3); - REQUIRE(e_momentum.count("x") == 1); - REQUIRE(e_momentum.count("y") == 1); - REQUIRE(e_momentum.count("z") == 1); + { + Series s = Series("../samples/no_particles_1.1.0_compliant." + backend, AccessType::READ_ONLY); + auto attrs = s.attributes(); + REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 1); + REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 0); + REQUIRE(s.iterations[1].meshes.size() == 1); + REQUIRE(s.iterations[1].particles.empty()); + } +} - RecordComponent& e_momentum_x = e_momentum["x"]; - REQUIRE(e_momentum_x.unitSI() == 8.1782437594864961e-18); - REQUIRE(e_momentum_x.getDatatype() == Datatype::FLOAT); - REQUIRE(e_momentum_x.getExtent() == e); - REQUIRE(e_momentum_x.getDimensionality() == 1); - RecordComponent& e_momentum_y = e_momentum["y"]; - REQUIRE(e_momentum_y.unitSI() == 8.1782437594864961e-18); - REQUIRE(e_momentum_y.getDatatype() == Datatype::FLOAT); - REQUIRE(e_momentum_y.getExtent() == e); - REQUIRE(e_momentum_y.getDimensionality() == 1); +#if openPMD_HAVE_HDF5 +TEST_CASE( "optional_paths_110_test", "[serial]" ) +{ + optional_paths_110_test("h5"); // samples only present for hdf5 +} - RecordComponent& e_momentum_z = e_momentum["z"]; - REQUIRE(e_momentum_z.unitSI() == 8.1782437594864961e-18); - REQUIRE(e_momentum_z.getDatatype() == Datatype::FLOAT); - REQUIRE(e_momentum_z.getExtent() == e); - REQUIRE(e_momentum_z.getDimensionality() == 1); +TEST_CASE( "git_hdf5_sample_structure_test", "[serial][hdf5]" ) +{ +#if openPMD_USE_INVASIVE_TESTS + try + { + Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); - ud = {{1., 0., 0., 0., 0., 0., 0.}}; - Record& e_position = species_e["position"]; - REQUIRE(e_position.unitDimension() == ud); - REQUIRE(e_position.timeOffset< float >() == static_cast< float >(0.0f)); + REQUIRE(!o.parent); + REQUIRE(o.iterations.parent == getWritable(&o)); + REQUIRE_THROWS_AS(o.iterations[42], std::out_of_range); + REQUIRE(o.iterations[100].parent == getWritable(&o.iterations)); + REQUIRE(o.iterations[100].meshes.parent == getWritable(&o.iterations[100])); + REQUIRE(o.iterations[100].meshes["E"].parent == getWritable(&o.iterations[100].meshes)); + REQUIRE(o.iterations[100].meshes["E"]["x"].parent == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE(o.iterations[100].meshes["E"]["y"].parent == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE(o.iterations[100].meshes["E"]["z"].parent == getWritable(&o.iterations[100].meshes["E"])); + REQUIRE(o.iterations[100].meshes["rho"].parent == getWritable(&o.iterations[100].meshes)); + REQUIRE(o.iterations[100].meshes["rho"][MeshRecordComponent::SCALAR].parent == getWritable(&o.iterations[100].meshes)); + REQUIRE_THROWS_AS(o.iterations[100].meshes["cherries"], std::out_of_range); + REQUIRE(o.iterations[100].particles.parent == getWritable(&o.iterations[100])); + REQUIRE(o.iterations[100].particles["electrons"].parent == getWritable(&o.iterations[100].particles)); + REQUIRE(o.iterations[100].particles["electrons"]["charge"].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["charge"][RecordComponent::SCALAR].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["mass"].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["mass"][RecordComponent::SCALAR].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["momentum"].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["momentum"]["x"].parent == getWritable(&o.iterations[100].particles["electrons"]["momentum"])); + REQUIRE(o.iterations[100].particles["electrons"]["momentum"]["y"].parent == getWritable(&o.iterations[100].particles["electrons"]["momentum"])); + REQUIRE(o.iterations[100].particles["electrons"]["momentum"]["z"].parent == getWritable(&o.iterations[100].particles["electrons"]["momentum"])); + REQUIRE(o.iterations[100].particles["electrons"]["position"].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["position"]["x"].parent == getWritable(&o.iterations[100].particles["electrons"]["position"])); + REQUIRE(o.iterations[100].particles["electrons"]["position"]["y"].parent == getWritable(&o.iterations[100].particles["electrons"]["position"])); + REQUIRE(o.iterations[100].particles["electrons"]["position"]["z"].parent == getWritable(&o.iterations[100].particles["electrons"]["position"])); + REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"]["x"].parent == getWritable(&o.iterations[100].particles["electrons"]["positionOffset"])); + REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"]["y"].parent == getWritable(&o.iterations[100].particles["electrons"]["positionOffset"])); + REQUIRE(o.iterations[100].particles["electrons"]["positionOffset"]["z"].parent == getWritable(&o.iterations[100].particles["electrons"]["positionOffset"])); + REQUIRE(o.iterations[100].particles["electrons"]["weighting"].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE(o.iterations[100].particles["electrons"]["weighting"][RecordComponent::SCALAR].parent == getWritable(&o.iterations[100].particles["electrons"])); + REQUIRE_THROWS_AS(o.iterations[100].particles["electrons"]["numberOfLegs"], std::out_of_range); + REQUIRE_THROWS_AS(o.iterations[100].particles["apples"], std::out_of_range); - REQUIRE(e_position.size() == 3); - REQUIRE(e_position.count("x") == 1); - REQUIRE(e_position.count("y") == 1); - REQUIRE(e_position.count("z") == 1); + int32_t i32 = 32; + REQUIRE_THROWS(o.setAttribute("setAttributeFail", i32)); + } catch (no_such_file_error& e) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } +#else + std::cerr << "Invasive tests not enabled. Hierarchy is not visible.\n"; +#endif +} - RecordComponent& e_position_x = e_position["x"]; - REQUIRE(e_position_x.unitSI() == 2.599999993753294e-07); - REQUIRE(e_position_x.getDatatype() == Datatype::FLOAT); - REQUIRE(e_position_x.getExtent() == e); - REQUIRE(e_position_x.getDimensionality() == 1); +TEST_CASE( "git_hdf5_sample_attribute_test", "[serial][hdf5]" ) +{ + try + { + Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); - RecordComponent& e_position_y = e_position["y"]; - REQUIRE(e_position_y.unitSI() == 4.4299999435019118e-08); - REQUIRE(e_position_y.getDatatype() == Datatype::FLOAT); - REQUIRE(e_position_y.getExtent() == e); - REQUIRE(e_position_y.getDimensionality() == 1); + REQUIRE(o.openPMD() == "1.1.0"); + REQUIRE(o.openPMDextension() == 1); + REQUIRE(o.basePath() == "/data/%T/"); + REQUIRE(o.meshesPath() == "fields/"); + REQUIRE(o.particlesPath() == "particles/"); + REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); + REQUIRE(o.iterationFormat() == "data%T.h5"); + REQUIRE(o.name() == "data%T"); - RecordComponent& e_position_z = e_position["z"]; - REQUIRE(e_position_z.unitSI() == 2.599999993753294e-07); - REQUIRE(e_position_z.getDatatype() == Datatype::FLOAT); - REQUIRE(e_position_z.getExtent() == e); - REQUIRE(e_position_z.getDimensionality() == 1); + REQUIRE(o.iterations.size() == 5); + REQUIRE(o.iterations.count(100) == 1); - ud = {{1., 0., 0., 0., 0., 0., 0.}}; - Record& e_positionOffset = species_e["positionOffset"]; - REQUIRE(e_positionOffset.unitDimension() == ud); - REQUIRE(e_positionOffset.timeOffset< float >() == static_cast< float >(0.0f)); + Iteration& iteration_100 = o.iterations[100]; + REQUIRE(iteration_100.time< double >() == 3.2847121452090077e-14); + REQUIRE(iteration_100.dt< double >() == 3.2847121452090093e-16); + REQUIRE(iteration_100.timeUnitSI() == 1.0); - REQUIRE(e_positionOffset.size() == 3); - REQUIRE(e_positionOffset.count("x") == 1); - REQUIRE(e_positionOffset.count("y") == 1); - REQUIRE(e_positionOffset.count("z") == 1); + REQUIRE(iteration_100.meshes.size() == 2); + REQUIRE(iteration_100.meshes.count("E") == 1); + REQUIRE(iteration_100.meshes.count("rho") == 1); - RecordComponent& e_positionOffset_x = e_positionOffset["x"]; - REQUIRE(e_positionOffset_x.unitSI() == 2.599999993753294e-07); - REQUIRE(e_positionOffset_x.getDatatype() == determineDatatype< int32_t >()); - REQUIRE(e_positionOffset_x.getExtent() == e); - REQUIRE(e_positionOffset_x.getDimensionality() == 1); + std::vector< std::string > al{"x", "y", "z"}; + std::vector< double > gs{8.0000000000000007e-07, + 8.0000000000000007e-07, + 1.0000000000000001e-07}; + std::vector< double > ggo{-1.0000000000000001e-05, + -1.0000000000000001e-05, + -5.1999999999999993e-06}; + std::array< double, 7 > ud{{1., 1., -3., -1., 0., 0., 0.}}; + Mesh& E = iteration_100.meshes["E"]; + REQUIRE(E.geometry() == Mesh::Geometry::cartesian); + REQUIRE(E.dataOrder() == Mesh::DataOrder::C); + REQUIRE(E.axisLabels() == al); + REQUIRE(E.gridSpacing< double >() == gs); + REQUIRE(E.gridGlobalOffset() == ggo); + REQUIRE(E.gridUnitSI() == 1.0); + REQUIRE(E.unitDimension() == ud); + REQUIRE(E.timeOffset< double >() == 0.0); - RecordComponent& e_positionOffset_y = e_positionOffset["y"]; - REQUIRE(e_positionOffset_y.unitSI() == 4.4299999435019118e-08); - REQUIRE(e_positionOffset_y.getDatatype() == determineDatatype< int32_t >()); - REQUIRE(e_positionOffset_y.getExtent() == e); - REQUIRE(e_positionOffset_y.getDimensionality() == 1); + REQUIRE(E.size() == 3); + REQUIRE(E.count("x") == 1); + REQUIRE(E.count("y") == 1); + REQUIRE(E.count("z") == 1); - RecordComponent& e_positionOffset_z = e_positionOffset["z"]; - REQUIRE(e_positionOffset_z.unitSI() == 2.599999993753294e-07); - REQUIRE(e_positionOffset_z.getDatatype() == determineDatatype< int32_t >()); - REQUIRE(e_positionOffset_z.getExtent() == e); - REQUIRE(e_positionOffset_z.getDimensionality() == 1); + std::vector< double > p{0.5, 0., 0.}; + Extent e{26, 26, 201}; + MeshRecordComponent& E_x = E["x"]; + REQUIRE(E_x.unitSI() == 1.0); + REQUIRE(E_x.position< double >() == p); + REQUIRE(E_x.getDatatype() == Datatype::DOUBLE); + REQUIRE(E_x.getExtent() == e); + REQUIRE(E_x.getDimensionality() == 3); - ud = {{0., 0., 0., 0., 0., 0., 0.}}; - Record& e_weighting = species_e["weighting"]; - REQUIRE(e_weighting.unitDimension() == ud); - REQUIRE(e_weighting.timeOffset< float >() == static_cast< float >(0.0f)); - - REQUIRE(e_weighting.size() == 1); - REQUIRE(e_weighting.count(RecordComponent::SCALAR) == 1); + p = {0., 0.5, 0.}; + MeshRecordComponent& E_y = E["y"]; + REQUIRE(E_y.unitSI() == 1.0); + REQUIRE(E_y.position< double >() == p); + REQUIRE(E_y.getDatatype() == Datatype::DOUBLE); + REQUIRE(E_y.getExtent() == e); + REQUIRE(E_y.getDimensionality() == 3); - RecordComponent& e_weighting_scalar = e_weighting[RecordComponent::SCALAR]; - REQUIRE(e_weighting_scalar.unitSI() == 1.0); - REQUIRE(e_weighting_scalar.getDatatype() == Datatype::FLOAT); - REQUIRE(e_weighting_scalar.getExtent() == e); - REQUIRE(e_weighting_scalar.getDimensionality() == 1); + p = {0., 0., 0.5}; + MeshRecordComponent& E_z = E["z"]; + REQUIRE(E_z.unitSI() == 1.0); + REQUIRE(E_z.position< double >() == p); + REQUIRE(E_z.getDatatype() == Datatype::DOUBLE); + REQUIRE(E_z.getExtent() == e); + REQUIRE(E_z.getDimensionality() == 3); - ParticlePatches& e_patches = species_e.particlePatches; - REQUIRE(e_patches.size() == 4); /* extent, numParticles, numParticlesOffset, offset */ - REQUIRE(e_patches.count("extent") == 1); - REQUIRE(e_patches.count("numParticles") == 1); - REQUIRE(e_patches.count("numParticlesOffset") == 1); - REQUIRE(e_patches.count("offset") == 1); - REQUIRE(e_patches.numPatches() == 4); + gs = {8.0000000000000007e-07, + 8.0000000000000007e-07, + 1.0000000000000001e-07}; + ggo = {-1.0000000000000001e-05, + -1.0000000000000001e-05, + -5.1999999999999993e-06}; + ud = {{-3., 0., 1., 1., 0., 0., 0.}}; + Mesh& rho = iteration_100.meshes["rho"]; + REQUIRE(rho.geometry() == Mesh::Geometry::cartesian); + REQUIRE(rho.dataOrder() == Mesh::DataOrder::C); + REQUIRE(rho.axisLabels() == al); + REQUIRE(rho.gridSpacing< double >() == gs); + REQUIRE(rho.gridGlobalOffset() == ggo); + REQUIRE(rho.gridUnitSI() == 1.0); + REQUIRE(rho.unitDimension() == ud); + REQUIRE(rho.timeOffset< double >() == 0.0); - ud = {{1., 0., 0., 0., 0., 0., 0.}}; - PatchRecord& e_extent = e_patches["extent"]; - REQUIRE(e_extent.unitDimension() == ud); + REQUIRE(rho.size() == 1); + REQUIRE(rho.count(MeshRecordComponent::SCALAR) == 1); - REQUIRE(e_extent.size() == 3); - REQUIRE(e_extent.count("x") == 1); - REQUIRE(e_extent.count("y") == 1); - REQUIRE(e_extent.count("z") == 1); + p = {0., 0., 0.}; + e = {26, 26, 201}; + MeshRecordComponent& rho_scalar = rho[MeshRecordComponent::SCALAR]; + REQUIRE(rho_scalar.unitSI() == 1.0); + REQUIRE(rho_scalar.position< double >() == p); + REQUIRE(rho_scalar.getDatatype() == Datatype::DOUBLE); + REQUIRE(rho_scalar.getExtent() == e); + REQUIRE(rho_scalar.getDimensionality() == 3); - PatchRecordComponent& e_extent_x = e_extent["x"]; - REQUIRE(e_extent_x.unitSI() == 2.599999993753294e-07); -#if !defined(_MSC_VER) - REQUIRE(e_extent_x.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_extent_x.getDatatype(), determineDatatype< uint64_t >())); + REQUIRE(iteration_100.particles.size() == 1); + REQUIRE(iteration_100.particles.count("electrons") == 1); - PatchRecordComponent& e_extent_y = e_extent["y"]; - REQUIRE(e_extent_y.unitSI() == 4.429999943501912e-08); -#if !defined(_MSC_VER) - REQUIRE(e_extent_y.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_extent_y.getDatatype(), determineDatatype< uint64_t >())); + ParticleSpecies& electrons = iteration_100.particles["electrons"]; - PatchRecordComponent& e_extent_z = e_extent["z"]; - REQUIRE(e_extent_z.unitSI() == 2.599999993753294e-07); -#if !defined(_MSC_VER) - REQUIRE(e_extent_z.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_extent_z.getDatatype(), determineDatatype< uint64_t >())); + REQUIRE(electrons.size() == 6); + REQUIRE(electrons.count("charge") == 1); + REQUIRE(electrons.count("mass") == 1); + REQUIRE(electrons.count("momentum") == 1); + REQUIRE(electrons.count("position") == 1); + REQUIRE(electrons.count("positionOffset") == 1); + REQUIRE(electrons.count("weighting") == 1); - std::vector< uint64_t > data( e_patches.size() ); - e_extent_z.load(shareRaw(data.data())); - o.flush(); - REQUIRE(data.at(0) == static_cast< uint64_t >(80)); - REQUIRE(data.at(1) == static_cast< uint64_t >(80)); - REQUIRE(data.at(2) == static_cast< uint64_t >(80)); - REQUIRE(data.at(3) == static_cast< uint64_t >(80)); + ud = {{0., 0., 1., 1., 0., 0., 0.}}; + Record& charge = electrons["charge"]; + REQUIRE(charge.unitDimension() == ud); + REQUIRE(charge.timeOffset< double >() == 0.0); - PatchRecord& e_numParticles = e_patches["numParticles"]; - REQUIRE(e_numParticles.size() == 1); - REQUIRE(e_numParticles.count(RecordComponent::SCALAR) == 1); + REQUIRE(charge.size() == 1); + REQUIRE(charge.count(RecordComponent::SCALAR) == 1); - PatchRecordComponent& e_numParticles_scalar = e_numParticles[RecordComponent::SCALAR]; -#if !defined(_MSC_VER) - REQUIRE(e_numParticles_scalar.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_numParticles_scalar.getDatatype(), determineDatatype< uint64_t >())); + e = {85625}; + RecordComponent& charge_scalar = charge[RecordComponent::SCALAR]; + REQUIRE(charge_scalar.unitSI() == 1.0); + REQUIRE(charge_scalar.getDatatype() == Datatype::DOUBLE); + REQUIRE(charge_scalar.getDimensionality() == 1); + REQUIRE(charge_scalar.getExtent() == e); - e_numParticles_scalar.load(shareRaw(data.data())); - o.flush(); - REQUIRE(data.at(0) == static_cast< uint64_t >(512000)); - REQUIRE(data.at(1) == static_cast< uint64_t >(819200)); - REQUIRE(data.at(2) == static_cast< uint64_t >(819200)); - REQUIRE(data.at(3) == static_cast< uint64_t >(0)); + ud = {{1., 0., 0., 0., 0., 0., 0.}}; + Record& mass = electrons["mass"]; + REQUIRE(mass.unitDimension() == ud); + REQUIRE(mass.timeOffset< double >() == 0.0); - PatchRecord& e_numParticlesOffset = e_patches["numParticlesOffset"]; - REQUIRE(e_numParticlesOffset.size() == 1); - REQUIRE(e_numParticlesOffset.count(RecordComponent::SCALAR) == 1); + REQUIRE(mass.size() == 1); + REQUIRE(mass.count(RecordComponent::SCALAR) == 1); - PatchRecordComponent& e_numParticlesOffset_scalar = e_numParticlesOffset[RecordComponent::SCALAR]; -#if !defined(_MSC_VER) - REQUIRE(e_numParticlesOffset_scalar.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_numParticlesOffset_scalar.getDatatype(), determineDatatype< uint64_t >())); + RecordComponent& mass_scalar = mass[RecordComponent::SCALAR]; + REQUIRE(mass_scalar.unitSI() == 1.0); + REQUIRE(mass_scalar.getDatatype() == Datatype::DOUBLE); + REQUIRE(mass_scalar.getDimensionality() == 1); + REQUIRE(mass_scalar.getExtent() == e); - PatchRecord& e_offset = e_patches["offset"]; - REQUIRE(e_offset.unitDimension() == ud); + ud = {{1., 1., -1., 0., 0., 0., 0.}}; + Record& momentum = electrons["momentum"]; + REQUIRE(momentum.unitDimension() == ud); + REQUIRE(momentum.timeOffset< double >() == 0.0); - REQUIRE(e_offset.size() == 3); - REQUIRE(e_offset.count("x") == 1); - REQUIRE(e_offset.count("y") == 1); - REQUIRE(e_offset.count("z") == 1); + REQUIRE(momentum.size() == 3); + REQUIRE(momentum.count("x") == 1); + REQUIRE(momentum.count("y") == 1); + REQUIRE(momentum.count("z") == 1); - PatchRecordComponent& e_offset_x = e_offset["x"]; - REQUIRE(e_offset_x.unitSI() == 2.599999993753294e-07); -#if !defined(_MSC_VER) - REQUIRE(e_offset_x.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_offset_x.getDatatype(), determineDatatype< uint64_t >())); + RecordComponent& momentum_x = momentum["x"]; + REQUIRE(momentum_x.unitSI() == 1.0); + REQUIRE(momentum_x.getDatatype() == Datatype::DOUBLE); + REQUIRE(momentum_x.getDimensionality() == 1); + REQUIRE(momentum_x.getExtent() == e); - PatchRecordComponent& e_offset_y = e_offset["y"]; - REQUIRE(e_offset_y.unitSI() == 4.429999943501912e-08); -#if !defined(_MSC_VER) - REQUIRE(e_offset_y.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_offset_y.getDatatype(), determineDatatype< uint64_t >())); + RecordComponent& momentum_y = momentum["y"]; + REQUIRE(momentum_y.unitSI() == 1.0); + REQUIRE(momentum_y.getDatatype() == Datatype::DOUBLE); + REQUIRE(momentum_y.getDimensionality() == 1); + REQUIRE(momentum_y.getExtent() == e); - e_offset_y.load(shareRaw(data.data())); - o.flush(); - REQUIRE(data.at(0) == static_cast< uint64_t >(0)); - REQUIRE(data.at(1) == static_cast< uint64_t >(128)); - REQUIRE(data.at(2) == static_cast< uint64_t >(256)); - REQUIRE(data.at(3) == static_cast< uint64_t >(384)); + RecordComponent& momentum_z = momentum["z"]; + REQUIRE(momentum_z.unitSI() == 1.0); + REQUIRE(momentum_z.getDatatype() == Datatype::DOUBLE); + REQUIRE(momentum_z.getDimensionality() == 1); + REQUIRE(momentum_z.getExtent() == e); - PatchRecordComponent& e_offset_z = e_offset["z"]; - REQUIRE(e_offset_z.unitSI() == 2.599999993753294e-07); -#if !defined(_MSC_VER) - REQUIRE(e_offset_z.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(e_offset_z.getDatatype(), determineDatatype< uint64_t >())); - } catch (no_such_file_error& e) - { - std::cerr << "HZDR sample not accessible. (" << e.what() << ")\n"; - return; - } -} + ud = {{1., 0., 0., 0., 0., 0., 0.}}; + Record& position = electrons["position"]; + REQUIRE(position.unitDimension() == ud); + REQUIRE(position.timeOffset< double >() == 0.0); -TEST_CASE( "hdf5_dtype_test", "[serial][hdf5]" ) -{ - { - Series s = Series("../samples/dtype_test.h5", AccessType::CREATE); + REQUIRE(position.size() == 3); + REQUIRE(position.count("x") == 1); + REQUIRE(position.count("y") == 1); + REQUIRE(position.count("z") == 1); - char c = 'c'; - s.setAttribute("char", c); - unsigned char uc = 'u'; - s.setAttribute("uchar", uc); - int16_t i16 = 16; - s.setAttribute("int16", i16); - int32_t i32 = 32; - s.setAttribute("int32", i32); - int64_t i64 = 64; - s.setAttribute("int64", i64); - uint16_t u16 = 16u; - s.setAttribute("uint16", u16); - uint32_t u32 = 32u; - s.setAttribute("uint32", u32); - uint64_t u64 = 64u; - s.setAttribute("uint64", u64); - float f = 16.e10f; - s.setAttribute("float", f); - double d = 1.e64; - s.setAttribute("double", d); - long double ld = 1.e80L; - s.setAttribute("longdouble", ld); - std::string str = "string"; - s.setAttribute("string", str); - s.setAttribute("vecChar", std::vector< char >({'c', 'h', 'a', 'r'})); - s.setAttribute("vecInt16", std::vector< int16_t >({32766, 32767})); - s.setAttribute("vecInt32", std::vector< int32_t >({2147483646, 2147483647})); - s.setAttribute("vecInt64", std::vector< int64_t >({9223372036854775806, 9223372036854775807})); - s.setAttribute("vecUchar", std::vector< char >({'u', 'c', 'h', 'a', 'r'})); - s.setAttribute("vecUint16", std::vector< uint16_t >({65534u, 65535u})); - s.setAttribute("vecUint32", std::vector< uint32_t >({4294967294u, 4294967295u})); - s.setAttribute("vecUint64", std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); - s.setAttribute("vecFloat", std::vector< float >({0.f, 3.40282e+38f})); - s.setAttribute("vecDouble", std::vector< double >({0., 1.79769e+308})); - s.setAttribute("vecLongdouble", std::vector< long double >({0.L, std::numeric_limits::max()})); - s.setAttribute("vecString", std::vector< std::string >({"vector", "of", "strings"})); - s.setAttribute("bool", true); - s.setAttribute("boolF", false); + RecordComponent& position_x = position["x"]; + REQUIRE(position_x.unitSI() == 1.0); + REQUIRE(position_x.getDatatype() == Datatype::DOUBLE); + REQUIRE(position_x.getDimensionality() == 1); + REQUIRE(position_x.getExtent() == e); - // non-fixed size integer types - short ss = 16; - s.setAttribute("short", ss); - int si = 32; - s.setAttribute("int", si); - long sl = 64; - s.setAttribute("long", sl); - long long sll = 128; - s.setAttribute("longlong", sll); - unsigned short us = 16u; - s.setAttribute("ushort", us); - unsigned int ui = 32u; - s.setAttribute("uint", ui); - unsigned long ul = 64u; - s.setAttribute("ulong", ul); - unsigned long long ull = 128u; - s.setAttribute("ulonglong", ull); - s.setAttribute("vecShort", std::vector< short >({32766, 32767})); - s.setAttribute("vecInt", std::vector< int >({32766, 32767})); - s.setAttribute("vecLong", std::vector< long >({2147483646, 2147483647})); - s.setAttribute("vecLongLong", std::vector< long long >({2147483644, 2147483643})); - s.setAttribute("vecUShort", std::vector< unsigned short >({65534u, 65535u})); - s.setAttribute("vecUInt", std::vector< unsigned int >({65533u, 65531u})); - s.setAttribute("vecULong", std::vector< unsigned long >({65532u, 65530u})); - s.setAttribute("vecULongLong", std::vector< unsigned long long >({65531u, 65529u})); - } + RecordComponent& position_y = position["y"]; + REQUIRE(position_y.unitSI() == 1.0); + REQUIRE(position_y.getDatatype() == Datatype::DOUBLE); + REQUIRE(position_y.getDimensionality() == 1); + REQUIRE(position_y.getExtent() == e); - Series s = Series("../samples/dtype_test.h5", AccessType::READ_ONLY); + RecordComponent& position_z = position["z"]; + REQUIRE(position_z.unitSI() == 1.0); + REQUIRE(position_z.getDatatype() == Datatype::DOUBLE); + REQUIRE(position_z.getDimensionality() == 1); + REQUIRE(position_z.getExtent() == e); - REQUIRE(s.getAttribute("char").get< char >() == 'c'); - REQUIRE(s.getAttribute("uchar").get< unsigned char >() == 'u'); - REQUIRE(s.getAttribute("int16").get< int16_t >() == 16); - REQUIRE(s.getAttribute("int32").get< int32_t >() == 32); - REQUIRE(s.getAttribute("int64").get< int64_t >() == 64); - REQUIRE(s.getAttribute("uint16").get< uint16_t >() == 16u); - REQUIRE(s.getAttribute("uint32").get< uint32_t >() == 32u); - REQUIRE(s.getAttribute("uint64").get< uint64_t >() == 64u); - REQUIRE(s.getAttribute("float").get< float >() == 16.e10f); - REQUIRE(s.getAttribute("double").get< double >() == 1.e64); - REQUIRE(s.getAttribute("longdouble").get< long double >() == 1.e80L); - REQUIRE(s.getAttribute("string").get< std::string >() == "string"); - REQUIRE(s.getAttribute("vecChar").get< std::vector< char > >() == std::vector< char >({'c', 'h', 'a', 'r'})); - REQUIRE(s.getAttribute("vecInt16").get< std::vector< int16_t > >() == std::vector< int16_t >({32766, 32767})); - REQUIRE(s.getAttribute("vecInt32").get< std::vector< int32_t > >() == std::vector< int32_t >({2147483646, 2147483647})); - REQUIRE(s.getAttribute("vecInt64").get< std::vector< int64_t > >() == std::vector< int64_t >({9223372036854775806, 9223372036854775807})); - REQUIRE(s.getAttribute("vecUchar").get< std::vector< char > >() == std::vector< char >({'u', 'c', 'h', 'a', 'r'})); - REQUIRE(s.getAttribute("vecUint16").get< std::vector< uint16_t > >() == std::vector< uint16_t >({65534u, 65535u})); - REQUIRE(s.getAttribute("vecUint32").get< std::vector< uint32_t > >() == std::vector< uint32_t >({4294967294u, 4294967295u})); - REQUIRE(s.getAttribute("vecUint64").get< std::vector< uint64_t > >() == std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); - REQUIRE(s.getAttribute("vecFloat").get< std::vector< float > >() == std::vector< float >({0.f, 3.40282e+38f})); - REQUIRE(s.getAttribute("vecDouble").get< std::vector< double > >() == std::vector< double >({0., 1.79769e+308})); - REQUIRE(s.getAttribute("vecLongdouble").get< std::vector< long double > >() == std::vector< long double >({0.L, std::numeric_limits::max()})); - REQUIRE(s.getAttribute("vecString").get< std::vector< std::string > >() == std::vector< std::string >({"vector", "of", "strings"})); - REQUIRE(s.getAttribute("bool").get< bool >() == true); - REQUIRE(s.getAttribute("boolF").get< bool >() == false); + Record& positionOffset = electrons["positionOffset"]; + REQUIRE(positionOffset.unitDimension() == ud); + REQUIRE(positionOffset.timeOffset< double >() == 0.0); - // same implementation types (not necessary aliases) detection -#if !defined(_MSC_VER) - REQUIRE(s.getAttribute("short").dtype == Datatype::SHORT); - REQUIRE(s.getAttribute("int").dtype == Datatype::INT); - REQUIRE(s.getAttribute("long").dtype == Datatype::LONG); - REQUIRE(s.getAttribute("longlong").dtype == Datatype::LONGLONG); - REQUIRE(s.getAttribute("ushort").dtype == Datatype::USHORT); - REQUIRE(s.getAttribute("uint").dtype == Datatype::UINT); - REQUIRE(s.getAttribute("ulong").dtype == Datatype::ULONG); - REQUIRE(s.getAttribute("ulonglong").dtype == Datatype::ULONGLONG); + REQUIRE(positionOffset.size() == 3); + REQUIRE(positionOffset.count("x") == 1); + REQUIRE(positionOffset.count("y") == 1); + REQUIRE(positionOffset.count("z") == 1); - REQUIRE(s.getAttribute("vecShort").dtype == Datatype::VEC_SHORT); - REQUIRE(s.getAttribute("vecInt").dtype == Datatype::VEC_INT); - REQUIRE(s.getAttribute("vecLong").dtype == Datatype::VEC_LONG); - REQUIRE(s.getAttribute("vecLongLong").dtype == Datatype::VEC_LONGLONG); - REQUIRE(s.getAttribute("vecUShort").dtype == Datatype::VEC_USHORT); - REQUIRE(s.getAttribute("vecUInt").dtype == Datatype::VEC_UINT); - REQUIRE(s.getAttribute("vecULong").dtype == Datatype::VEC_ULONG); - REQUIRE(s.getAttribute("vecULongLong").dtype == Datatype::VEC_ULONGLONG); -#endif - REQUIRE(isSame(s.getAttribute("short").dtype, Datatype::SHORT)); - REQUIRE(isSame(s.getAttribute("int").dtype, Datatype::INT)); - REQUIRE(isSame(s.getAttribute("long").dtype, Datatype::LONG)); - REQUIRE(isSame(s.getAttribute("longlong").dtype, Datatype::LONGLONG)); - REQUIRE(isSame(s.getAttribute("ushort").dtype, Datatype::USHORT)); - REQUIRE(isSame(s.getAttribute("uint").dtype, Datatype::UINT)); - REQUIRE(isSame(s.getAttribute("ulong").dtype, Datatype::ULONG)); - REQUIRE(isSame(s.getAttribute("ulonglong").dtype, Datatype::ULONGLONG)); + RecordComponent& positionOffset_x = positionOffset["x"]; + REQUIRE(positionOffset_x.unitSI() == 1.0); + REQUIRE(positionOffset_x.getDatatype() == Datatype::DOUBLE); + REQUIRE(positionOffset_x.getDimensionality() == 1); + REQUIRE(positionOffset_x.getExtent() == e); - REQUIRE(isSame(s.getAttribute("vecShort").dtype, Datatype::VEC_SHORT)); - REQUIRE(isSame(s.getAttribute("vecInt").dtype, Datatype::VEC_INT)); - REQUIRE(isSame(s.getAttribute("vecLong").dtype, Datatype::VEC_LONG)); - REQUIRE(isSame(s.getAttribute("vecLongLong").dtype, Datatype::VEC_LONGLONG)); - REQUIRE(isSame(s.getAttribute("vecUShort").dtype, Datatype::VEC_USHORT)); - REQUIRE(isSame(s.getAttribute("vecUInt").dtype, Datatype::VEC_UINT)); - REQUIRE(isSame(s.getAttribute("vecULong").dtype, Datatype::VEC_ULONG)); - REQUIRE(isSame(s.getAttribute("vecULongLong").dtype, Datatype::VEC_ULONGLONG)); -} + RecordComponent& positionOffset_y = positionOffset["y"]; + REQUIRE(positionOffset_y.unitSI() == 1.0); + REQUIRE(positionOffset_y.getDatatype() == Datatype::DOUBLE); + REQUIRE(positionOffset_y.getDimensionality() == 1); + REQUIRE(positionOffset_y.getExtent() == e); -TEST_CASE( "hdf5_write_test", "[serial][hdf5]" ) -{ - Series o = Series("../samples/serial_write.h5", AccessType::CREATE); + RecordComponent& positionOffset_z = positionOffset["z"]; + REQUIRE(positionOffset_z.unitSI() == 1.0); + REQUIRE(positionOffset_z.getDatatype() == Datatype::DOUBLE); + REQUIRE(positionOffset_z.getDimensionality() == 1); + REQUIRE(positionOffset_z.getExtent() == e); - o.setAuthor("Serial HDF5"); - ParticleSpecies& e = o.iterations[1].particles["e"]; + ud = {{0., 0., 0., 0., 0., 0., 0.}}; + Record& weighting = electrons["weighting"]; + REQUIRE(weighting.unitDimension() == ud); + REQUIRE(weighting.timeOffset< double >() == 0.0); - std::vector< double > position_global(4); - double pos{0.}; - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::vector< double > position_local = {0.}; - e["position"]["x"].resetDataset(Dataset(determineDatatype(), {4})); - std::vector< uint64_t > positionOffset_global(4); - uint64_t posOff{0}; - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::array< uint64_t, 1 > positionOffset_local = {{ 0u }}; - auto dataset = Dataset(determineDatatype(), {4}); - REQUIRE_THROWS_AS(dataset.setCompression("zlib", 10), std::runtime_error); - dataset.setCompression("zlib", 9); - e["positionOffset"]["x"].resetDataset(dataset); + REQUIRE(weighting.size() == 1); + REQUIRE(weighting.count(RecordComponent::SCALAR) == 1); - for( uint64_t i = 0; i < 4; ++i ) + RecordComponent& weighting_scalar = weighting[RecordComponent::SCALAR]; + REQUIRE(weighting_scalar.unitSI() == 1.0); + REQUIRE(weighting_scalar.getDatatype() == Datatype::DOUBLE); + REQUIRE(weighting_scalar.getDimensionality() == 1); + REQUIRE(weighting_scalar.getExtent() == e); + } catch (no_such_file_error& e) { - position_local.at(0) = position_global[i]; - e["position"]["x"].storeChunk(shareRaw(position_local), {i}, {1}); - positionOffset_local[0] = positionOffset_global[i]; - e["positionOffset"]["x"].storeChunk(shareRaw(positionOffset_local), {i}, {1}); - o.flush(); + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; } - - //TODO close file, read back, verify } -TEST_CASE( "hdf5_fileBased_write_empty_test", "[serial][hdf5]" ) +TEST_CASE( "git_hdf5_sample_content_test", "[serial][hdf5]" ) { - if( auxiliary::directory_exists("../samples/subdir") ) - auxiliary::remove_directory("../samples/subdir"); - - Dataset dset = Dataset(Datatype::DOUBLE, {2}); + try { - Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::CREATE); + Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); - ParticleSpecies& e_1 = o.iterations[1].particles["e"]; - e_1["position"][RecordComponent::SCALAR].resetDataset(dset); - e_1["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[1].setTime(1.f); - ParticleSpecies& e_2 = o.iterations[2].particles["e"]; - e_2["position"][RecordComponent::SCALAR].resetDataset(dset); - e_2["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[2].setTime(2.f); - ParticleSpecies& e_3 = o.iterations[3].particles["e"]; - e_3["position"][RecordComponent::SCALAR].resetDataset(dset); - e_3["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[3].setTime(3.f); - } + { + double actual[3][3][3] = {{{-1.9080703683727052e-09, -1.5632650729457964e-10, 1.1497536256399599e-09}, + {-1.9979540244463578e-09, -2.5512036927466397e-10, 1.0402234629225404e-09}, + {-1.7353589676361025e-09, -8.0899198451334087e-10, -1.6443779671249104e-10}}, - REQUIRE(auxiliary::directory_exists("../samples/subdir")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write1.h5")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write2.h5")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write3.h5")); + {{-2.0029988778702545e-09, -1.9543477947081556e-10, 1.0916454407094989e-09}, + {-2.3890367462087170e-09, -4.7158010829662089e-10, 9.0026075483251589e-10}, + {-1.9033881137886510e-09, -7.5192119197708962e-10, 5.0038861942880430e-10}}, - { - Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_ONLY); + {{-1.3271805876513554e-09, -5.9243276950837753e-10, -2.2445734160214670e-10}, + {-7.4578609954301101e-10, -1.1995737736469891e-10, 2.5611823772919706e-10}, + {-9.4806251738077663e-10, -1.5472800818372434e-10, -3.6461900165818406e-10}}}; + MeshRecordComponent& rho = o.iterations[100].meshes["rho"][MeshRecordComponent::SCALAR]; + Offset offset{20, 20, 190}; + Extent extent{3, 3, 3}; + auto data = rho.loadChunk(offset, extent); + o.flush(); + double* raw_ptr = data.get(); - REQUIRE(o.iterations.size() == 3); - REQUIRE(o.iterations.count(1) == 1); - REQUIRE(o.iterations.count(2) == 1); - REQUIRE(o.iterations.count(3) == 1); + for( int i = 0; i < 3; ++i ) + for( int j = 0; j < 3; ++j ) + for( int k = 0; k < 3; ++k ) + REQUIRE(raw_ptr[((i*3) + j)*3 + k] == actual[i][j][k]); + } - REQUIRE(o.iterations[1].time< float >() == 1.f); - REQUIRE(o.iterations[2].time< float >() == 2.f); - REQUIRE(o.iterations[3].time< float >() == 3.f); + { + double constant_value = 9.1093829099999999e-31; + RecordComponent& electrons_mass = o.iterations[100].particles["electrons"]["mass"][RecordComponent::SCALAR]; + Offset offset{15}; + Extent extent{3}; + auto data = electrons_mass.loadChunk(offset, extent); + o.flush(); + double* raw_ptr = data.get(); - REQUIRE(o.iterations[1].particles.size() == 1); - REQUIRE(o.iterations[1].particles.count("e") == 1); - REQUIRE(o.iterations[2].particles.size() == 1); - REQUIRE(o.iterations[2].particles.count("e") == 1); - REQUIRE(o.iterations[3].particles.size() == 1); - REQUIRE(o.iterations[3].particles.count("e") == 1); + for( int i = 0; i < 3; ++i ) + REQUIRE(raw_ptr[i] == constant_value); + } + } catch (no_such_file_error& e) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } +} - REQUIRE(o.iterations[1].particles["e"].size() == 2); - REQUIRE(o.iterations[1].particles["e"].count("position") == 1); - REQUIRE(o.iterations[1].particles["e"].count("positionOffset") == 1); - REQUIRE(o.iterations[2].particles["e"].size() == 2); - REQUIRE(o.iterations[2].particles["e"].count("position") == 1); - REQUIRE(o.iterations[2].particles["e"].count("positionOffset") == 1); - REQUIRE(o.iterations[3].particles["e"].size() == 2); - REQUIRE(o.iterations[3].particles["e"].count("position") == 1); - REQUIRE(o.iterations[3].particles["e"].count("positionOffset") == 1); +TEST_CASE( "git_hdf5_sample_fileBased_read_test", "[serial][hdf5]" ) +{ + try + { + Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_ONLY); - REQUIRE(o.iterations[1].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[1].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[1].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[1].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); - REQUIRE(o.iterations[2].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[2].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[2].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[2].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); - REQUIRE(o.iterations[3].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[3].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[3].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[3].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); - } + REQUIRE(o.iterations.size() == 5); + REQUIRE(o.iterations.count(100) == 1); + REQUIRE(o.iterations.count(200) == 1); + REQUIRE(o.iterations.count(300) == 1); + REQUIRE(o.iterations.count(400) == 1); + REQUIRE(o.iterations.count(500) == 1); +#if openPMD_USE_INVASIVE_TESTS + REQUIRE(*o.m_filenamePadding == 8); +#endif + } catch (no_such_file_error& e) { - Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_WRITE); - ParticleSpecies& e_4 = o.iterations[4].particles["e"]; - e_4["position"][RecordComponent::SCALAR].resetDataset(dset); - e_4["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[4].setTime(4.f); + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; } + try { - Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_ONLY); - - REQUIRE(o.iterations.size() == 4); - REQUIRE(o.iterations.count(4) == 1); - - REQUIRE(o.iterations[4].time< float >() == 4.f); - - REQUIRE(o.iterations[4].particles.size() == 1); - REQUIRE(o.iterations[4].particles.count("e") == 1); + Series o = Series("../samples/git-sample/data%08T.h5", AccessType::READ_ONLY); - REQUIRE(o.iterations[4].particles["e"].size() == 2); - REQUIRE(o.iterations[4].particles["e"].count("position") == 1); - REQUIRE(o.iterations[4].particles["e"].count("positionOffset") == 1); + REQUIRE(o.iterations.size() == 5); + REQUIRE(o.iterations.count(100) == 1); + REQUIRE(o.iterations.count(200) == 1); + REQUIRE(o.iterations.count(300) == 1); + REQUIRE(o.iterations.count(400) == 1); + REQUIRE(o.iterations.count(500) == 1); - REQUIRE(o.iterations[4].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[4].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[4].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[4].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); +#if openPMD_USE_INVASIVE_TESTS + REQUIRE(*o.m_filenamePadding == 8); +#endif + } catch (no_such_file_error& e) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; } -} -TEST_CASE( "hdf5_fileBased_write_test", "[serial][hdf5]" ) -{ - if( auxiliary::directory_exists("../samples/subdir") ) - auxiliary::remove_directory("../samples/subdir"); + REQUIRE_THROWS_WITH(Series("../samples/git-sample/data%07T.h5", AccessType::READ_ONLY), + Catch::Equals("No matching iterations found: data%07T")); + try { - Series o = Series("../samples/subdir/serial_fileBased_write%08T.h5", AccessType::CREATE); - - ParticleSpecies& e_1 = o.iterations[1].particles["e"]; + std::vector< std::string > newFiles{"../samples/git-sample/data00000001.h5", + "../samples/git-sample/data00000010.h5", + "../samples/git-sample/data00001000.h5", + "../samples/git-sample/data00010000.h5", + "../samples/git-sample/data00100000.h5"}; - std::vector< double > position_global(4); - double pos{0.}; - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::shared_ptr< double > position_local_1(new double); - e_1["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_1), {4})); - std::vector< uint64_t > positionOffset_global(4); - uint64_t posOff{0}; - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); - e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); + for( auto const& file : newFiles ) + if( auxiliary::file_exists(file) ) + auxiliary::remove_file(file); - for( uint64_t i = 0; i < 4; ++i ) { - *position_local_1 = position_global[i]; - e_1["position"]["x"].storeChunk(position_local_1, {i}, {1}); - *positionOffset_local_1 = positionOffset_global[i]; - e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); - o.flush(); - } - - o.iterations[1].setTime(static_cast< double >(1)); - - ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + Series o = Series("../samples/git-sample/data%T.h5", AccessType::READ_WRITE); - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - e_2["position"]["x"].resetDataset(Dataset(determineDatatype(), {4})); - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_2(new uint64_t); - e_2["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_2), {4})); +#if openPMD_USE_INVASIVE_TESTS + REQUIRE(*o.m_filenamePadding == 8); +#endif - for( uint64_t i = 0; i < 4; ++i ) - { - double const position_local_2 = position_global.at(i); - e_2["position"]["x"].storeChunk(shareRaw(&position_local_2), {i}, {1}); - *positionOffset_local_2 = positionOffset_global[i]; - e_2["positionOffset"]["x"].storeChunk(positionOffset_local_2, {i}, {1}); + o.iterations[1]; + o.iterations[10]; + o.iterations[1000]; + o.iterations[10000]; + o.iterations[100000]; o.flush(); } - o.iterations[2].setTime(static_cast< double >(2)); - - ParticleSpecies& e_3 = o.iterations[3].particles["e"]; - - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::shared_ptr< double > position_local_3(new double); - e_3["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_3), {4})); - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_3(new uint64_t); - e_3["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_3), {4})); - - for( uint64_t i = 0; i < 4; ++i ) + for( auto const& file : newFiles ) { - *position_local_3 = position_global[i]; - e_3["position"]["x"].storeChunk(position_local_3, {i}, {1}); - *positionOffset_local_3 = positionOffset_global[i]; - e_3["positionOffset"]["x"].storeChunk(positionOffset_local_3, {i}, {1}); - o.flush(); + REQUIRE(auxiliary::file_exists(file)); + auxiliary::remove_file(file); } - - o.setOpenPMDextension(1); - o.iterations[3].setTime(static_cast< double >(3)); + } catch (no_such_file_error& e) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; } - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000001.h5")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000002.h5")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000003.h5")); +} +TEST_CASE( "hzdr_hdf5_sample_content_test", "[serial][hdf5]" ) +{ + // since this file might not be publicly available, gracefully handle errors + try { - Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_ONLY); - - REQUIRE(o.iterations.size() == 3); - REQUIRE(o.iterations.count(1) == 1); - REQUIRE(o.iterations.count(2) == 1); - REQUIRE(o.iterations.count(3) == 1); - -#if openPMD_USE_INVASIVE_TESTS - REQUIRE(*o.m_filenamePadding == 8); -#endif + /* HZDR: /bigdata/hplsim/development/huebl/lwfa-openPMD-062-smallLWFA-h5 + * DOI:10.14278/rodare.57 */ + Series o = Series("../samples/hzdr-sample/h5/simData_%T.h5", AccessType::READ_ONLY); + REQUIRE(o.openPMD() == "1.0.0"); + REQUIRE(o.openPMDextension() == 1); REQUIRE(o.basePath() == "/data/%T/"); - REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); - REQUIRE(o.iterationFormat() == "serial_fileBased_write%08T"); - REQUIRE(o.openPMD() == "1.1.0"); - REQUIRE(o.openPMDextension() == 1u); + REQUIRE(o.meshesPath() == "fields/"); REQUIRE(o.particlesPath() == "particles/"); - REQUIRE_FALSE(o.containsAttribute("meshesPath")); - REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error); - std::array< double, 7 > udim{{1, 0, 0, 0, 0, 0, 0}}; - Extent ext{4}; - for( auto& entry : o.iterations ) - { - auto& it = entry.second; - REQUIRE(it.dt< double >() == 1.); - REQUIRE(it.time< double >() == static_cast< double >(entry.first)); - REQUIRE(it.timeUnitSI() == 1.); - auto& pos = it.particles.at("e").at("position"); - REQUIRE(pos.timeOffset< float >() == 0.f); - REQUIRE(pos.unitDimension() == udim); - auto& pos_x = pos.at("x"); - REQUIRE(pos_x.unitSI() == 1.); - REQUIRE(pos_x.getExtent() == ext); - REQUIRE(pos_x.getDatatype() == Datatype::DOUBLE); - auto& posOff = it.particles.at("e").at("positionOffset"); - REQUIRE(posOff.timeOffset< float >() == 0.f); - REQUIRE(posOff.unitDimension() == udim); - auto& posOff_x = posOff.at("x"); - REQUIRE(posOff_x.unitSI() == 1.); - REQUIRE(posOff_x.getExtent() == ext); -#if !defined(_MSC_VER) - REQUIRE(posOff_x.getDatatype() == determineDatatype< uint64_t >()); -#endif - REQUIRE(isSame(posOff_x.getDatatype(), determineDatatype< uint64_t >())); + REQUIRE(o.author() == "Axel Huebl "); + REQUIRE(o.software() == "PIConGPU"); + REQUIRE(o.softwareVersion() == "0.2.0"); + REQUIRE(o.date() == "2016-11-04 00:59:14 +0100"); + REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); + REQUIRE(o.iterationFormat() == "h5/simData_%T.h5"); + REQUIRE(o.name() == "simData_%T"); - auto position = pos_x.loadChunk< double >({0}, {4}); - auto position_raw = position.get(); - auto positionOffset = posOff_x.loadChunk< uint64_t >({0}, {4}); - auto positionOffset_raw = positionOffset.get(); - o.flush(); - for( uint64_t j = 0; j < 4; ++j ) - { - REQUIRE(position_raw[j] == static_cast< double >(j + (entry.first-1)*4)); - REQUIRE(positionOffset_raw[j] == j + (entry.first-1)*4); - } - } - } + REQUIRE(o.iterations.size() >= 1); + REQUIRE(o.iterations.count(0) == 1); - // extend existing series with new step and auto-detection of iteration padding - { - Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_WRITE); + Iteration& i = o.iterations[0]; + REQUIRE(i.time< float >() == static_cast< float >(0.0f)); + REQUIRE(i.dt< float >() == static_cast< float >(1.0f)); + REQUIRE(i.timeUnitSI() == 1.3899999999999999e-16); - REQUIRE(o.iterations.size() == 3); - o.iterations[4]; - REQUIRE(o.iterations.size() == 4); - } - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000004.h5")); + REQUIRE(i.meshes.size() == 4); + REQUIRE(i.meshes.count("B") == 1); + REQUIRE(i.meshes.count("E") == 1); + REQUIRE(i.meshes.count("e_chargeDensity") == 1); + REQUIRE(i.meshes.count("e_energyDensity") == 1); - // additional iteration with different iteration padding but similar content - { - Series o = Series("../samples/subdir/serial_fileBased_write%01T.h5", AccessType::READ_WRITE); + std::vector< std::string > al{"z", "y", "x"}; + std::vector< float > gs{static_cast< float >(6.2393283843994141f), + static_cast< float >(1.0630855560302734f), + static_cast< float >(6.2393283843994141f)}; + std::vector< double > ggo{0., 0., 0.}; + std::array< double, 7 > ud{{0., 1., -2., -1., 0., 0., 0.}}; + Mesh& B = i.meshes["B"]; + REQUIRE(B.geometry() == Mesh::Geometry::cartesian); + REQUIRE(B.dataOrder() == Mesh::DataOrder::C); + REQUIRE(B.axisLabels() == al); + REQUIRE(B.gridSpacing< float >() == gs); + REQUIRE(B.gridGlobalOffset() == ggo); + REQUIRE(B.gridUnitSI() == 4.1671151661999998e-08); + REQUIRE(B.unitDimension() == ud); + REQUIRE(B.timeOffset< float >() == static_cast< float >(0.0f)); - REQUIRE(o.iterations.empty()); + REQUIRE(B.size() == 3); + REQUIRE(B.count("x") == 1); + REQUIRE(B.count("y") == 1); + REQUIRE(B.count("z") == 1); - auto& it = o.iterations[10]; - ParticleSpecies& e = it.particles["e"]; - e["position"]["x"].resetDataset(Dataset(Datatype::DOUBLE, {42})); - e["positionOffset"]["x"].resetDataset(Dataset(Datatype::DOUBLE, {42})); - e["position"]["x"].makeConstant(1.23); - e["positionOffset"]["x"].makeConstant(1.23); + std::vector< float > p{static_cast< float >(0.0f), + static_cast< float >(0.5f), + static_cast< float >(0.5f)}; + Extent e{80, 384, 80}; + MeshRecordComponent& B_x = B["x"]; + REQUIRE(B_x.unitSI() == 40903.822240601701); + REQUIRE(B_x.position< float >() == p); + REQUIRE(B_x.getDatatype() == Datatype::FLOAT); + REQUIRE(B_x.getExtent() == e); + REQUIRE(B_x.getDimensionality() == 3); - REQUIRE(o.iterations.size() == 1); - } - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write10.h5")); + p = {static_cast< float >(0.5f), + static_cast< float >(0.0f), + static_cast< float >(0.5f)}; + MeshRecordComponent& B_y = B["y"]; + REQUIRE(B_y.unitSI() == 40903.822240601701); + REQUIRE(B_y.position< float >() == p); + REQUIRE(B_y.getDatatype() == Datatype::FLOAT); + REQUIRE(B_y.getExtent() == e); + REQUIRE(B_y.getDimensionality() == 3); - // read back with auto-detection and non-fixed padding - { - Series s = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_ONLY); - REQUIRE(s.iterations.size() == 5); - } - - // write with auto-detection and in-consistent padding - { - REQUIRE_THROWS_WITH(Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_WRITE), - Catch::Equals("Cannot write to a series with inconsistent iteration padding. Please specify '%0T' or open as read-only.")); - } - - // read back with auto-detection and fixed padding - { - Series s = Series("../samples/subdir/serial_fileBased_write%08T.h5", AccessType::READ_ONLY); - REQUIRE(s.iterations.size() == 4); - } -} - -TEST_CASE( "hdf5_bool_test", "[serial][hdf5]" ) -{ - { - Series o = Series("../samples/serial_bool.h5", AccessType::CREATE); - - o.setAttribute("Bool attribute (true)", true); - o.setAttribute("Bool attribute (false)", false); - } - { - Series o = Series("../samples/serial_bool.h5", AccessType::READ_ONLY); - - auto attrs = o.attributes(); - REQUIRE(std::count(attrs.begin(), attrs.end(), "Bool attribute (true)") == 1); - REQUIRE(std::count(attrs.begin(), attrs.end(), "Bool attribute (false)") == 1); - REQUIRE(o.getAttribute("Bool attribute (true)").get< bool >() == true); - REQUIRE(o.getAttribute("Bool attribute (false)").get< bool >() == false); - } -} - -TEST_CASE( "hdf5_patch_test", "[serial][hdf5]" ) -{ - Series o = Series("../samples/serial_patch.h5", AccessType::CREATE); - - auto dset = Dataset(Datatype::DOUBLE, {1}); - o.iterations[1].particles["e"].particlePatches["offset"]["x"].resetDataset(dset); - o.iterations[1].particles["e"].particlePatches["offset"]["x"].setUnitSI(42); - o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[1].particles["e"]["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); -} - -TEST_CASE( "hdf5_deletion_test", "[serial][hdf5]" ) -{ - Series o = Series("../samples/serial_deletion.h5", AccessType::CREATE); - - - o.setAttribute("removed", - "this attribute will be removed after being written to disk"); - o.flush(); - - o.deleteAttribute("removed"); - o.flush(); - - ParticleSpecies& e = o.iterations[1].particles["e"]; - auto dset = Dataset(Datatype::DOUBLE, {1}); - e["position"][RecordComponent::SCALAR].resetDataset(dset); - e["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - e.erase("deletion"); - o.flush(); - - e["deletion_scalar"][RecordComponent::SCALAR].resetDataset(dset); - o.flush(); - - e["deletion_scalar"].erase(RecordComponent::SCALAR); - e.erase("deletion_scalar"); - o.flush(); - - double value = 0.; - e["deletion_scalar_constant"][RecordComponent::SCALAR].resetDataset(dset); - e["deletion_scalar_constant"][RecordComponent::SCALAR].makeConstant(value); - o.flush(); - - e["deletion_scalar_constant"].erase(RecordComponent::SCALAR); - e.erase("deletion_scalar_constant"); - o.flush(); -} - -TEST_CASE( "hdf5_110_optional_paths", "[serial][hdf5]" ) -{ - try - { - { - Series s = Series("../samples/issue-sample/no_fields/data%T.h5", AccessType::READ_ONLY); - auto attrs = s.attributes(); - REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 1); - REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 1); - REQUIRE(s.iterations[400].meshes.empty()); - REQUIRE(s.iterations[400].particles.size() == 1); - } - - { - Series s = Series("../samples/issue-sample/no_particles/data%T.h5", AccessType::READ_ONLY); - auto attrs = s.attributes(); - REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 1); - REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 1); - REQUIRE(s.iterations[400].meshes.size() == 2); - REQUIRE(s.iterations[400].particles.empty()); - } - } catch (no_such_file_error& e) - { - std::cerr << "issue sample not accessible. (" << e.what() << ")\n"; - } - - { - Series s = Series("../samples/no_meshes_1.1.0_compliant.h5", AccessType::CREATE); - auto foo = s.iterations[1].particles["foo"]; - Dataset dset = Dataset(Datatype::DOUBLE, {1}); - foo["position"][RecordComponent::SCALAR].resetDataset(dset); - foo["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - } - - { - Series s = Series("../samples/no_particles_1.1.0_compliant.h5", AccessType::CREATE); - auto foo = s.iterations[1].meshes["foo"]; - Dataset dset = Dataset(Datatype::DOUBLE, {1}); - foo[RecordComponent::SCALAR].resetDataset(dset); - } - - { - Series s = Series("../samples/no_meshes_1.1.0_compliant.h5", AccessType::READ_ONLY); - auto attrs = s.attributes(); - REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 0); - REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 1); - REQUIRE(s.iterations[1].meshes.empty()); - REQUIRE(s.iterations[1].particles.size() == 1); - } - - { - Series s = Series("../samples/no_particles_1.1.0_compliant.h5", AccessType::READ_ONLY); - auto attrs = s.attributes(); - REQUIRE(std::count(attrs.begin(), attrs.end(), "meshesPath") == 1); - REQUIRE(std::count(attrs.begin(), attrs.end(), "particlesPath") == 0); - REQUIRE(s.iterations[1].meshes.size() == 1); - REQUIRE(s.iterations[1].particles.empty()); - } -} + p = {static_cast< float >(0.5f), + static_cast< float >(0.5f), + static_cast< float >(0.0f)}; + MeshRecordComponent& B_z = B["z"]; + REQUIRE(B_z.unitSI() == 40903.822240601701); + REQUIRE(B_z.position< float >() == p); + REQUIRE(B_z.getDatatype() == Datatype::FLOAT); + REQUIRE(B_z.getExtent() == e); + REQUIRE(B_z.getDimensionality() == 3); -TEST_CASE( "hdf5_constant_scalar", "[serial][hdf5]" ) -{ - constant_scalar("h5"); -} + ud = {{1., 1., -3., -1., 0., 0., 0.}}; + Mesh& E = i.meshes["E"]; + REQUIRE(E.geometry() == Mesh::Geometry::cartesian); + REQUIRE(E.dataOrder() == Mesh::DataOrder::C); + REQUIRE(E.axisLabels() == al); + REQUIRE(E.gridSpacing< float >() == gs); + REQUIRE(E.gridGlobalOffset() == ggo); + REQUIRE(E.gridUnitSI() == 4.1671151661999998e-08); + REQUIRE(E.unitDimension() == ud); + REQUIRE(E.timeOffset< float >() == static_cast< float >(0.0f)); -TEST_CASE( "hdf5_particle_patches", "[serial][hdf5]" ) -{ - particle_patches("h5"); -} -#else -TEST_CASE( "no_serial_hdf5", "[serial][hdf5]" ) -{ - REQUIRE(true); -} -#endif -#if openPMD_HAVE_ADIOS1 -TEST_CASE( "adios1_dtype_test", "[serial][adios1]" ) -{ - { - Series s = Series("../samples/dtype_test.bp", AccessType::CREATE); + REQUIRE(E.size() == 3); + REQUIRE(E.count("x") == 1); + REQUIRE(E.count("y") == 1); + REQUIRE(E.count("z") == 1); - char c = 'c'; - s.setAttribute("char", c); - unsigned char uc = 'u'; - s.setAttribute("uchar", uc); - int16_t i16 = 16; - s.setAttribute("int16", i16); - int32_t i32 = 32; - s.setAttribute("int32", i32); - int64_t i64 = 64; - s.setAttribute("int64", i64); - uint16_t u16 = 16u; - s.setAttribute("uint16", u16); - uint32_t u32 = 32u; - s.setAttribute("uint32", u32); - uint64_t u64 = 64u; - s.setAttribute("uint64", u64); - float f = 16.e10f; - s.setAttribute("float", f); - double d = 1.e64; - s.setAttribute("double", d); - long double ld = 1.e80L; - s.setAttribute("longdouble", ld); - std::string str = "string"; - s.setAttribute("string", str); - s.setAttribute("vecChar", std::vector< char >({'c', 'h', 'a', 'r'})); - s.setAttribute("vecInt16", std::vector< int16_t >({32766, 32767})); - s.setAttribute("vecInt32", std::vector< int32_t >({2147483646, 2147483647})); - s.setAttribute("vecInt64", std::vector< int64_t >({9223372036854775806, 9223372036854775807})); - s.setAttribute("vecUchar", std::vector< char >({'u', 'c', 'h', 'a', 'r'})); - s.setAttribute("vecUint16", std::vector< uint16_t >({65534u, 65535u})); - s.setAttribute("vecUint32", std::vector< uint32_t >({4294967294u, 4294967295u})); - s.setAttribute("vecUint64", std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); - s.setAttribute("vecFloat", std::vector< float >({0.f, 3.40282e+38f})); - s.setAttribute("vecDouble", std::vector< double >({0., 1.79769e+308})); - s.setAttribute("vecLongdouble", std::vector< long double >({0.L, std::numeric_limits::max()})); - s.setAttribute("vecString", std::vector< std::string >({"vector", "of", "strings"})); - s.setAttribute("bool", true); - s.setAttribute("boolF", false); + p = {static_cast< float >(0.5f), + static_cast< float >(0.0f), + static_cast< float >(0.0f)}; + e = {80, 384, 80}; + MeshRecordComponent& E_x = E["x"]; + REQUIRE(E_x.unitSI() == 12262657411105.049); + REQUIRE(E_x.position< float >() == p); + REQUIRE(E_x.getDatatype() == Datatype::FLOAT); + REQUIRE(E_x.getExtent() == e); + REQUIRE(E_x.getDimensionality() == 3); - // translation from non-fixed size integer types - short rs = 8; - s.setAttribute("short", rs); - int i = 16; - s.setAttribute("int", i); - long l = 32; - s.setAttribute("long", l); - long long ll = 64; - s.setAttribute("longlong", ll); - unsigned short us = 8u; - s.setAttribute("ushort", us); - unsigned int ui = 16u; - s.setAttribute("uint", ui); - unsigned long ul = 32u; - s.setAttribute("ulong", ul); - unsigned long long ull = 64u; - s.setAttribute("ulonglong", ull); - s.setAttribute("vecShort", std::vector< short >({32766, 32767})); - s.setAttribute("vecInt", std::vector< int >({32766, 32767})); - s.setAttribute("vecLong", std::vector< long >({2147483646, 2147483647})); - s.setAttribute("vecLongLong", std::vector< long long >({2147483644, 2147483643})); - s.setAttribute("vecUShort", std::vector< unsigned short >({65534u, 65535u})); - s.setAttribute("vecUInt", std::vector< unsigned int >({65533u, 65531u})); - s.setAttribute("vecULong", std::vector< unsigned long >({65532u, 65530u})); - s.setAttribute("vecULongLong", std::vector< unsigned long long >({65529u, 65528u})); - } + p = {static_cast< float >(0.0f), + static_cast< float >(0.5f), + static_cast< float >(0.0f)}; + MeshRecordComponent& E_y = E["y"]; + REQUIRE(E_y.unitSI() == 12262657411105.049); + REQUIRE(E_y.position< float >() == p); + REQUIRE(E_y.getDatatype() == Datatype::FLOAT); + REQUIRE(E_y.getExtent() == e); + REQUIRE(E_y.getDimensionality() == 3); - Series s = Series("../samples/dtype_test.bp", AccessType::READ_ONLY); + p = {static_cast< float >(0.0f), + static_cast< float >(0.0f), + static_cast< float >(0.5f)}; + MeshRecordComponent& E_z = E["z"]; + REQUIRE(E_z.unitSI() == 12262657411105.049); + REQUIRE(E_z.position< float >() == p); + REQUIRE(E_z.getDatatype() == Datatype::FLOAT); + REQUIRE(E_z.getExtent() == e); + REQUIRE(E_z.getDimensionality() == 3); - REQUIRE(s.getAttribute("char").get< char >() == 'c'); - REQUIRE(s.getAttribute("uchar").get< unsigned char >() == 'u'); - REQUIRE(s.getAttribute("int16").get< int16_t >() == 16); - REQUIRE(s.getAttribute("int32").get< int32_t >() == 32); - REQUIRE(s.getAttribute("int64").get() == 64); - REQUIRE(s.getAttribute("uint16").get< uint16_t >() == 16u); - REQUIRE(s.getAttribute("uint32").get< uint32_t >() == 32u); - REQUIRE(s.getAttribute("uint64").get< uint64_t >() == 64u); - REQUIRE(s.getAttribute("float").get< float >() == 16.e10f); - REQUIRE(s.getAttribute("double").get< double >() == 1.e64); - REQUIRE(s.getAttribute("longdouble").get< long double >() == 1.e80L); - REQUIRE(s.getAttribute("string").get< std::string >() == "string"); - REQUIRE(s.getAttribute("vecChar").get< std::vector< char > >() == std::vector< char >({'c', 'h', 'a', 'r'})); - REQUIRE(s.getAttribute("vecInt16").get< std::vector< int16_t > >() == std::vector< int16_t >({32766, 32767})); - REQUIRE(s.getAttribute("vecInt32").get< std::vector< int32_t > >() == std::vector< int32_t >({2147483646, 2147483647})); - REQUIRE(s.getAttribute("vecInt64").get< std::vector< int64_t > >() == std::vector< int64_t >({9223372036854775806, 9223372036854775807})); - REQUIRE(s.getAttribute("vecUchar").get< std::vector< char > >() == std::vector< char >({'u', 'c', 'h', 'a', 'r'})); - REQUIRE(s.getAttribute("vecUint16").get< std::vector< uint16_t > >() == std::vector< uint16_t >({65534u, 65535u})); - REQUIRE(s.getAttribute("vecUint32").get< std::vector< uint32_t > >() == std::vector< uint32_t >({4294967294u, 4294967295u})); - REQUIRE(s.getAttribute("vecUint64").get< std::vector< uint64_t > >() == std::vector< uint64_t >({18446744073709551614u, 18446744073709551615u})); - REQUIRE(s.getAttribute("vecFloat").get< std::vector< float > >() == std::vector< float >({0.f, 3.40282e+38f})); - REQUIRE(s.getAttribute("vecDouble").get< std::vector< double > >() == std::vector< double >({0., 1.79769e+308})); - REQUIRE(s.getAttribute("vecLongdouble").get< std::vector< long double > >() == std::vector< long double >({0.L, std::numeric_limits::max()})); - REQUIRE(s.getAttribute("vecString").get< std::vector< std::string > >() == std::vector< std::string >({"vector", "of", "strings"})); - REQUIRE(s.getAttribute("bool").get< bool >() == true); - REQUIRE(s.getAttribute("boolF").get< bool >() == false); + ud = {{-3., 0., 1., 1., 0., 0., 0.}}; + Mesh& e_chargeDensity = i.meshes["e_chargeDensity"]; + REQUIRE(e_chargeDensity.geometry() == Mesh::Geometry::cartesian); + REQUIRE(e_chargeDensity.dataOrder() == Mesh::DataOrder::C); + REQUIRE(e_chargeDensity.axisLabels() == al); + REQUIRE(e_chargeDensity.gridSpacing< float >() == gs); + REQUIRE(e_chargeDensity.gridGlobalOffset() == ggo); + REQUIRE(e_chargeDensity.gridUnitSI() == 4.1671151661999998e-08); + REQUIRE(e_chargeDensity.unitDimension() == ud); + REQUIRE(e_chargeDensity.timeOffset< float >() == static_cast< float >(0.0f)); - // translation from non-fixed size integer types - REQUIRE(s.getAttribute("short").get< short >() == 8); - REQUIRE(s.getAttribute("int").get< int >() == 16); - REQUIRE(s.getAttribute("long").get< long >() == 32); - REQUIRE(s.getAttribute("longlong").get< long long >() == 64); - REQUIRE(s.getAttribute("ushort").get< unsigned short >() == 8u); - REQUIRE(s.getAttribute("uint").get< unsigned int >() == 16u); - REQUIRE(s.getAttribute("ulong").get< unsigned long >() == 32u); - REQUIRE(s.getAttribute("ulonglong").get< unsigned long long >() == 64u); - REQUIRE(s.getAttribute("vecShort").get< std::vector< short > >() == std::vector< short >({32766, 32767})); - REQUIRE(s.getAttribute("vecInt").get< std::vector< int > >() == std::vector< int >({32766, 32767})); - REQUIRE(s.getAttribute("vecLong").get< std::vector< long > >() == std::vector< long >({2147483646, 2147483647})); - REQUIRE(s.getAttribute("vecLongLong").get< std::vector< long long > >() == std::vector< long long >({2147483644, 2147483643})); - REQUIRE(s.getAttribute("vecUShort").get< std::vector< unsigned short > >() == std::vector< unsigned short >({65534u, 65535u})); - REQUIRE(s.getAttribute("vecUInt").get< std::vector< unsigned int > >() == std::vector< unsigned int >({65533u, 65531u})); - REQUIRE(s.getAttribute("vecULong").get< std::vector< unsigned long > >() == std::vector< unsigned long >({65532u, 65530u})); - REQUIRE(s.getAttribute("vecULongLong").get< std::vector< unsigned long long > >() == std::vector< unsigned long long >({65529u, 65528u})); -} + REQUIRE(e_chargeDensity.size() == 1); + REQUIRE(e_chargeDensity.count(MeshRecordComponent::SCALAR) == 1); -TEST_CASE( "adios1_write_test", "[serial][adios1]") -{ - Series o = Series("../samples/serial_write.bp", AccessType::CREATE); + p = {static_cast< float >(0.f), + static_cast< float >(0.f), + static_cast< float >(0.f)}; + MeshRecordComponent& e_chargeDensity_scalar = e_chargeDensity[MeshRecordComponent::SCALAR]; + REQUIRE(e_chargeDensity_scalar.unitSI() == 66306201.002331272); + REQUIRE(e_chargeDensity_scalar.position< float >() == p); + REQUIRE(e_chargeDensity_scalar.getDatatype() == Datatype::FLOAT); + REQUIRE(e_chargeDensity_scalar.getExtent() == e); + REQUIRE(e_chargeDensity_scalar.getDimensionality() == 3); - ParticleSpecies& e_1 = o.iterations[1].particles["e"]; + ud = {{-1., 1., -2., 0., 0., 0., 0.}}; + Mesh& e_energyDensity = i.meshes["e_energyDensity"]; + REQUIRE(e_energyDensity.geometry() == Mesh::Geometry::cartesian); + REQUIRE(e_energyDensity.dataOrder() == Mesh::DataOrder::C); + REQUIRE(e_energyDensity.axisLabels() == al); + REQUIRE(e_energyDensity.gridSpacing< float >() == gs); + REQUIRE(e_energyDensity.gridGlobalOffset() == ggo); + REQUIRE(e_energyDensity.gridUnitSI() == 4.1671151661999998e-08); + REQUIRE(e_energyDensity.unitDimension() == ud); + REQUIRE(e_energyDensity.timeOffset< float >() == static_cast< float >(0.0f)); - std::vector< double > position_global(4); - double pos{0.}; - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::shared_ptr< double > position_local_1(new double); - e_1["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_1), {4})); + REQUIRE(e_energyDensity.size() == 1); + REQUIRE(e_energyDensity.count(MeshRecordComponent::SCALAR) == 1); - for( uint64_t i = 0; i < 4; ++i ) - { - *position_local_1 = position_global[i]; - e_1["position"]["x"].storeChunk(position_local_1, {i}, {1}); - } + MeshRecordComponent& e_energyDensity_scalar = e_energyDensity[MeshRecordComponent::SCALAR]; + REQUIRE(e_energyDensity_scalar.unitSI() == 1.0146696675429705e+18); + REQUIRE(e_energyDensity_scalar.position< float >() == p); + REQUIRE(e_energyDensity_scalar.getDatatype() == Datatype::FLOAT); + REQUIRE(e_energyDensity_scalar.getExtent() == e); + REQUIRE(e_energyDensity_scalar.getDimensionality() == 3); - std::vector< uint64_t > positionOffset_global(4); - uint64_t posOff{0}; - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); - e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); + REQUIRE(i.particles.size() == 1); + REQUIRE(i.particles.count("e") == 1); - for( uint64_t i = 0; i < 4; ++i ) - { - *positionOffset_local_1 = positionOffset_global[i]; - e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); - } + ParticleSpecies& species_e = i.particles["e"]; - ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + REQUIRE(species_e.size() == 6); + REQUIRE(species_e.count("charge") == 1); + REQUIRE(species_e.count("mass") == 1); + REQUIRE(species_e.count("momentum") == 1); + REQUIRE(species_e.count("particlePatches") == 0); + REQUIRE(species_e.count("position") == 1); + REQUIRE(species_e.count("positionOffset") == 1); + REQUIRE(species_e.count("weighting") == 1); - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::shared_ptr< double > position_local_2(new double); - e_2["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_2), {4})); + ud = {{0., 0., 1., 1., 0., 0., 0.}}; + Record& e_charge = species_e["charge"]; + REQUIRE(e_charge.unitDimension() == ud); + REQUIRE(e_charge.timeOffset< float >() == static_cast< float >(0.0f)); - for( uint64_t i = 0; i < 4; ++i ) - { - *position_local_2 = position_global[i]; - e_2["position"]["x"].storeChunk(position_local_2, {i}, {1}); - } + REQUIRE(e_charge.size() == 1); + REQUIRE(e_charge.count(RecordComponent::SCALAR) == 1); - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_2(new uint64_t); - e_2["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_2), {4})); + e = {2150400}; + RecordComponent& e_charge_scalar = e_charge[RecordComponent::SCALAR]; + REQUIRE(e_charge_scalar.unitSI() == 4.7980045488500004e-15); + REQUIRE(e_charge_scalar.getDatatype() == Datatype::DOUBLE); + REQUIRE(e_charge_scalar.getExtent() == e); + REQUIRE(e_charge_scalar.getDimensionality() == 1); - for( uint64_t i = 0; i < 4; ++i ) - { - *positionOffset_local_2 = positionOffset_global[i]; - e_2["positionOffset"]["x"].storeChunk(positionOffset_local_2, {i}, {1}); - } + ud = {{0., 1., 0., 0., 0., 0., 0.}}; + Record& e_mass = species_e["mass"]; + REQUIRE(e_mass.unitDimension() == ud); + REQUIRE(e_mass.timeOffset< float >() == static_cast< float >(0.0f)); - o.flush(); + REQUIRE(e_mass.size() == 1); + REQUIRE(e_mass.count(RecordComponent::SCALAR) == 1); - ParticleSpecies& e_3 = o.iterations[3].particles["e"]; + RecordComponent& e_mass_scalar = e_mass[RecordComponent::SCALAR]; + REQUIRE(e_mass_scalar.unitSI() == 2.7279684799430467e-26); + REQUIRE(e_mass_scalar.getDatatype() == Datatype::DOUBLE); + REQUIRE(e_mass_scalar.getExtent() == e); + REQUIRE(e_mass_scalar.getDimensionality() == 1); - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::shared_ptr< double > position_local_3(new double); - e_3["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_3), {4})); + ud = {{1., 1., -1., 0., 0., 0., 0.}}; + Record& e_momentum = species_e["momentum"]; + REQUIRE(e_momentum.unitDimension() == ud); + REQUIRE(e_momentum.timeOffset< float >() == static_cast< float >(0.0f)); - for( uint64_t i = 0; i < 4; ++i ) - { - *position_local_3 = position_global[i]; - e_3["position"]["x"].storeChunk(position_local_3, {i}, {1}); - } + REQUIRE(e_momentum.size() == 3); + REQUIRE(e_momentum.count("x") == 1); + REQUIRE(e_momentum.count("y") == 1); + REQUIRE(e_momentum.count("z") == 1); - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_3(new uint64_t); - e_3["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_3), {4})); + RecordComponent& e_momentum_x = e_momentum["x"]; + REQUIRE(e_momentum_x.unitSI() == 8.1782437594864961e-18); + REQUIRE(e_momentum_x.getDatatype() == Datatype::FLOAT); + REQUIRE(e_momentum_x.getExtent() == e); + REQUIRE(e_momentum_x.getDimensionality() == 1); - for( uint64_t i = 0; i < 4; ++i ) - { - *positionOffset_local_3 = positionOffset_global[i]; - e_3["positionOffset"]["x"].storeChunk(positionOffset_local_3, {i}, {1}); - } + RecordComponent& e_momentum_y = e_momentum["y"]; + REQUIRE(e_momentum_y.unitSI() == 8.1782437594864961e-18); + REQUIRE(e_momentum_y.getDatatype() == Datatype::FLOAT); + REQUIRE(e_momentum_y.getExtent() == e); + REQUIRE(e_momentum_y.getDimensionality() == 1); - o.flush(); -} + RecordComponent& e_momentum_z = e_momentum["z"]; + REQUIRE(e_momentum_z.unitSI() == 8.1782437594864961e-18); + REQUIRE(e_momentum_z.getDatatype() == Datatype::FLOAT); + REQUIRE(e_momentum_z.getExtent() == e); + REQUIRE(e_momentum_z.getDimensionality() == 1); -TEST_CASE( "adios1_fileBased_write_empty_test", "[serial][adios1]" ) -{ - if( auxiliary::directory_exists("../samples/subdir") ) - auxiliary::remove_directory("../samples/subdir"); + ud = {{1., 0., 0., 0., 0., 0., 0.}}; + Record& e_position = species_e["position"]; + REQUIRE(e_position.unitDimension() == ud); + REQUIRE(e_position.timeOffset< float >() == static_cast< float >(0.0f)); - Dataset dset = Dataset(Datatype::DOUBLE, {2}); - { - Series o = Series("../samples/subdir/serial_fileBased_write%T.bp", AccessType::CREATE); + REQUIRE(e_position.size() == 3); + REQUIRE(e_position.count("x") == 1); + REQUIRE(e_position.count("y") == 1); + REQUIRE(e_position.count("z") == 1); - ParticleSpecies& e_1 = o.iterations[1].particles["e"]; - e_1["position"][RecordComponent::SCALAR].resetDataset(dset); - e_1["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[1].setTime(1.f); - ParticleSpecies& e_2 = o.iterations[2].particles["e"]; - e_2["position"][RecordComponent::SCALAR].resetDataset(dset); - e_2["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[2].setTime(2.f); - ParticleSpecies& e_3 = o.iterations[3].particles["e"]; - e_3["position"][RecordComponent::SCALAR].resetDataset(dset); - e_3["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[3].setTime(3.f); - } + RecordComponent& e_position_x = e_position["x"]; + REQUIRE(e_position_x.unitSI() == 2.599999993753294e-07); + REQUIRE(e_position_x.getDatatype() == Datatype::FLOAT); + REQUIRE(e_position_x.getExtent() == e); + REQUIRE(e_position_x.getDimensionality() == 1); - REQUIRE(auxiliary::directory_exists("../samples/subdir")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write1.bp")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write2.bp")); - REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write3.bp")); + RecordComponent& e_position_y = e_position["y"]; + REQUIRE(e_position_y.unitSI() == 4.4299999435019118e-08); + REQUIRE(e_position_y.getDatatype() == Datatype::FLOAT); + REQUIRE(e_position_y.getExtent() == e); + REQUIRE(e_position_y.getDimensionality() == 1); - { - Series o = Series("../samples/subdir/serial_fileBased_write%T.bp", AccessType::READ_ONLY); + RecordComponent& e_position_z = e_position["z"]; + REQUIRE(e_position_z.unitSI() == 2.599999993753294e-07); + REQUIRE(e_position_z.getDatatype() == Datatype::FLOAT); + REQUIRE(e_position_z.getExtent() == e); + REQUIRE(e_position_z.getDimensionality() == 1); - REQUIRE(o.iterations.size() == 3); - REQUIRE(o.iterations.count(1) == 1); - REQUIRE(o.iterations.count(2) == 1); - REQUIRE(o.iterations.count(3) == 1); + ud = {{1., 0., 0., 0., 0., 0., 0.}}; + Record& e_positionOffset = species_e["positionOffset"]; + REQUIRE(e_positionOffset.unitDimension() == ud); + REQUIRE(e_positionOffset.timeOffset< float >() == static_cast< float >(0.0f)); - REQUIRE(o.iterations[1].time< float >() == 1.f); - REQUIRE(o.iterations[2].time< float >() == 2.f); - REQUIRE(o.iterations[3].time< float >() == 3.f); + REQUIRE(e_positionOffset.size() == 3); + REQUIRE(e_positionOffset.count("x") == 1); + REQUIRE(e_positionOffset.count("y") == 1); + REQUIRE(e_positionOffset.count("z") == 1); - REQUIRE(o.iterations[1].particles.size() == 1); - REQUIRE(o.iterations[1].particles.count("e") == 1); - REQUIRE(o.iterations[2].particles.size() == 1); - REQUIRE(o.iterations[2].particles.count("e") == 1); - REQUIRE(o.iterations[3].particles.size() == 1); - REQUIRE(o.iterations[3].particles.count("e") == 1); + RecordComponent& e_positionOffset_x = e_positionOffset["x"]; + REQUIRE(e_positionOffset_x.unitSI() == 2.599999993753294e-07); + REQUIRE(e_positionOffset_x.getDatatype() == determineDatatype< int32_t >()); + REQUIRE(e_positionOffset_x.getExtent() == e); + REQUIRE(e_positionOffset_x.getDimensionality() == 1); - REQUIRE(o.iterations[1].particles["e"].size() == 2); - REQUIRE(o.iterations[1].particles["e"].count("position") == 1); - REQUIRE(o.iterations[1].particles["e"].count("positionOffset") == 1); - REQUIRE(o.iterations[2].particles["e"].size() == 2); - REQUIRE(o.iterations[2].particles["e"].count("position") == 1); - REQUIRE(o.iterations[2].particles["e"].count("positionOffset") == 1); - REQUIRE(o.iterations[3].particles["e"].size() == 2); - REQUIRE(o.iterations[3].particles["e"].count("position") == 1); - REQUIRE(o.iterations[3].particles["e"].count("positionOffset") == 1); + RecordComponent& e_positionOffset_y = e_positionOffset["y"]; + REQUIRE(e_positionOffset_y.unitSI() == 4.4299999435019118e-08); + REQUIRE(e_positionOffset_y.getDatatype() == determineDatatype< int32_t >()); + REQUIRE(e_positionOffset_y.getExtent() == e); + REQUIRE(e_positionOffset_y.getDimensionality() == 1); - REQUIRE(o.iterations[1].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[1].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[1].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[1].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[1].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); - REQUIRE(o.iterations[2].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[2].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[2].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[2].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[2].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); - REQUIRE(o.iterations[3].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[3].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[3].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[3].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[3].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); - } + RecordComponent& e_positionOffset_z = e_positionOffset["z"]; + REQUIRE(e_positionOffset_z.unitSI() == 2.599999993753294e-07); + REQUIRE(e_positionOffset_z.getDatatype() == determineDatatype< int32_t >()); + REQUIRE(e_positionOffset_z.getExtent() == e); + REQUIRE(e_positionOffset_z.getDimensionality() == 1); - { - Series o = Series("../samples/subdir/serial_fileBased_write%T.bp", AccessType::READ_WRITE); - ParticleSpecies& e_4 = o.iterations[4].particles["e"]; - e_4["position"][RecordComponent::SCALAR].resetDataset(dset); - e_4["positionOffset"][RecordComponent::SCALAR].resetDataset(dset); - o.iterations[4].setTime(4.f); - } + ud = {{0., 0., 0., 0., 0., 0., 0.}}; + Record& e_weighting = species_e["weighting"]; + REQUIRE(e_weighting.unitDimension() == ud); + REQUIRE(e_weighting.timeOffset< float >() == static_cast< float >(0.0f)); - { - Series o = Series("../samples/subdir/serial_fileBased_write%T.bp", AccessType::READ_ONLY); + REQUIRE(e_weighting.size() == 1); + REQUIRE(e_weighting.count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations.size() == 4); - REQUIRE(o.iterations.count(4) == 1); + RecordComponent& e_weighting_scalar = e_weighting[RecordComponent::SCALAR]; + REQUIRE(e_weighting_scalar.unitSI() == 1.0); + REQUIRE(e_weighting_scalar.getDatatype() == Datatype::FLOAT); + REQUIRE(e_weighting_scalar.getExtent() == e); + REQUIRE(e_weighting_scalar.getDimensionality() == 1); - REQUIRE(o.iterations[4].time< float >() == 4.f); + ParticlePatches& e_patches = species_e.particlePatches; + REQUIRE(e_patches.size() == 4); /* extent, numParticles, numParticlesOffset, offset */ + REQUIRE(e_patches.count("extent") == 1); + REQUIRE(e_patches.count("numParticles") == 1); + REQUIRE(e_patches.count("numParticlesOffset") == 1); + REQUIRE(e_patches.count("offset") == 1); + REQUIRE(e_patches.numPatches() == 4); - REQUIRE(o.iterations[4].particles.size() == 1); - REQUIRE(o.iterations[4].particles.count("e") == 1); + ud = {{1., 0., 0., 0., 0., 0., 0.}}; + PatchRecord& e_extent = e_patches["extent"]; + REQUIRE(e_extent.unitDimension() == ud); - REQUIRE(o.iterations[4].particles["e"].size() == 2); - REQUIRE(o.iterations[4].particles["e"].count("position") == 1); - REQUIRE(o.iterations[4].particles["e"].count("positionOffset") == 1); + REQUIRE(e_extent.size() == 3); + REQUIRE(e_extent.count("x") == 1); + REQUIRE(e_extent.count("y") == 1); + REQUIRE(e_extent.count("z") == 1); - REQUIRE(o.iterations[4].particles["e"]["position"].size() == 1); - REQUIRE(o.iterations[4].particles["e"]["position"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[4].particles["e"]["positionOffset"].size() == 1); - REQUIRE(o.iterations[4].particles["e"]["positionOffset"].count(RecordComponent::SCALAR) == 1); - REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDatatype() == Datatype::DOUBLE); - REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getDimensionality() == 1); - REQUIRE(o.iterations[4].particles["e"]["position"][RecordComponent::SCALAR].getExtent() == Extent{2}); - } -} + PatchRecordComponent& e_extent_x = e_extent["x"]; + REQUIRE(e_extent_x.unitSI() == 2.599999993753294e-07); +#if !defined(_MSC_VER) + REQUIRE(e_extent_x.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_extent_x.getDatatype(), determineDatatype< uint64_t >())); -TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" ) -{ - { - Series o = Series("../samples/serial_fileBased_write%T.bp", AccessType::CREATE); + PatchRecordComponent& e_extent_y = e_extent["y"]; + REQUIRE(e_extent_y.unitSI() == 4.429999943501912e-08); +#if !defined(_MSC_VER) + REQUIRE(e_extent_y.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_extent_y.getDatatype(), determineDatatype< uint64_t >())); - ParticleSpecies& e_1 = o.iterations[1].particles["e"]; + PatchRecordComponent& e_extent_z = e_extent["z"]; + REQUIRE(e_extent_z.unitSI() == 2.599999993753294e-07); +#if !defined(_MSC_VER) + REQUIRE(e_extent_z.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_extent_z.getDatatype(), determineDatatype< uint64_t >())); - std::vector< double > position_global(4); - double pos{0.}; - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::shared_ptr< double > position_local_1(new double); - e_1["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_1), {4})); - std::vector< uint64_t > positionOffset_global(4); - uint64_t posOff{0}; - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_1(new uint64_t); - e_1["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_1), {4})); + std::vector< uint64_t > data( e_patches.size() ); + e_extent_z.load(shareRaw(data.data())); + o.flush(); + REQUIRE(data.at(0) == static_cast< uint64_t >(80)); + REQUIRE(data.at(1) == static_cast< uint64_t >(80)); + REQUIRE(data.at(2) == static_cast< uint64_t >(80)); + REQUIRE(data.at(3) == static_cast< uint64_t >(80)); - for( uint64_t i = 0; i < 4; ++i ) - { - *position_local_1 = position_global[i]; - e_1["position"]["x"].storeChunk(position_local_1, {i}, {1}); - *positionOffset_local_1 = positionOffset_global[i]; - e_1["positionOffset"]["x"].storeChunk(positionOffset_local_1, {i}, {1}); - o.flush(); - } + PatchRecord& e_numParticles = e_patches["numParticles"]; + REQUIRE(e_numParticles.size() == 1); + REQUIRE(e_numParticles.count(RecordComponent::SCALAR) == 1); - o.iterations[1].setTime(1.f); + PatchRecordComponent& e_numParticles_scalar = e_numParticles[RecordComponent::SCALAR]; +#if !defined(_MSC_VER) + REQUIRE(e_numParticles_scalar.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_numParticles_scalar.getDatatype(), determineDatatype< uint64_t >())); - ParticleSpecies& e_2 = o.iterations[2].particles["e"]; + e_numParticles_scalar.load(shareRaw(data.data())); + o.flush(); + REQUIRE(data.at(0) == static_cast< uint64_t >(512000)); + REQUIRE(data.at(1) == static_cast< uint64_t >(819200)); + REQUIRE(data.at(2) == static_cast< uint64_t >(819200)); + REQUIRE(data.at(3) == static_cast< uint64_t >(0)); - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - e_2["position"]["x"].resetDataset(Dataset(determineDatatype(), {4})); - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_2(new uint64_t); - e_2["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_2), {4})); + PatchRecord& e_numParticlesOffset = e_patches["numParticlesOffset"]; + REQUIRE(e_numParticlesOffset.size() == 1); + REQUIRE(e_numParticlesOffset.count(RecordComponent::SCALAR) == 1); - for( uint64_t i = 0; i < 4; ++i ) - { - double const position_local_2 = position_global.at(i); - e_2["position"]["x"].storeChunk(shareRaw(&position_local_2), {i}, {1}); - *positionOffset_local_2 = positionOffset_global[i]; - e_2["positionOffset"]["x"].storeChunk(positionOffset_local_2, {i}, {1}); - o.flush(); - } + PatchRecordComponent& e_numParticlesOffset_scalar = e_numParticlesOffset[RecordComponent::SCALAR]; +#if !defined(_MSC_VER) + REQUIRE(e_numParticlesOffset_scalar.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_numParticlesOffset_scalar.getDatatype(), determineDatatype< uint64_t >())); - o.iterations[2].setTime(2.f); + PatchRecord& e_offset = e_patches["offset"]; + REQUIRE(e_offset.unitDimension() == ud); - ParticleSpecies& e_3 = o.iterations[3].particles["e"]; + REQUIRE(e_offset.size() == 3); + REQUIRE(e_offset.count("x") == 1); + REQUIRE(e_offset.count("y") == 1); + REQUIRE(e_offset.count("z") == 1); - std::generate(position_global.begin(), position_global.end(), [&pos]{ return pos++; }); - std::shared_ptr< double > position_local_3(new double); - e_3["position"]["x"].resetDataset(Dataset(determineDatatype(position_local_3), {4})); - std::generate(positionOffset_global.begin(), positionOffset_global.end(), [&posOff]{ return posOff++; }); - std::shared_ptr< uint64_t > positionOffset_local_3(new uint64_t); - e_3["positionOffset"]["x"].resetDataset(Dataset(determineDatatype(positionOffset_local_3), {4})); + PatchRecordComponent& e_offset_x = e_offset["x"]; + REQUIRE(e_offset_x.unitSI() == 2.599999993753294e-07); +#if !defined(_MSC_VER) + REQUIRE(e_offset_x.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_offset_x.getDatatype(), determineDatatype< uint64_t >())); - for( uint64_t i = 0; i < 4; ++i ) - { - *position_local_3 = position_global[i]; - e_3["position"]["x"].storeChunk(position_local_3, {i}, {1}); - *positionOffset_local_3 = positionOffset_global[i]; - e_3["positionOffset"]["x"].storeChunk(positionOffset_local_3, {i}, {1}); - o.flush(); - } + PatchRecordComponent& e_offset_y = e_offset["y"]; + REQUIRE(e_offset_y.unitSI() == 4.429999943501912e-08); +#if !defined(_MSC_VER) + REQUIRE(e_offset_y.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_offset_y.getDatatype(), determineDatatype< uint64_t >())); - o.setOpenPMDextension(1); - o.iterations[3].setTime(3.f); - } - REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write1.bp")); - REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write2.bp")); - REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write3.bp")); + e_offset_y.load(shareRaw(data.data())); + o.flush(); + REQUIRE(data.at(0) == static_cast< uint64_t >(0)); + REQUIRE(data.at(1) == static_cast< uint64_t >(128)); + REQUIRE(data.at(2) == static_cast< uint64_t >(256)); + REQUIRE(data.at(3) == static_cast< uint64_t >(384)); + PatchRecordComponent& e_offset_z = e_offset["z"]; + REQUIRE(e_offset_z.unitSI() == 2.599999993753294e-07); +#if !defined(_MSC_VER) + REQUIRE(e_offset_z.getDatatype() == determineDatatype< uint64_t >()); +#endif + REQUIRE(isSame(e_offset_z.getDatatype(), determineDatatype< uint64_t >())); + } catch (no_such_file_error& e) { - Series o = Series("../samples/serial_fileBased_write%T.bp", AccessType::READ_ONLY); + std::cerr << "HZDR sample not accessible. (" << e.what() << ")\n"; + return; + } +} - REQUIRE(o.iterations.size() == 3); - REQUIRE(o.iterations.count(1) == 1); - REQUIRE(o.iterations.count(2) == 1); - REQUIRE(o.iterations.count(3) == 1); +TEST_CASE( "hdf5_dtype_test", "[serial][hdf5]" ) +{ + dtype_test("h5"); +} - REQUIRE(o.iterations.at(1).time< float >() == 1.f); - REQUIRE(o.iterations.at(2).time< float >() == 2.f); - REQUIRE(o.iterations.at(3).time< float >() == 3.f); +TEST_CASE( "hdf5_write_test", "[serial][hdf5]" ) +{ + write_test("h5"); +} - REQUIRE(o.basePath() == "/data/%T/"); - REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); - REQUIRE(o.iterationFormat() == "serial_fileBased_write%T"); - REQUIRE(o.openPMD() == "1.1.0"); - REQUIRE(o.openPMDextension() == 1u); - REQUIRE(o.particlesPath() == "particles/"); - REQUIRE_FALSE(o.containsAttribute("meshesPath")); - REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error); +TEST_CASE( "hdf5_fileBased_write_empty_test", "[serial][hdf5]" ) +{ + fileBased_write_empty_test("h5"); +} - for( uint64_t i = 1; i <= 3; ++i ) - { - Iteration iteration = o.iterations.at(i); +TEST_CASE( "hdf5_fileBased_write_test", "[serial][hdf5]" ) +{ + fileBased_write_test("h5"); +} + +TEST_CASE( "hdf5_bool_test", "[serial][hdf5]" ) +{ + bool_test("h5"); +} + +TEST_CASE( "hdf5_patch_test", "[serial][hdf5]" ) +{ + patch_test("h5"); +} - REQUIRE(iteration.particles.size() == 1); - REQUIRE(iteration.particles.count("e") == 1); +TEST_CASE( "hdf5_deletion_test", "[serial][hdf5]" ) +{ + deletion_test("h5"); +} - ParticleSpecies& species = iteration.particles.at("e"); +TEST_CASE( "hdf5_110_optional_paths", "[serial][hdf5]" ) +{ + optional_paths_110_test("h5"); +} - REQUIRE(species.size() == 2); - REQUIRE(species.count("position") == 1); - REQUIRE(species.count("positionOffset") == 1); +TEST_CASE( "hdf5_constant_scalar", "[serial][hdf5]" ) +{ + constant_scalar("h5"); +} - REQUIRE(species.at("position").size() == 1); - REQUIRE(species.at("position").count("x") == 1); - REQUIRE(species.at("position").at("x").getDatatype() == Datatype::DOUBLE); - REQUIRE(species.at("position").at("x").getDimensionality() == 1); - REQUIRE(species.at("position").at("x").getExtent() == Extent{4}); - REQUIRE(species.at("positionOffset").size() == 1); - REQUIRE(species.at("positionOffset").count("x") == 1); -#if !defined(_MSC_VER) - REQUIRE(species.at("positionOffset").at("x").getDatatype() == determineDatatype< uint64_t >()); +TEST_CASE( "hdf5_particle_patches", "[serial][hdf5]" ) +{ + particle_patches("h5"); +} +#else +TEST_CASE( "no_serial_hdf5", "[serial][hdf5]" ) +{ + REQUIRE(true); +} #endif - REQUIRE(isSame(species.at("positionOffset").at("x").getDatatype(), determineDatatype< uint64_t >())); - REQUIRE(species.at("positionOffset").at("x").getDimensionality() == 1); - REQUIRE(species.at("positionOffset").at("x").getExtent() == Extent{4}); +#if openPMD_HAVE_ADIOS1 +TEST_CASE( "adios1_dtype_test", "[serial][adios1]" ) +{ + dtype_test(".bp"); +} - auto position = species.at("position").at("x").loadChunk< double >({0}, {4}); - auto position_raw = position.get(); - auto positionOffset = species.at("positionOffset").at("x").loadChunk< uint64_t >({0}, {4}); - auto positionOffset_raw = positionOffset.get(); - o.flush(); - for( uint64_t j = 0; j < 4; ++j ) - { - REQUIRE(position_raw[j] == static_cast< double >(j + (i-1)*4)); - REQUIRE(positionOffset_raw[j] == j + (i-1)*4); - } - } - } +TEST_CASE( "adios1_write_test", "[serial][adios1]") +{ + write_test("bp"); +} + +TEST_CASE( "adios1_fileBased_write_empty_test", "[serial][adios1]" ) +{ + fileBased_write_empty_test("bp"); +} + +TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" ) +{ + fileBased_write_test("bp"); } TEST_CASE( "hzdr_adios1_sample_content_test", "[serial][adios1]" ) @@ -2294,3 +2109,4 @@ TEST_CASE( "no_serial_adios1", "[serial][adios]") REQUIRE(true); } #endif + From 061029f7a42834ac0522c033447d88ba628cf055 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 12 Dec 2018 17:24:17 +0100 Subject: [PATCH 2/2] Nlohmann-JSON: isystem Should have been automatically since wrapped in an `IMPORTED` target... but for some reason did include still with `-I`. --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 417a507607..963d0297b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,8 +127,10 @@ else() endif() if(openPMD_HAVE_JSON) add_library(openPMD::thirdparty::nlohmann_json INTERFACE IMPORTED) - target_link_libraries(openPMD::thirdparty::nlohmann_json - INTERFACE nlohmann_json::nlohmann_json) + #target_link_libraries(openPMD::thirdparty::nlohmann_json + # INTERFACE nlohmann_json::nlohmann_json) + get_target_property(lib_include_dirs nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES) + target_include_directories(openPMD::thirdparty::nlohmann_json SYSTEM INTERFACE ${lib_include_dirs}) endif() # external library: HDF5 (optional)