diff --git a/CMakeLists.txt b/CMakeLists.txt index c0f4191ef9397..01ae1f7216365 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,6 +224,7 @@ add_subdirectory (macro) add_subdirectory (o2cdb) add_subdirectory (test) add_subdirectory (o2qa) +add_subdirectory (DataFormats) Option(BUILD_DOXYGEN "Build Doxygen" OFF) diff --git a/DataFormats/CMakeLists.txt b/DataFormats/CMakeLists.txt new file mode 100644 index 0000000000000..ea3c28753faeb --- /dev/null +++ b/DataFormats/CMakeLists.txt @@ -0,0 +1,3 @@ +# @brief cmake setup for the DataFormats module of AliceO2 + +add_subdirectory (Generic) diff --git a/DataFormats/Generic/CMakeLists.txt b/DataFormats/Generic/CMakeLists.txt new file mode 100644 index 0000000000000..326b52234a511 --- /dev/null +++ b/DataFormats/Generic/CMakeLists.txt @@ -0,0 +1,3 @@ +# @brief cmake setup for DataFormats/Generic of AliceO2 + +add_subdirectory (test) diff --git a/DataFormats/Generic/message_list.cxx b/DataFormats/Generic/message_list.cxx new file mode 100644 index 0000000000000..ffbabc3f93fc6 --- /dev/null +++ b/DataFormats/Generic/message_list.cxx @@ -0,0 +1,10 @@ +// @file message_list.cxx +// @author Matthias Richter +// @since 2016-02-11 +// @brief Container class for messages in the O2 framework + +#include "message_list.h" +#include "memory-format.h" + +using namespace AliceO2; +using namespace Format; diff --git a/DataFormats/Generic/message_list.h b/DataFormats/Generic/message_list.h new file mode 100644 index 0000000000000..068cffa1bee47 --- /dev/null +++ b/DataFormats/Generic/message_list.h @@ -0,0 +1,185 @@ +//-*- Mode: C++ -*- + +#ifndef MESSAGELIST_H +#define MESSAGELIST_H +//**************************************************************************** +//* This file is free software: you can redistribute it and/or modify * +//* it under the terms of the GNU General Public License as published by * +//* the Free Software Foundation, either version 3 of the License, or * +//* (at your option) any later version. * +//* * +//* Primary Authors: Matthias Richter * +//* * +//* The authors make no claims about the suitability of this software for * +//* any purpose. It is provided "as is" without express or implied warranty. * +//**************************************************************************** + +// @file message_list.h +// @author Matthias Richter +// @since 2016-02-11 +// @brief Container class for messages in the O2 framework + +#include +#include +#include +#include // memset +#include // std::function + +namespace AliceO2 { +namespace Format { + +// Ideally it does not matter for the implementation of the container class +// whether the type is the message container or the pointer to the payload +template +class messageList { + public: + typedef MsgT message_type; + typedef HdrT header_type; + /// comparison metric for selection of elements + /// an external function can be defined by the caller of begin() to + /// apply a selection of elements + typedef std::function HdrComparison; + + messageList() {} + messageList(const messageList& other); // not yet implemented + messageList& operator=(const messageList& other); // not yet implemented + ~messageList() {} + + /// add data block to list + /// both header and payload message parts are required to add an entry + /// the actual header of type HdrT is extracted from the header + /// message part. + int add(MsgT& headerMsg, MsgT& payloadMsg) { + // conversion relies on the conversion operator for complex types + const uint8_t* headerData = headerMsg; + + const HdrT* srcHeader = reinterpret_cast(headerData); + // TODO: consistency check + mDataArray.push_back(messagePair(*srcHeader, payloadMsg)); + + return mDataArray.size(); + } + /** number of data blocks in the list */ + size_t size() {return mDataArray.size();} + /** clear the list */ + void clear() {mDataArray.clear();} + /** check if list is empty */ + bool empty() {mDataArray.empty();} + + /** + * messagePair describes the two sequential message parts for header and payload + * respectively. + * + * TODO: decide whether to use pointer to message or pointer to payload in the + * message. Whith the template approach and appropriate conversion operators in the + * message class possibly both ways can be served at the same time + */ + struct messagePair { + HdrT mHeader; + MsgT* mPayload; + + messagePair(MsgT& payload) : mHeader(), mPayload(&payload) { + memset(&mHeader, 0, sizeof(HdrT)); + } + + messagePair(const HdrT& header, MsgT& payload) : mHeader(), mPayload(&payload) { + memcpy(&mHeader, &header, sizeof(HdrT)); + } + }; + typedef typename std::vector::iterator pairIt_t; + // TODO: operators inside a class can only have one parameter + // check whether to create a functor class + //bool operator==(const messageList::pairIt_t& first, const messageList::pairIt_t& second) { + // return (first->mHeader == second->mHeader) && (first->mPayload == second->mPayload); + //} + // + //bool operator!=(const messageList::pairIt_t& first, const messageList::pairIt_t& second) { + // return (first->mHeader != second->mHeader) || (first->mPayload != second->mPayload); + //} + + /** + * @class iterator + * Implementation of navigation through the list and access to elements. + * + * An optional comparison metric @ref HdrComparison can be used to provide + * a selection of elements. + */ + class iterator { + public: + typedef iterator self_type; + typedef MsgT value_type; + + iterator(const pairIt_t& dataIterator, const pairIt_t& iteratorRange, + const HdrComparison hdrsel = HdrComparison()) + : mDataIterator(dataIterator) + , mEnd(iteratorRange) + , mHdrSelection(hdrsel) + { } + iterator(const pairIt_t& dataIterator) + : mDataIterator(dataIterator) + , mEnd(dataIterator) + , mHdrSelection(HdrComparison()) + { } + // prefix increment + self_type& operator++() { + while (++mDataIterator != mEnd) { + // operator bool() of std::function is used to determine whether + // a selector is set or not, the default is not callable. + // if the std::function container has an assigned target, this is + // called with the header as parameter + if (!mHdrSelection || mHdrSelection(mDataIterator->mHeader)) break; + } + return *this; + } + // postfix increment + self_type operator++(int unused) {self_type copy(*this); ++*this; return copy;} + // TODO: given the fact that the data which is hold is a pointer, it always needs to be + // valid for dereference + MsgT& operator*() { return *((*mDataIterator).mPayload);} + MsgT* operator->() { return (*mDataIterator).mPayload;} + + /** return header at iterator position */ + HdrT& getHdr() const {return (*mDataIterator).mHeader;} + + bool operator==(const self_type& other) { return mDataIterator == other.mDataIterator; } + bool operator!=(const self_type& other) { return mDataIterator != other.mDataIterator; } + + /** conversion operator to PayloadMetaData_t struct */ + operator HdrT() {return (*mDataIterator).mHeader;} + /** return size of payload */ + size_t size() {return (*mDataIterator).mHeader.mPayloadSize;} + + private: + pairIt_t mDataIterator; + pairIt_t mEnd; + HdrComparison mHdrSelection; + }; + + /** to be defined + class const_iterator + { + }; + */ + + iterator begin(const HdrComparison hdrsel = HdrComparison()) { + iterator ret(mDataArray.begin(), mDataArray.end(), hdrsel); + // if the std::function container has an assigned target, this is + // is used used to check whether the iterator matches the selection + // further checks are in the increment operator. + // the iterator class implements a type cast operator which allows + // to use it directly in the HdrComparison + if (hdrsel && !hdrsel(ret)) ++ret; + return ret; + } + + iterator end() { + return iterator(mDataArray.end()); + } + + private: + std::vector mDataArray; +}; + +}; // namespace Format +}; // namespace AliceO2 +#endif diff --git a/DataFormats/Generic/test/CMakeLists.txt b/DataFormats/Generic/test/CMakeLists.txt new file mode 100644 index 0000000000000..2460a5565406e --- /dev/null +++ b/DataFormats/Generic/test/CMakeLists.txt @@ -0,0 +1,59 @@ +# @author Matthias Richter +# @brief cmake setup for the test of generic format implementation + +set(INCLUDE_DIRECTORIES + ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +set(SYSTEM_INCLUDE_DIRECTORIES + ${Boost_INCLUDE_DIR} +) + +include_directories(${INCLUDE_DIRECTORIES}) +include_directories(SYSTEM ${SYSTEM_INCLUDE_DIRECTORIES}) + +set(LINK_DIRECTORIES + ${Boost_LIBRARY_DIRS} +) + +link_directories(${LINK_DIRECTORIES}) + +#library source +set(SRCS +) + +set(DEPENDENCIES + ${DEPENDENCIES} + ) + +set(DEPENDENCIES + ${DEPENDENCIES} + ${CMAKE_THREAD_LIBS_INIT} +) + +set(SRCS +) + +Set(Exe_Names +) + +set(Exe_Source +) + +list(LENGTH Exe_Names _length) + +if(${_length}) +math(EXPR _length ${_length}-1) +ForEach(_file RANGE 0 ${_length}) + list(GET Exe_Names ${_file} _name) + list(GET Exe_Source ${_file} _src) + set(EXE_NAME ${_name}) + set(SRCS ${_src}) + set(DEPENDENCIES ALICEHLT dl) + GENERATE_EXECUTABLE() +EndForEach(_file RANGE 0 ${_length}) +endif(${_length}) + +# avoid installation of test exe by not using macro GENERATE_EXECUTABLE +add_executable(testMessageList testMessageList.cxx) +add_test(testMessageList ${CMAKE_BINARY_DIR}/bin/testMessageList) diff --git a/DataFormats/Generic/test/header_versions.h b/DataFormats/Generic/test/header_versions.h new file mode 100644 index 0000000000000..2c902f01af21c --- /dev/null +++ b/DataFormats/Generic/test/header_versions.h @@ -0,0 +1,5 @@ +#ifndef HEADER_VERSIONS_H +#define HEADER_VERSIONS_H +#endif + +//TODO: to be filled diff --git a/DataFormats/Generic/test/memory-format.h b/DataFormats/Generic/test/memory-format.h new file mode 100644 index 0000000000000..d216cf5657e43 --- /dev/null +++ b/DataFormats/Generic/test/memory-format.h @@ -0,0 +1,114 @@ +#ifndef MEMORY_FORMAT_H +#define MEMORY_FORMAT_H +//**************************************************************************** +//* This file is free software: you can redistribute it and/or modify * +//* it under the terms of the GNU General Public License as published by * +//* the Free Software Foundation, either version 3 of the License, or * +//* (at your option) any later version. * +//* * +//* Primary Authors: Matthias Richter * +//* * +//* The authors make no claims about the suitability of this software for * +//* any purpose. It is provided "as is" without express or implied warranty. * +//**************************************************************************** + +/// @file memory_format.h +/// @author Matthias Richter +/// @since 2016-01-28 +/// @brief Helper structs for the ALICE O2 generic format API test +/// @note DO NOT USE OUTSIDE UNIT TEST OF FORMAT API +/// This definitions have been a first draft during discussion of +/// in memory data formats, the final header file has been placed +/// elsewhere, but this file is temporarily kept for the unit test + +// use the standard definitions of int variables +#include +#include "header_versions.h" + +/** +The defined headers are a tradeoff to provide the necessary information in a lightweight way and to allow for evolution and compatibility. + +General header format +- Starts with basic header information, never serialized, with unique version number +- Strict policy enforced: no changes to members (e.g. width) or sequence of members +- New members can be appended +- All basic header structs are defined with fixed endianess and padding +- Header-stack concept: optional headers can follow the basic header +*/ +namespace AliceO2 { +namespace Format { + /** + * Data header to be commonly used for all in-memory data blocks + * + * Unique header version; struct size included for consistency check + * and to facilitate later implementation of conversion handlers. + * + * A magic string makes identification of header simpler, e.g. after + * a data corruption; great help for low level debugging + * + * PayloadSize is a redundant information, to be used for integrity + * check and mandatory for disk dumped data + * + * Payload serialization method defined in the header, allows to build + * common functionality. Framework can choose the right tool for + * de-serialization + */ + struct DataHeader_t { + /** 4 bytes of a magic string */ + int32_t mMagicString; + /** size of the struct */ + int32_t mStructSize; + /** header version, bookkeeping in the software */ + int32_t mHeaderVersion; + /** Flags field, valid bits and their meaning defined by the header version */ + int32_t mFlags; + /** size of payload in memory */ + int64_t mPayloadSize; + /** payload serialization method for transfer */ + char mPayloadSerializationMethod[7+1]; + /** Payload meta data: Subsystem or detector */ + char mDataOrigin[3+1]; + /** Payload meta data: Data description, e.g. raw, clusters, tracks */ + char mDataDescriptor[15+1]; + /** Payload meta data: A system or detector specific sub specification */ + int64_t mSubSpec; + }; + + /** + * Helper struct for the payload meta data + * + * This struct is an addition to DataHeader_t to allow implementation + * of operators. All meta data members are directly included in DataHeader_t + * for easier access and consistency. + */ + struct PayloadMetaData_t { + /** Subsystem or detector */ + char mDataOrigin[3+1]; + /** Data description, e.g. raw, clusters, tracks */ + char mDataDescriptor[15+1]; + /** A system or detector specific sub specification */ + int64_t mSubSpec; + }; + + /** + * Header-stack:: optional headers can follow the basic header + * A next header is indicated in the flag member of preceeding header + * Optional headers consist of a fixed NextHeaderDescription and a variable + * NextHeaderContent + */ + struct NextHeaderDescription_t { + /** size of this next header description */ + int32_t mStructSize; + /** size of the next header payload */ + int32_t mNextHeaderContentSize; + /** Common flags for all next-headers, includes next-header flag */ + int32_t mFlags; + /** Descriptor */ + char mHeaderDescriptor[15+1]; + /** serialization method */ + char mSerializationMethod[7+1]; + }; + +}; // namespace Format +}; // namespace AliceO2 +#endif diff --git a/DataFormats/Generic/test/testMessageList.cxx b/DataFormats/Generic/test/testMessageList.cxx new file mode 100644 index 0000000000000..f7726975c8ac9 --- /dev/null +++ b/DataFormats/Generic/test/testMessageList.cxx @@ -0,0 +1,125 @@ +/// @file testMessageList.cxx +/// @brief Unit test for message_list.h template class + +// TODO: this is for the moment only testing compilation +// ideally, this test program is only compiled when the test +// is going to be executed. By using add_executable in the cmake +// configuartion, the program is always build (though not installed) + +#include "message_list.h" +#include "memory-format.h" + +#include // memset +#include + +using namespace AliceO2; +using namespace Format; + +// a simple message type, just a pointer to some payload +typedef const uint8_t* SimpleMsg_t; + +// a simple header definition +struct SimpleHeader_t { + uint32_t id; + uint32_t specification; + + SimpleHeader_t() : id(0), specification(0) {} + SimpleHeader_t(uint32_t _id, uint32_t _spec) : id(_id), specification(_spec) {} +}; + +// print operator for the simple header +std::ostream& operator<<(std::ostream& stream, SimpleHeader_t header) { + stream << "Header ID: " << header.id << std::endl; + stream << "Header Specification: " << std::hex << header.specification; + + return stream; +} + +// more complex message type, some class which wraps around payload +// implements type conversion operator to return pointer to payload +// buffer +class TestMsg { +public: + TestMsg() : mBuffer(nullptr), mBufferSize(0) {} + TestMsg(const uint8_t* buffer, unsigned size) : mBuffer(new uint8_t[size]), mBufferSize(size) { + memcpy(mBuffer, buffer, size); + } + ~TestMsg() {clear();} + + int alloc(int size) { + // not yet implemented + return 0; + } + + uint8_t* get() const { + return mBuffer; + } + + operator uint8_t*() { return get();} + + void clear() { + if (mBuffer) { + delete mBuffer; + } + mBuffer = NULL; + mBufferSize = 0; + } + +private: + uint8_t* mBuffer; + unsigned mBufferSize; +}; + +// helper function to print entries of the message list by using +// iterator +template +void print_list(ListType& list, typename ListType::HdrComparison hdrsel = typename ListType::HdrComparison()) { + for (typename ListType::iterator it = list.begin(hdrsel); + it != list.end(); + ++it) { + // the iterator defines a conversion operator to the header type + std::cout << static_cast(it) << std::endl; + // dereferencing of the iterator gives the payload + std::cout << *it << std::endl; + } +} + +int main(int argc, char** argv) +{ + int iResult = 0; + + typedef messageList SimpleList_t; + SimpleList_t input; + input.size(); + + SimpleHeader_t hdr1(1, 0xdeadbeef); + const char* payload1="payload1"; + SimpleMsg_t headerMsg1 = reinterpret_cast(&hdr1); + SimpleMsg_t payloadMsg1 = reinterpret_cast(payload1); + input.add(headerMsg1, payloadMsg1); + + SimpleHeader_t hdr2(2, 0xf00); + const char* payload2="payload2"; + SimpleMsg_t headerMsg2 = reinterpret_cast(&hdr2); + SimpleMsg_t payloadMsg2 = reinterpret_cast(payload2); + input.add(headerMsg2, payloadMsg2); + + std::cout << "simple list:" << std::endl; + print_list(input); + std::cout << std::endl; + + typedef messageList TestMsgList_t; + TestMsgList_t msglist; + TestMsg testHeaderMsg1((uint8_t*)&hdr1, sizeof(hdr1)); + TestMsg testPayloadMsg1((uint8_t*)payload1, strlen(payload1)+1); + msglist.add(testHeaderMsg1, testPayloadMsg1); + TestMsg testHeaderMsg2((uint8_t*)&hdr2, sizeof(hdr2)); + TestMsg testPayloadMsg2((uint8_t*)payload2, strlen(payload2)+1); + msglist.add(testHeaderMsg2, testPayloadMsg2); + + std::cout << "message class list:" << std::endl; + print_list(msglist, [](const TestMsgList_t::header_type& hdr){return hdr.specification==0xf00;} ); + std::cout << std::endl; + + return 0; +}