Development of the generic format API

CMakeLists.txt

@@ -224,6 +224,7 @@ add_subdirectory (macro)
 add_subdirectory (o2cdb)
 add_subdirectory (test)
 add_subdirectory (o2qa)
+add_subdirectory (DataFormats)
 
 
 Option(BUILD_DOXYGEN "Build Doxygen" OFF)

DataFormats/CMakeLists.txt

@@ -0,0 +1,3 @@
+# @brief  cmake setup for the DataFormats module of AliceO2
+
+add_subdirectory (Generic)

DataFormats/Generic/CMakeLists.txt

@@ -0,0 +1,3 @@
+# @brief  cmake setup for DataFormats/Generic of AliceO2
+
+add_subdirectory (test)

DataFormats/Generic/message_list.cxx

@@ -0,0 +1,10 @@
+//  @file   message_list.cxx
+//  @author Matthias Richter
+//  @since  2016-02-11
+//  @brief  Container class for messages in the O2 framework
+
+#include "message_list.h"
+#include "memory-format.h"
+
+using namespace AliceO2;
+using namespace Format;

DataFormats/Generic/message_list.h

@@ -0,0 +1,185 @@
+//-*- Mode: C++ -*-
+
+#ifndef MESSAGELIST_H
+#define MESSAGELIST_H
+//****************************************************************************
+//* This file is free software: you can redistribute it and/or modify        *
+//* it under the terms of the GNU General Public License as published by     *
+//* the Free Software Foundation, either version 3 of the License, or	     *
+//* (at your option) any later version.					     *
+//*                                                                          *
+//* Primary Authors: Matthias Richter <richterm@scieq.net>                   *
+//*                                                                          *
+//* The authors make no claims about the suitability of this software for    *
+//* any purpose. It is provided "as is" without express or implied warranty. *
+//****************************************************************************
+
+//  @file   message_list.h
+//  @author Matthias Richter
+//  @since  2016-02-11
+//  @brief  Container class for messages in the O2 framework
+
+#include <cstdint>
+#include <cstddef>
+#include <vector>
+#include <cstring> // memset
+#include <functional> // std::function
+
+namespace AliceO2 {
+namespace Format {
+
+// Ideally it does not matter for the implementation of the container class
+// whether the type is the message container or the pointer to the payload
+template<class MsgT, class HdrT>
+class messageList {
+ public:
+  typedef MsgT message_type;
+  typedef HdrT header_type;
+  /// comparison metric for selection of elements
+  /// an external function can be defined by the caller of begin() to
+  /// apply a selection of elements
+  typedef std::function<bool(const HdrT& hdr)> HdrComparison;
+
+  messageList() {}
+  messageList(const messageList& other); // not yet implemented
+  messageList& operator=(const messageList& other); // not yet implemented
+  ~messageList() {}
+
+  /// add data block to list
+  /// both header and payload message parts are required to add an entry
+  /// the actual header of type HdrT is extracted from the header
+  /// message part.
+  int add(MsgT& headerMsg, MsgT& payloadMsg) {
+    // conversion relies on the conversion operator for complex types
+    const uint8_t* headerData = headerMsg;
+
+    const HdrT* srcHeader = reinterpret_cast<const HdrT*>(headerData);
+    // TODO: consistency check
+    mDataArray.push_back(messagePair(*srcHeader, payloadMsg));
+
+    return mDataArray.size();
+  }
+  /** number of data blocks in the list */
+  size_t size() {return mDataArray.size();}
+  /** clear the list */
+  void clear() {mDataArray.clear();}
+  /** check if list is empty */
+  bool empty() {mDataArray.empty();}
+
+  /**
+   * messagePair describes the two sequential message parts for header and payload
+   * respectively.
+   *
+   * TODO: decide whether to use pointer to message or pointer to payload in the
+   * message. Whith the template approach and appropriate conversion operators in the
+   * message class possibly both ways can be served at the same time
+   */
+  struct messagePair {
+    HdrT  mHeader;
+    MsgT* mPayload;
+
+    messagePair(MsgT& payload) : mHeader(), mPayload(&payload) {
+      memset(&mHeader, 0, sizeof(HdrT));
+    }
+
+    messagePair(const HdrT& header, MsgT& payload) : mHeader(), mPayload(&payload) {
+      memcpy(&mHeader, &header, sizeof(HdrT));
+    }
+  };
+  typedef typename std::vector<messagePair>::iterator pairIt_t;
+  // TODO: operators inside a class can only have one parameter
+  // check whether to create a functor class
+  //bool operator==(const messageList::pairIt_t& first, const messageList::pairIt_t& second) {
+  //  return (first->mHeader == second->mHeader) && (first->mPayload == second->mPayload);
+  //}
+  //
+  //bool operator!=(const messageList::pairIt_t& first, const messageList::pairIt_t& second) {
+  //  return (first->mHeader != second->mHeader) || (first->mPayload != second->mPayload);
+  //}
+
+  /**
+   * @class iterator
+   * Implementation of navigation through the list and access to elements.
+   *
+   * An optional comparison metric @ref HdrComparison can be used to provide
+   * a selection of elements.
+   */
+  class iterator {
+   public:
+    typedef iterator self_type;
+    typedef MsgT value_type;
+
+    iterator(const pairIt_t& dataIterator, const pairIt_t& iteratorRange,
+	     const HdrComparison hdrsel = HdrComparison())
+      : mDataIterator(dataIterator)
+      , mEnd(iteratorRange)
+      , mHdrSelection(hdrsel)
+    { }
+    iterator(const pairIt_t& dataIterator)
+      : mDataIterator(dataIterator)
+      , mEnd(dataIterator)
+      , mHdrSelection(HdrComparison())
+    { }
+    // prefix increment
+    self_type& operator++() {
+      while (++mDataIterator != mEnd) {
+	// operator bool() of std::function is used to determine whether
+	// a selector is set or not, the default is not callable.
+	// if the std::function container has an assigned target, this is
+	// called with the header as parameter
+	if (!mHdrSelection || mHdrSelection(mDataIterator->mHeader)) break;
+      }
+      return *this;
+    }
+    // postfix increment
+    self_type operator++(int unused) {self_type copy(*this); ++*this; return copy;}
+    // TODO: given the fact that the data which is hold is a pointer, it always needs to be
+    // valid for dereference
+    MsgT& operator*() { return *((*mDataIterator).mPayload);}
+    MsgT* operator->() { return (*mDataIterator).mPayload;}
+
+    /** return header at iterator position */
+    HdrT& getHdr() const {return (*mDataIterator).mHeader;}
+
+    bool operator==(const self_type& other) { return mDataIterator == other.mDataIterator; }
+    bool operator!=(const self_type& other) { return mDataIterator != other.mDataIterator; }
+
+    /** conversion operator to PayloadMetaData_t struct */
+    operator HdrT() {return (*mDataIterator).mHeader;}
+    /** return size of payload */
+    size_t size() {return (*mDataIterator).mHeader.mPayloadSize;}
+
+  private:
+    pairIt_t mDataIterator;
+    pairIt_t mEnd;
+    HdrComparison mHdrSelection;
+  };
+
+  /** to be defined
+  class const_iterator
+  {
+  };
+  */
+
+  iterator begin(const HdrComparison hdrsel = HdrComparison()) {
+    iterator ret(mDataArray.begin(), mDataArray.end(), hdrsel);
+    // if the std::function container has an assigned target, this is
+    // is used used to check whether the iterator matches the selection
+    // further checks are in the increment operator.
+    // the iterator class implements a type cast operator which allows
+    // to use it directly in the HdrComparison
+    if (hdrsel && !hdrsel(ret)) ++ret;
+    return ret;
+  }
+
+  iterator end() {
+    return iterator(mDataArray.end());
+  }
+
+ private:
+  std::vector<messagePair> mDataArray;
+};
+
+}; // namespace Format
+}; // namespace AliceO2
+#endif

DataFormats/Generic/test/CMakeLists.txt

@@ -0,0 +1,59 @@
+# @author Matthias Richter
+# @brief  cmake setup for the test of generic format implementation
+
+set(INCLUDE_DIRECTORIES
+  ${CMAKE_CURRENT_SOURCE_DIR}/..
+)
+
+set(SYSTEM_INCLUDE_DIRECTORIES
+  ${Boost_INCLUDE_DIR}
+)
+
+include_directories(${INCLUDE_DIRECTORIES})
+include_directories(SYSTEM ${SYSTEM_INCLUDE_DIRECTORIES})
+
+set(LINK_DIRECTORIES
+  ${Boost_LIBRARY_DIRS}
+)
+
+link_directories(${LINK_DIRECTORIES})
+
+#library source
+set(SRCS
+)
+
+set(DEPENDENCIES
+    ${DEPENDENCIES}
+  )
+
+set(DEPENDENCIES
+  ${DEPENDENCIES}
+  ${CMAKE_THREAD_LIBS_INIT}
+)
+
+set(SRCS
+)
+
+Set(Exe_Names
+)
+
+set(Exe_Source
+)
+
+list(LENGTH Exe_Names _length)
+
+if(${_length})
+math(EXPR _length ${_length}-1)
+ForEach(_file RANGE 0 ${_length})
+  list(GET Exe_Names ${_file} _name)
+  list(GET Exe_Source ${_file} _src)
+  set(EXE_NAME ${_name})
+  set(SRCS ${_src})
+  set(DEPENDENCIES ALICEHLT dl)
+  GENERATE_EXECUTABLE()
+EndForEach(_file RANGE 0 ${_length})
+endif(${_length})
+
+# avoid installation of test exe by not using macro GENERATE_EXECUTABLE
+add_executable(testMessageList testMessageList.cxx)
+add_test(testMessageList ${CMAKE_BINARY_DIR}/bin/testMessageList)

DataFormats/Generic/test/header_versions.h

@@ -0,0 +1,5 @@
+#ifndef HEADER_VERSIONS_H
+#define HEADER_VERSIONS_H
+#endif
+
+//TODO: to be filled

DataFormats/Generic/test/memory-format.h

@@ -0,0 +1,114 @@
+#ifndef MEMORY_FORMAT_H
+#define MEMORY_FORMAT_H
+//****************************************************************************
+//* This file is free software: you can redistribute it and/or modify        *
+//* it under the terms of the GNU General Public License as published by     *
+//* the Free Software Foundation, either version 3 of the License, or	     *
+//* (at your option) any later version.					     *
+//*                                                                          *
+//* Primary Authors: Matthias Richter <richterm@scieq.net>                   *
+//*                                                                          *
+//* The authors make no claims about the suitability of this software for    *
+//* any purpose. It is provided "as is" without express or implied warranty. *
+//****************************************************************************
+
+/// @file   memory_format.h
+/// @author Matthias Richter
+/// @since  2016-01-28
+/// @brief  Helper structs for the ALICE O2 generic format API test
+/// @note   DO NOT USE OUTSIDE UNIT TEST OF FORMAT API
+///         This definitions have been a first draft during discussion of
+///         in memory data formats, the final header file has been placed
+///         elsewhere, but this file is temporarily kept for the unit test
+
+// use the standard definitions of int variables
+#include <stdint.h>
+#include "header_versions.h"
+
+/**
+The defined headers are a tradeoff to provide the necessary information in a lightweight way and to allow for evolution and compatibility.
+
+General header format
+- Starts with basic header information, never serialized, with unique version number
+- Strict policy enforced: no changes to members (e.g. width) or sequence of members
+- New members can be appended
+- All basic header structs are defined with fixed endianess and padding
+- Header-stack concept: optional headers can follow the basic header
+*/
+namespace AliceO2 {
+namespace Format {
+  /**
+   * Data header to be commonly used for all in-memory data blocks
+   *
+   * Unique header version; struct size included for consistency check
+   * and to facilitate later implementation of conversion handlers.
+   *
+   * A magic string makes identification of header simpler, e.g. after
+   * a data corruption; great help for low level debugging
+   *
+   * PayloadSize is a redundant information, to be used for integrity
+   * check and mandatory for disk dumped data
+   *
+   * Payload serialization method defined in the header, allows to build
+   * common functionality. Framework can choose the right tool for
+   * de-serialization
+   */
+  struct  DataHeader_t {
+    /** 4 bytes of a magic string */
+    int32_t  mMagicString;
+    /** size of the struct */
+    int32_t  mStructSize;
+    /** header version, bookkeeping in the software */
+    int32_t  mHeaderVersion;
+    /** Flags field, valid bits and their meaning defined by the header version */
+    int32_t  mFlags;
+    /** size of payload in memory */
+    int64_t  mPayloadSize;
+    /** payload serialization method for transfer */
+    char mPayloadSerializationMethod[7+1];
+    /** Payload meta data: Subsystem or detector */
+    char mDataOrigin[3+1];
+    /** Payload meta data: Data description, e.g. raw, clusters, tracks */
+    char mDataDescriptor[15+1];
+    /** Payload meta data: A system or detector specific sub specification */
+    int64_t mSubSpec;
+  };
+
+  /**
+   * Helper struct for the payload meta data
+   *
+   * This struct is an addition to DataHeader_t to allow implementation
+   * of operators. All meta data members are directly included in DataHeader_t
+   * for easier access and consistency.
+   */
+  struct PayloadMetaData_t {
+    /** Subsystem or detector */
+    char mDataOrigin[3+1];
+    /** Data description, e.g. raw, clusters, tracks */
+    char mDataDescriptor[15+1];
+    /** A system or detector specific sub specification */
+    int64_t mSubSpec;
+  };
+
+  /**
+   * Header-stack:: optional headers can follow the basic header
+   * A next header is indicated in the flag member of preceeding header
+   * Optional headers consist of a fixed NextHeaderDescription and a variable
+   * NextHeaderContent
+   */
+  struct NextHeaderDescription_t {
+    /** size of this next header description */
+    int32_t mStructSize;
+    /** size of the next header payload */
+    int32_t mNextHeaderContentSize;
+    /** Common flags for all next-headers, includes next-header flag */
+    int32_t mFlags;
+    /** Descriptor */
+    char mHeaderDescriptor[15+1];
+    /** serialization method */
+    char mSerializationMethod[7+1];
+  };
+
+}; // namespace Format
+}; // namespace AliceO2
+#endif