Merge pull request #9 from sy-c/master

added numa support

Author: sy-c

CMakeLists.txt

@@ -63,32 +63,18 @@ set(MODULE_NAME "Readout")
 
 O2_SETUP(NAME ${MODULE_NAME})
 
-set(SRCS
-        src/Consumer.cxx
-        src/ConsumerStats.cxx
-        src/ConsumerFileRecorder.cxx
-        src/ConsumerDataChecker.cxx
-        src/ConsumerDataSampling.cxx
-        #    src/ConsumerFMQ.cxx
-        src/ReadoutEquipment.cxx
-        src/ReadoutEquipmentDummy.cxx
-        src/ReadoutEquipmentRORC.cxx
-        src/DataBlockAggregator.cxx
-        src/mainReadout.cxx
-        src/ReadoutUtils.cxx
-        )
-
 include_directories(
-        ${CMAKE_CURRENT_SOURCE_DIR}/../RORC/include
         ${Boost_INCLUDE_DIRS}
         ${Monitoring_INCLUDE_DIRS}
         ${FAIRROOT_INCLUDE_DIR}
         ${FAIRROOT_INCLUDE_DIR}/fairmq
-        ${ZeroMQ_INCLUDE_DIR}
+#        ${ZeroMQ_INCLUDE_DIR}
         ${Common_INCLUDE_DIRS}
         ${Configuration_INCLUDE_DIRS}
         ${InfoLogger_INCLUDE_DIRS}
         ${ReadoutCard_INCLUDE_DIRS}
+        ${DataSampling_INCLUDE_DIRS}
+        ${Numa_INCLUDE_DIRS}       
 )
 
 add_library(
@@ -131,17 +117,11 @@ add_library(
         ${CONSUMERS_SRCS}
 )
 
-
-set(LIBRARY_NAME ${MODULE_NAME})
 set(BUCKET_NAME o2_readout_bucket)
-
 if (FAIRROOT_FOUND)
-    list(APPEND SRCS src/ConsumerFMQ.cxx)
     set(BUCKET_NAME o2_readout_with_fair)
 endif ()
 
-O2_GENERATE_LIBRARY()
-
 add_subdirectory(doc)
 
 O2_GENERATE_EXECUTABLE(
@@ -183,7 +163,7 @@ O2_GENERATE_EXECUTABLE(
 O2_GENERATE_EXECUTABLE(
         EXE_NAME testROC.exe
         SOURCES src/testROC.cxx $<TARGET_OBJECTS:objMemUtils>
-        BUCKET_NAME o2_readout_numa_bucket
+        BUCKET_NAME ${BUCKET_NAME}
 )
 
 # Install some extra files

cmake/FindNuma.cmake

@@ -0,0 +1,31 @@
+# Try to find the Numa package include dirs and libraries
+# Author: Sylvain Chapeland
+#
+# This script will set the following variables:
+#  Numa_FOUND - System has Numa
+#  Numa_INCLUDE_DIRS - The Numa include directories
+#  Numa_LIBRARIES - The libraries needed to use Numa
+#
+# This script can use the following variables:
+#  Numa_ROOT - Installation root to tell this module where to look.
+
+
+find_path(
+  Numa_INCLUDE_DIRS NAMES numa.h
+  PATHS ${Numa_ROOT} ${Numa_ROOT}/include)
+  
+find_library(
+  Numa_LIBRARIES NAMES numa
+  PATHS ${Numa_ROOT} ${Numa_ROOT}/lib)
+  
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+  Numa DEFAULT_MSG Numa_INCLUDE_DIRS Numa_LIBRARIES)
+
+if(${Numa_FOUND})
+    message(
+    STATUS
+    "Found Numa (include: ${Numa_INCLUDE_DIRS} library: ${Numa_LIBRARIES}")    
+endif()
+
+mark_as_advanced(Numa_INCLUDE_DIRS Numa_LIBRARIES)

cmake/ReadoutDependencies.cmake

@@ -7,7 +7,8 @@ find_package(Common REQUIRED)
 find_package(InfoLogger REQUIRED)
 find_package(ReadoutCard REQUIRED)
 find_package(DataSampling REQUIRED)
-find_package(ZeroMQ REQUIRED)
+#find_package(ZeroMQ REQUIRED)
+find_package(Numa)
 
 if (FAIRROOT_FOUND)
     # this should go away when fairrot provides a proper Find script or proper config scripts
@@ -19,6 +20,14 @@ else (FAIRROOT_FOUND)
     message(WARNING "FairRoot not found, corresponding classes will not be compiled.")
 endif (FAIRROOT_FOUND)
 
+if (Numa_FOUND)   
+  ADD_DEFINITIONS(-DWITH_NUMA)
+else (Numa_FOUND)
+  message(WARNING "Numa not found, corresponding features will be disabled.")
+  set(Numa_LIBRARIES "")
+  set(Numa_INCLUDE_DIRS "")
+endif (Numa_FOUND)
+
 ADD_DEFINITIONS(-DWITH_DATASAMPLING)
 
 o2_define_bucket(
@@ -37,6 +46,7 @@ o2_define_bucket(
         ${InfoLogger_LIBRARIES}
         ${ReadoutCard_LIBRARIES}
         ${DataSampling_LIBRARIES}
+        ${Numa_LIBRARIES}
 
         SYSTEMINCLUDE_DIRECTORIES
         ${Boost_INCLUDE_DIRS}
@@ -45,6 +55,7 @@ o2_define_bucket(
         ${ReadoutCard_INCLUDE_DIRS}
         ${DataSampling_INCLUDE_DIRS}
         ${Configuration_INCLUDE_DIRS}
+        ${Numa_INCLUDE_DIRS}
 )
 
 
@@ -61,12 +72,3 @@ o2_define_bucket(
         ${FAIRROOT_INCLUDE_DIR}
         ${FAIRROOT_INCLUDE_DIR}/fairmq
 )
-
-o2_define_bucket(
-	NAME
-	o2_readout_numa_bucket
-	
-	DEPENDENCIES
-	o2_readout_bucket
-	numa
-)

src/mainReadout.cxx

@@ -33,6 +33,9 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
+#ifdef WITH_NUMA
+#include <numa.h>
+#endif
 
 // option to add callgrind instrumentation
 // to use: valgrind --tool=callgrind --instr-atstart=no --dump-instr=yes ./a.out
@@ -100,7 +103,11 @@ int main(int argc, char* argv[])
   #else
     theLog.log("FAIRMQ : no");
   #endif
-
+  #ifdef WITH_NUMA
+    theLog.log("NUMA : yes");
+  #else
+    theLog.log("NUMA : no");
+  #endif
   // load configuration file
   theLog.log("Reading configuration from %s",cfgFileURI);
   try {
@@ -114,7 +121,7 @@ int main(int argc, char* argv[])
 
   // try to prevent deep sleeps  
   theLog.log("Disabling CPU deep sleep for process");
-  int maxLatency=1;
+  int maxLatency=2;
   int latencyFd = open("/dev/cpu_dma_latency", O_WRONLY);
   if (latencyFd < 0) {
     theLog.log("Error opening /dev/cpu_dma_latency");
@@ -131,6 +138,7 @@ int main(int argc, char* argv[])
 
 
   // configuration of memory banks
+  int numaNodeChanged=0;
   for (auto kName : ConfigFileBrowser (&cfg,"bank-")) {
      // skip disabled
     int enabled=1;
@@ -161,7 +169,24 @@ int main(int argc, char* argv[])
     }
     if (cfgType.length()==0) {continue;}
 
+    // numa node
+    int cfgNumaNode=-1;
+    cfg.getOptionalValue<int>(kName + ".numaNode",cfgNumaNode);
+
     // instanciate new memory pool
+    if (cfgNumaNode>=0) {
+      #ifdef WITH_NUMA
+      struct bitmask *nodemask;
+      nodemask=numa_allocate_nodemask();
+      if (nodemask==NULL) {return -1;}
+      numa_bitmask_clearall(nodemask);
+      numa_bitmask_setbit(nodemask,cfgNumaNode);
+      numa_set_membind(nodemask);
+      numa_free_nodemask(nodemask);
+      theLog.log("Enforcing memory allocated on NUMA node %d",cfgNumaNode);
+      numaNodeChanged=1;
+      #endif
+    }
     theLog.log("Creating memory bank %s: type %s size %lld",kName.c_str(),cfgType.c_str(),mSize);
     std::shared_ptr<MemoryBank> b=nullptr;
     try {
@@ -180,6 +205,18 @@ int main(int argc, char* argv[])
     theLog.log("Bank %s added",kName.c_str());
   }
 
+  // releasing memory bind policy
+  if (numaNodeChanged){
+    #ifdef WITH_NUMA
+    struct bitmask *nodemask;
+    nodemask=numa_get_mems_allowed();
+    numa_set_membind(nodemask);
+    // is this needed? not specified in doc...
+    //numa_free_nodemask(nodemask);
+    theLog.log("Releasing memory NUMA node enforcment");
+    #endif
+  }
+  
 
   // configuration of data consumers
   std::vector<std::unique_ptr<Consumer>> dataConsumers;

src/testROC.cxx

@@ -18,7 +18,11 @@ InfoLogger theLog;
 #include <time.h>
 
 #include <sys/mman.h>
+
+#ifdef WITH_NUMA
 #include <numa.h>
+#endif
+
 #include <sched.h>
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -220,13 +224,17 @@ int main(int argc, char**argv) {
 
   // bind alloc/exec on numa node
   if (numaNodeId>=0) {
+    #ifdef WITH_NUMA
     struct bitmask *nodemask;
     nodemask=numa_allocate_nodemask();
     if (nodemask==NULL) {return -1;}
     numa_bitmask_clearall(nodemask);
     numa_bitmask_setbit(nodemask,numaNodeId);
     numa_bind(nodemask);
     printf("Locked to numa node %d\n",numaNodeId);
+    #else
+    printf("Can not set numaNode ... program compiled without NUMA support\n");
+    #endif
   }