Hugepage support for iobuffer

doc/admin-guide/files/records.config.en.rst

@@ -4903,6 +4903,52 @@ Sockets
    platforms.  (Currently only Linux).  IO buffers are allocated with the MADV_DONTDUMP
    with madvise() on Linux platforms that support MADV_DONTDUMP.  Enabled by default.
 
+.. ts:cv:: CONFIG proxy.config.allocator.iobuf_chunk_sizes STRING
+
+   This configures the chunk sizes of each of the IO buffer allocators.  The chunk size is the number
+   of buffers allocated in a batch when the allocator's freelist is exhausted.  This must be specified as a
+   space separated list of up to 15 numbers.  If not specified or if any value specified is 0, the default
+   value will be used.
+
+   The list of numbers will specify the chunk sizes in the following order:
+
+   ``128 256 512 1k 2k 4k 8k 16k 32k 64k 128k 256k 512k 1M 2M``
+
+   The defaults for each allocator is:
+
+   ``128 128 128 128 128 128 32 32 32 32 32 32 32 32 32``
+
+   Even though this is specified, the actual chunk size might be modified based on the system's page size (or hugepage
+   size if enabled).
+
+   You might want to adjust these values to reduce the overall number of allocations that ATS needs to make based
+   on your configured RAM cache size.  On a running system, you can send SIGUSR1 to the ATS process to have it
+   log the allocator statistics and see how many of each buffer size have been allocated.
+
+.. ts:cv:: CONFIG proxy.config.allocator.iobuf_use_hugepages INT 0
+
+   This setting controls whether huge pages allocations are used to allocate io buffers.  If enabled, and hugepages are
+   not available, this will fall back to normal size pages. Using hugepages for iobuffer can sometimes improve performance
+   by utilizing more of the TLB and reducing TLB misses.
+
+   ===== ======================================================================
+   Value Description
+   ===== ======================================================================
+   ``0`` IO buffer allocation uses normal pages sizes
+   ``1`` IO buffer allocation uses huge pages
+   ===== ======================================================================
+
+.. ts:cv:: CONFIG proxy.config.cache.dir.enable_hugepages INT 0
+
+   This setting controls whether huge pages allocations are used to allocate memory for cache volume dir entries.
+
+   ===== ======================================================================
+   Value Description
+   ===== ======================================================================
+   ``0`` Use normal pages sizes
+   ``1`` Use huge pages
+   ===== ======================================================================
+
 .. ts:cv:: CONFIG proxy.config.ssl.misc.io.max_buffer_index INT 8
 
    Configures the max IOBuffer Block index used for various SSL Operations

include/tscore/Allocator.h

@@ -97,16 +97,18 @@ class FreelistAllocator
     @param chunk_size number of units to be allocated if free pool is empty.
     @param alignment of objects must be a power of 2.
   */
-  FreelistAllocator(const char *name, unsigned int element_size, unsigned int chunk_size = 128, unsigned int alignment = 8)
+  FreelistAllocator(const char *name, unsigned int element_size, unsigned int chunk_size = 128, unsigned int alignment = 8,
+                    bool use_hugepages = false)
   {
-    ink_freelist_init(&fl, name, element_size, chunk_size, alignment);
+    ink_freelist_init(&fl, name, element_size, chunk_size, alignment, use_hugepages);
   }
 
   /** Re-initialize the parameters of the allocator. */
   void
-  re_init(const char *name, unsigned int element_size, unsigned int chunk_size, unsigned int alignment, int advice)
+  re_init(const char *name, unsigned int element_size, unsigned int chunk_size, unsigned int alignment, bool use_hugepages,
+          int advice)
   {
-    ink_freelist_madvise_init(&this->fl, name, element_size, chunk_size, alignment, advice);
+    ink_freelist_madvise_init(&this->fl, name, element_size, chunk_size, alignment, use_hugepages, advice);
   }
 
   // Dummies
@@ -186,14 +188,16 @@ class MallocAllocator
     @param chunk_size number of units to be allocated if free pool is empty.
     @param alignment of objects must be a power of 2.
   */
-  MallocAllocator(const char *name, unsigned int element_size, unsigned int chunk_size = 128, unsigned int alignment = 8)
+  MallocAllocator(const char *name, unsigned int element_size, unsigned int chunk_size = 128, unsigned int alignment = 8,
+                  bool use_hugepages = false)
     : element_size(element_size), alignment(alignment), advice(0)
   {
   }
 
   /** Re-initialize the parameters of the allocator. */
   void
-  re_init(const char *name, unsigned int element_size, unsigned int chunk_size, unsigned int alignment, int advice)
+  re_init(const char *name, unsigned int element_size, unsigned int chunk_size, unsigned int alignment, bool use_hugepages,
+          int advice)
   {
     this->element_size = element_size;
     this->alignment    = alignment;
@@ -227,7 +231,7 @@ using Allocator = FreelistAllocator;
   Allocator for Class objects.
 
 */
-template <class C, bool Destruct_on_free_ = false> class ClassAllocator : public Allocator
+template <class C, bool Destruct_on_free_ = false, typename BaseAllocator = Allocator> class ClassAllocator : public BaseAllocator
 {
 public:
   using Value_type                   = C;
@@ -238,7 +242,7 @@ template <class C, bool Destruct_on_free_ = false> class ClassAllocator : public
   C *
   alloc(Args &&... args)
   {
-    void *ptr = alloc_void();
+    void *ptr = this->alloc_void();
 
     ::new (ptr) C(std::forward<Args>(args)...);
     return reinterpret_cast<C *>(ptr);
@@ -254,7 +258,7 @@ template <class C, bool Destruct_on_free_ = false> class ClassAllocator : public
   {
     destroy_if_enabled(ptr);
 
-    free_void(ptr);
+    this->free_void(ptr);
   }
 
   /**
@@ -265,7 +269,7 @@ template <class C, bool Destruct_on_free_ = false> class ClassAllocator : public
     @param alignment of objects must be a power of 2.
   */
   ClassAllocator(const char *name, unsigned int chunk_size = 128, unsigned int alignment = 16)
-    : Allocator(name, static_cast<unsigned int>(RND16(sizeof(C))), chunk_size, static_cast<unsigned int>(RND16(alignment)))
+    : BaseAllocator(name, static_cast<unsigned int>(RND16(sizeof(C))), chunk_size, static_cast<unsigned int>(RND16(alignment)))
   {
   }
 

include/tscore/ink_queue.h

@@ -191,6 +191,8 @@ struct _InkFreeList {
   const char *name;
   uint32_t type_size, chunk_size, used, allocated, alignment;
   uint32_t allocated_base, used_base;
+  uint32_t hugepages_failure;
+  bool use_hugepages;
   int advice;
 };
 
@@ -204,11 +206,13 @@ void ink_freelist_init_ops(int nofl_class, int nofl_proxy);
 /*
  * alignment must be a power of 2
  */
-InkFreeList *ink_freelist_create(const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment);
+InkFreeList *ink_freelist_create(const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment,
+                                 bool use_hugepages = false);
 
-void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment);
+void ink_freelist_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment,
+                       bool use_hugepages);
 void ink_freelist_madvise_init(InkFreeList **fl, const char *name, uint32_t type_size, uint32_t chunk_size, uint32_t alignment,
-                               int advice);
+                               bool use_hugepages, int advice);
 void *ink_freelist_new(InkFreeList *f);
 void ink_freelist_free(InkFreeList *f, void *item);
 void ink_freelist_free_bulk(InkFreeList *f, void *head, void *tail, size_t num_item);

iocore/cache/Cache.cc

@@ -83,6 +83,7 @@ int cache_config_read_while_writer             = 0;
 int cache_config_mutex_retry_delay             = 2;
 int cache_read_while_writer_retry_delay        = 50;
 int cache_config_read_while_writer_max_retries = 10;
+int cache_config_dir_enable_hugepages          = 0;
 
 // Globals
 
@@ -1253,7 +1254,7 @@ Vol::init(char *s, off_t blocks, off_t dir_skip, bool clear)
         (long long)this->len, (double)dirlen() / (double)this->len * 100.0);
 
   raw_dir = nullptr;
-  if (ats_hugepage_enabled()) {
+  if (ats_hugepage_enabled() && cache_config_dir_enable_hugepages) {
     raw_dir = static_cast<char *>(ats_alloc_hugepage(this->dirlen()));
   }
   if (raw_dir == nullptr) {
@@ -3153,6 +3154,9 @@ ink_cache_init(ts::ModuleVersion v)
   REC_EstablishStaticConfigInt32(cache_config_dir_sync_frequency, "proxy.config.cache.dir.sync_frequency");
   Debug("cache_init", "proxy.config.cache.dir.sync_frequency = %d", cache_config_dir_sync_frequency);
 
+  REC_EstablishStaticConfigInt32(cache_config_dir_enable_hugepages, "proxy.config.cache.dir.enable_hugepages");
+  Debug("cache_init", "proxy.config.cache.dir.enable_hugepages = %d", cache_config_dir_enable_hugepages);
+
   REC_EstablishStaticConfigInt32(cache_config_select_alternate, "proxy.config.cache.select_alternate");
   Debug("cache_init", "proxy.config.cache.select_alternate = %d", cache_config_select_alternate);
 

iocore/cache/CacheDir.cc

@@ -994,7 +994,7 @@ sync_cache_dir_on_shutdown()
         buf = nullptr;
       }
       buflen = dirlen;
-      if (ats_hugepage_enabled()) {
+      if (ats_hugepage_enabled() && cache_config_dir_enable_hugepages) {
         buf      = static_cast<char *>(ats_alloc_hugepage(buflen));
         buf_huge = true;
       }
@@ -1126,7 +1126,7 @@ CacheSync::mainEvent(int event, Event *e)
           buf = nullptr;
         }
         buflen = dirlen;
-        if (ats_hugepage_enabled()) {
+        if (ats_hugepage_enabled() && cache_config_dir_enable_hugepages) {
           buf      = static_cast<char *>(ats_alloc_hugepage(buflen));
           buf_huge = true;
         }

iocore/cache/P_CacheInternal.h

@@ -209,6 +209,7 @@ extern RecRawStatBlock *cache_rsb;
 
 // Configuration
 extern int cache_config_dir_sync_frequency;
+extern int cache_config_dir_enable_hugepages;
 extern int cache_config_http_max_alts;
 extern int cache_config_log_alternate_eviction;
 extern int cache_config_permit_pinning;

iocore/eventsystem/EventSystem.cc

@@ -29,6 +29,7 @@
 ****************************************************************************/
 
 #include "P_EventSystem.h"
+#include "tscore/hugepages.h"
 
 void
 ink_event_system_init(ts::ModuleVersion v)
@@ -42,6 +43,27 @@ ink_event_system_init(ts::ModuleVersion v)
 
   REC_EstablishStaticConfigInt32(thread_freelist_low_watermark, "proxy.config.allocator.thread_freelist_low_watermark");
 
+  int chunk_sizes[DEFAULT_BUFFER_SIZES] = {0};
+  char *chunk_sizes_string              = REC_ConfigReadString("proxy.config.allocator.iobuf_chunk_sizes");
+  if (chunk_sizes_string != nullptr) {
+    ts::TextView src(chunk_sizes_string, ts::TextView::npos);
+    int n = 0;
+    while (n < DEFAULT_BUFFER_SIZES && !src.empty()) {
+      ts::TextView token{src.take_prefix_at(' ')};
+      auto x = ts::svto_radix<10>(token);
+      if (token.empty() && x <= std::numeric_limits<int>::max()) {
+        chunk_sizes[n++] = x;
+      } else {
+        break;
+      }
+    }
+    ats_free(chunk_sizes_string);
+  }
+
+  int hugepage_config = REC_ConfigReadInteger("proxy.config.allocator.iobuf_use_hugepages");
+
+  bool use_hugepages = ats_hugepage_enabled() && hugepage_config == 1;
+
 #ifdef MADV_DONTDUMP // This should only exist on Linux 3.4 and higher.
   RecBool dont_dump_enabled = true;
   RecGetRecordBool("proxy.config.allocator.dontdump_iobuffers", &dont_dump_enabled, false);
@@ -51,5 +73,5 @@ ink_event_system_init(ts::ModuleVersion v)
   }
 #endif
 
-  init_buffer_allocators(iobuffer_advice);
+  init_buffer_allocators(iobuffer_advice, chunk_sizes, use_hugepages);
 }

iocore/eventsystem/IOBuffer.cc

@@ -31,7 +31,7 @@
 //
 // General Buffer Allocator
 //
-Allocator ioBufAllocator[DEFAULT_BUFFER_SIZES];
+FreelistAllocator ioBufAllocator[DEFAULT_BUFFER_SIZES];
 ClassAllocator<MIOBuffer> ioAllocator("ioAllocator", DEFAULT_BUFFER_NUMBER);
 ClassAllocator<IOBufferData> ioDataAllocator("ioDataAllocator", DEFAULT_BUFFER_NUMBER);
 ClassAllocator<IOBufferBlock> ioBlockAllocator("ioBlockAllocator", DEFAULT_BUFFER_NUMBER);
@@ -43,22 +43,36 @@ int64_t max_iobuffer_size           = DEFAULT_BUFFER_SIZES - 1;
 // Initialization
 //
 void
-init_buffer_allocators(int iobuffer_advice)
+init_buffer_allocators(int iobuffer_advice, int chunk_sizes[DEFAULT_BUFFER_SIZES], bool use_hugepages)
 {
   for (int i = 0; i < DEFAULT_BUFFER_SIZES; i++) {
     int64_t s = DEFAULT_BUFFER_BASE_SIZE * ((static_cast<int64_t>(1)) << i);
     int64_t a = DEFAULT_BUFFER_ALIGNMENT;
-    int n     = i <= default_large_iobuffer_size ? DEFAULT_BUFFER_NUMBER : DEFAULT_HUGE_BUFFER_NUMBER;
+    int n     = chunk_sizes[i];
+    if (n == 0) {
+      n = i <= default_large_iobuffer_size ? DEFAULT_BUFFER_NUMBER : DEFAULT_HUGE_BUFFER_NUMBER;
+    }
     if (s < a) {
       a = s;
     }
 
     auto name = new char[64];
-    snprintf(name, 64, "ioBufAllocator[%d]", i);
-    ioBufAllocator[i].re_init(name, s, n, a, iobuffer_advice);
+    if (use_hugepages) {
+      snprintf(name, 64, "ioBufAllocatorHP[%d]", i);
+    } else {
+      snprintf(name, 64, "ioBufAllocator[%d]", i);
+    }
+    ioBufAllocator[i].re_init(name, s, n, a, use_hugepages, iobuffer_advice);
   }
 }
 
+void
+init_buffer_allocators(int iobuffer_advice)
+{
+  int chunk_sizes[DEFAULT_BUFFER_SIZES] = {0};
+  init_buffer_allocators(iobuffer_advice, chunk_sizes, false);
+}
+
 //
 // MIOBuffer
 //

iocore/eventsystem/I_IOBuffer.h

@@ -110,8 +110,9 @@ enum AllocType {
 #define BUFFER_SIZE_FOR_CONSTANT(_size) (_size - DEFAULT_BUFFER_SIZES)
 #define BUFFER_SIZE_INDEX_FOR_CONSTANT_SIZE(_size) (_size + DEFAULT_BUFFER_SIZES)
 
-extern Allocator ioBufAllocator[DEFAULT_BUFFER_SIZES];
+extern FreelistAllocator ioBufAllocator[DEFAULT_BUFFER_SIZES];
 
+void init_buffer_allocators(int iobuffer_advice, int chunk_sizes[DEFAULT_BUFFER_SIZES], bool use_hugepages);
 void init_buffer_allocators(int iobuffer_advice);
 
 /**

mgmt/RecordsConfig.cc

@@ -854,6 +854,8 @@ static const RecordElement RecordsConfig[] =
   //  # how often should the directory be synced (seconds)
   {RECT_CONFIG, "proxy.config.cache.dir.sync_frequency", RECD_INT, "60", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.cache.dir.enable_hugepages", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.cache.hostdb.disable_reverse_lookup", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.cache.select_alternate", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
@@ -1546,6 +1548,10 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.allocator.dontdump_iobuffers", RECD_INT, "1", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.allocator.iobuf_chunk_sizes", RECD_STRING, nullptr, RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.allocator.iobuf_use_hugepages", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
 
   // Controls for TLS ASYN_JOBS and engine loading
   {RECT_CONFIG, "proxy.config.ssl.async.handshake.enabled", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_NULL, "[0-1]", RECA_NULL},

proxy/http/remap/Makefile.am

@@ -72,6 +72,7 @@ COMMON_PLUGINDSO_LDADDS = \
 	$(top_builddir)/src/records/librecords_p.a \
 	$(top_builddir)/iocore/eventsystem/libinkevent.a \
 	$(top_builddir)/src/tscore/libtscore.la \
+	$(top_builddir)/src/tscpp/util/libtscpputil.la \
 	$(top_builddir)/mgmt/libmgmt_p.la \
 	$(top_builddir)/proxy/shared/libUglyLogStubs.a \
 	@HWLOC_LIBS@

src/traffic_server/traffic_server.cc

@@ -271,15 +271,15 @@ class SignalContinuation : public Continuation
     if (signal_received[SIGUSR1]) {
       signal_received[SIGUSR1] = false;
 
-      // TODO: TS-567 Integrate with debugging allocators "dump" features?
-      ink_freelists_dump(stderr);
-      ResourceTracker::dump(stderr);
-
 #if TS_HAS_JEMALLOC
       char buf[PATH_NAME_MAX] = "";
       RecGetRecordString("proxy.config.memory.malloc_stats_print_opts", buf, PATH_NAME_MAX);
       malloc_stats_print(nullptr, nullptr, buf);
 #endif
+
+      // TODO: TS-567 Integrate with debugging allocators "dump" features?
+      ink_freelists_dump(stderr);
+      ResourceTracker::dump(stderr);
     }
 
     if (signal_received[SIGUSR2]) {