Adds comptime option to link jemalloc instead of tcmalloc

.bazelrc

@@ -208,7 +208,6 @@ build:compile-time-options --config=boringssl-fips
 build:compile-time-options --define=log_debug_assert_in_release=enabled
 build:compile-time-options --define=path_normalization_by_default=true
 build:compile-time-options --define=deprecated_features=disabled
-build:compile-time-options --define=tcmalloc=gperftools
 build:compile-time-options --define=uhv=enabled
 # gRPC has a lot of deprecated-enum-enum-conversion warnings with C++20
 build:compile-time-options --copt=-Wno-error=deprecated-enum-enum-conversion

bazel/BUILD

@@ -227,6 +227,16 @@ config_setting(
     values = {"define": "tcmalloc=gperftools"},
 )
 
+bool_flag(
+    name = "jemalloc",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "jemalloc_enabled",
+    flag_values = {":jemalloc": "True"},
+)
+
 config_setting(
     name = "disable_signal_trace",
     values = {"define": "signal_trace=disabled"},

bazel/deps.yaml

@@ -910,6 +910,18 @@ gperftools:
   license: "BSD-3-Clause"
   license_url: "https://github.com/gperftools/gperftools/blob/gperftools-{version}/COPYING"
 
+jemalloc:
+  project_name: "jemalloc"
+  project_desc: "General-purpose scalable concurrent memory allocator"
+  project_url: "https://github.com/jemalloc/jemalloc"
+  release_date: "2022-05-06"
+  use_category:
+  - dataplane_core
+  - controlplane
+  cpe: "N/A"
+  license: "BSD-2-Clause"
+  license_url: "https://github.com/jemalloc/jemalloc/blob/{version}/COPYING"
+
 grpc_httpjson_transcoding:
   project_name: "grpc-httpjson-transcoding"
   project_desc: "Library that supports transcoding so that HTTP/JSON can be converted to gRPC"

bazel/envoy_internal.bzl

@@ -92,6 +92,7 @@ def envoy_copts(repository, test = False):
                _repo("//bazel:disable_tcmalloc"): ["-DABSL_MALLOC_HOOK_MMAP_DISABLE"],
                _repo("//bazel:debug_tcmalloc"): ["-DENVOY_MEMORY_DEBUG_ENABLED=1", "-DGPERFTOOLS_TCMALLOC"],
                _repo("//bazel:gperftools_tcmalloc"): ["-DGPERFTOOLS_TCMALLOC"],
+               _repo("//bazel:jemalloc_enabled"): ["-DJEMALLOC"],
                (
                    "@platforms//cpu:x86_64",
                    "@platforms//cpu:aarch64",
@@ -197,6 +198,7 @@ def tcmalloc_external_dep(repository):
             _repo("//bazel:debug_tcmalloc"),
             _repo("//bazel:gperftools_tcmalloc"),
         ): _repo("//bazel/external:gperftools"),
+        (_repo("//bazel:jemalloc_enabled"),): _repo("//bazel/foreign_cc:jemalloc"),
         "//conditions:default": _repo("//bazel:tcmalloc_lib"),
     })
 

bazel/envoy_library.bzl

@@ -30,6 +30,7 @@ def tcmalloc_external_deps(repository):
             _repo("//bazel:debug_tcmalloc"),
             _repo("//bazel:gperftools_tcmalloc"),
         ): [_repo("//bazel/external:gperftools")],
+        (_repo("//bazel:jemalloc_enabled"),): [_repo("//bazel/foreign_cc:jemalloc")],
         "//conditions:default": [_repo("//bazel:tcmalloc_all_libs")],
     })
 

bazel/foreign_cc/BUILD

@@ -446,6 +446,22 @@ envoy_cc_library(
     }),
 )
 
+configure_make(
+    name = "jemalloc",
+    configure_options = [
+        "--with-jemalloc-prefix=",
+        "--disable-shared",
+        "--enable-static",
+        "--disable-doc",
+        "--disable-cxx",
+        "--disable-prof",
+    ],
+    lib_source = "@jemalloc//:all",
+    out_static_libs = ["libjemalloc.a"],
+    tags = ["skip_on_windows"],
+    visibility = ["//visibility:public"],
+)
+
 cc_library(
     name = "libcxx_msan_wrapper",
     visibility = ["//visibility:public"],

bazel/repositories.bzl

@@ -182,6 +182,7 @@ def envoy_dependencies(skip_targets = []):
     _libsxg()
     _tcmalloc()
     _gperftools()
+    _jemalloc()
     _com_github_grpc_grpc()
     _rules_proto_grpc()
     _icu()
@@ -904,6 +905,12 @@ def _gperftools():
         name = "gperftools",
     )
 
+def _jemalloc():
+    external_http_archive(
+        name = "jemalloc",
+        build_file_content = BUILD_ALL_CONTENT,
+    )
+
 def _wamr():
     external_http_archive(
         name = "wamr",

bazel/repository_locations.bzl

@@ -176,6 +176,12 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         strip_prefix = "gperftools-{version}",
         urls = ["https://github.com/gperftools/gperftools/releases/download/gperftools-{version}/gperftools-{version}.tar.gz"],
     ),
+    jemalloc = dict(
+        version = "5.3.0",
+        sha256 = "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa",
+        strip_prefix = "jemalloc-{version}",
+        urls = ["https://github.com/jemalloc/jemalloc/releases/download/{version}/jemalloc-{version}.tar.bz2"],
+    ),
     com_github_grpc_grpc = dict(
         version = "1.76.0",
         sha256 = "0af37b800953130b47c075b56683ee60bdc3eda3c37fc6004193f5b569758204",

ci/do_ci.sh

@@ -409,26 +409,29 @@ case $CI_TARGET in
             bazel_with_collection \
                 test "${BAZEL_BUILD_OPTIONS[@]}" \
                 --config=compile-time-options \
+                --define tcmalloc=gperftools \
                 --define wasm=wamr \
                 -c fastbuild \
                 "${TEST_TARGETS[@]}"
         fi
         if [[ -z "$ENVOY_SKIP_CTO_WASMTIME" ]]; then
             exit 0
         fi
-        echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}"
+        echo "Building and testing with wasm=wasmtime and jemalloc: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}"
         bazel_with_collection \
             test "${BAZEL_BUILD_OPTIONS[@]}" \
             --config=compile-time-options \
             --define wasm=wasmtime \
             --define admin_functionality=disabled \
+            --@envoy//bazel:jemalloc=True \
             -c fastbuild \
             "${TEST_TARGETS[@]}"
         # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only
         # these tests under "-c opt" to save time in CI.
         bazel_with_collection \
             test "${BAZEL_BUILD_OPTIONS[@]}" \
             --config=compile-time-options \
+            --define tcmalloc=gperftools \
             --define wasm=wasmtime \
             -c opt \
             @envoy//test/common/common:assert_test \
@@ -437,6 +440,7 @@ case $CI_TARGET in
         bazel_with_collection \
             test "${BAZEL_BUILD_OPTIONS[@]}" \
             --config=compile-time-options \
+            --define tcmalloc=gperftools \
             --define wasm=wasmtime \
             -c opt \
             @envoy//test/common/common:assert_test \
@@ -445,6 +449,7 @@ case $CI_TARGET in
         echo "Building binary with wasm=wasmtime... and logging disabled"
         bazel build "${BAZEL_BUILD_OPTIONS[@]}" \
             --config=compile-time-options \
+            --define tcmalloc=gperftools \
             --define wasm=wasmtime \
             --define enable_logging=disabled \
             -c fastbuild \

source/common/memory/stats.cc

@@ -10,6 +10,8 @@
 #include "tcmalloc/malloc_extension.h"
 #elif defined(GPERFTOOLS_TCMALLOC)
 #include "gperftools/malloc_extension.h"
+#elif defined(JEMALLOC)
+#include <jemalloc/jemalloc.h>
 #endif
 
 namespace Envoy {
@@ -27,6 +29,17 @@ void setMaxUnfreedMemoryBytes(uint64_t value) {
   max_unfreed_memory_bytes.store(value, std::memory_order_relaxed);
 }
 
+#if defined(JEMALLOC)
+namespace {
+// Refresh jemalloc's epoch so that subsequently-read stats reflect current state.
+void refreshJemallocEpoch() {
+  uint64_t epoch = 1;
+  size_t sz = sizeof(epoch);
+  mallctl("epoch", &epoch, &sz, &epoch, sz);
+}
+} // namespace
+#endif
+
 uint64_t Stats::totalCurrentlyAllocated() {
 #if defined(TCMALLOC)
   return tcmalloc::MallocExtension::GetNumericProperty("generic.current_allocated_bytes")
@@ -35,6 +48,12 @@ uint64_t Stats::totalCurrentlyAllocated() {
   size_t value = 0;
   MallocExtension::instance()->GetNumericProperty("generic.current_allocated_bytes", &value);
   return value;
+#elif defined(JEMALLOC)
+  refreshJemallocEpoch();
+  size_t allocated = 0;
+  size_t sz = sizeof(allocated);
+  mallctl("stats.allocated", &allocated, &sz, nullptr, 0);
+  return allocated;
 #else
   return 0;
 #endif
@@ -51,6 +70,12 @@ uint64_t Stats::totalCurrentlyReserved() {
   size_t value = 0;
   MallocExtension::instance()->GetNumericProperty("generic.heap_size", &value);
   return value;
+#elif defined(JEMALLOC)
+  refreshJemallocEpoch();
+  size_t mapped = 0;
+  size_t sz = sizeof(mapped);
+  mallctl("stats.mapped", &mapped, &sz, nullptr, 0);
+  return mapped;
 #else
   return 0;
 #endif
@@ -65,6 +90,9 @@ uint64_t Stats::totalThreadCacheBytes() {
   MallocExtension::instance()->GetNumericProperty("tcmalloc.current_total_thread_cache_bytes",
                                                   &value);
   return value;
+#elif defined(JEMALLOC)
+  // jemalloc uses per-arena caches rather than per-thread caches; no direct equivalent.
+  return 0;
 #else
   return 0;
 #endif
@@ -77,6 +105,13 @@ uint64_t Stats::totalPageHeapFree() {
   size_t value = 0;
   MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes", &value);
   return value;
+#elif defined(JEMALLOC)
+  refreshJemallocEpoch();
+  size_t active = 0, allocated = 0;
+  size_t sz = sizeof(size_t);
+  mallctl("stats.active", &active, &sz, nullptr, 0);
+  mallctl("stats.allocated", &allocated, &sz, nullptr, 0);
+  return active > allocated ? active - allocated : 0;
 #else
   return 0;
 #endif
@@ -90,6 +125,12 @@ uint64_t Stats::totalPageHeapUnmapped() {
   size_t value = 0;
   MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes", &value);
   return value;
+#elif defined(JEMALLOC)
+  refreshJemallocEpoch();
+  size_t retained = 0;
+  size_t sz = sizeof(retained);
+  mallctl("stats.retained", &retained, &sz, nullptr, 0);
+  return retained;
 #else
   return 0;
 #endif
@@ -102,6 +143,12 @@ uint64_t Stats::totalPhysicalBytes() {
   size_t value = 0;
   MallocExtension::instance()->GetNumericProperty("generic.total_physical_bytes", &value);
   return value;
+#elif defined(JEMALLOC)
+  refreshJemallocEpoch();
+  size_t resident = 0;
+  size_t sz = sizeof(resident);
+  mallctl("stats.resident", &resident, &sz, nullptr, 0);
+  return resident;
 #else
   return 0;
 #endif
@@ -115,6 +162,12 @@ void Stats::dumpStatsToLog() {
   auto buffer = std::make_unique<char[]>(buffer_size);
   MallocExtension::instance()->GetStats(buffer.get(), buffer_size);
   ENVOY_LOG_MISC(debug, "TCMalloc stats:\n{}", buffer.get());
+#elif defined(JEMALLOC)
+  std::string output;
+  malloc_stats_print(
+      [](void* opaque, const char* msg) { reinterpret_cast<std::string*>(opaque)->append(msg); },
+      &output, nullptr);
+  ENVOY_LOG_MISC(debug, "jemalloc stats:\n{}", output);
 #else
   return;
 #endif
@@ -129,6 +182,12 @@ absl::optional<std::string> Stats::dumpStats() {
   MallocExtension::instance()->GetStats(buffer.data(), buffer_size);
   buffer.resize(strlen(buffer.c_str()));
   return buffer;
+#elif defined(JEMALLOC)
+  std::string output;
+  malloc_stats_print(
+      [](void* opaque, const char* msg) { reinterpret_cast<std::string*>(opaque)->append(msg); },
+      &output, nullptr);
+  return output;
 #else
   return absl::nullopt;
 #endif

source/common/memory/utils.cc

@@ -7,6 +7,10 @@
 #include "tcmalloc/malloc_extension.h"
 #elif defined(GPERFTOOLS_TCMALLOC)
 #include "gperftools/malloc_extension.h"
+#elif defined(JEMALLOC)
+#include <jemalloc/jemalloc.h>
+
+#include <cstdio>
 #endif
 
 namespace Envoy {
@@ -19,6 +23,13 @@ void Utils::releaseFreeMemory(uint64_t max_unfreed_bytes) {
 #elif defined(GPERFTOOLS_TCMALLOC)
   UNREFERENCED_PARAMETER(max_unfreed_bytes);
   MallocExtension::instance()->ReleaseFreeMemory();
+#elif defined(JEMALLOC)
+  UNREFERENCED_PARAMETER(max_unfreed_bytes);
+  // Purge all arenas to release dirty pages back to the OS.
+  // `MALLCTL_ARENAS_ALL` is jemalloc's pseudo-index for addressing all arenas at once.
+  char purge_cmd[32];
+  snprintf(purge_cmd, sizeof(purge_cmd), "arena.%u.purge", MALLCTL_ARENAS_ALL);
+  mallctl(purge_cmd, nullptr, nullptr, nullptr, 0);
 #else
   UNREFERENCED_PARAMETER(max_unfreed_bytes);
 #endif
@@ -31,7 +42,7 @@ void Utils::releaseFreeMemory(uint64_t max_unfreed_bytes) {
   Ref: https://github.com/envoyproxy/envoy/pull/9471#discussion_r363825985
 */
 void Utils::tryShrinkHeap() {
-#if defined(TCMALLOC) || defined(GPERFTOOLS_TCMALLOC)
+#if defined(TCMALLOC) || defined(GPERFTOOLS_TCMALLOC) || defined(JEMALLOC)
   auto total_physical_bytes = Stats::totalPhysicalBytes();
   auto allocated_size_by_app = Stats::totalCurrentlyAllocated();
   const uint64_t threshold = maxUnfreedMemoryBytes();

tools/spelling/spelling_dictionary.txt

@@ -987,6 +987,7 @@ iteratively
 javascript
 jitter
 jittered
+jemalloc
 joinable
 js
 kafka