diff --git a/.bazelrc b/.bazelrc index f199441858119..31a635decae8d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -208,7 +208,6 @@ build:compile-time-options --config=boringssl-fips build:compile-time-options --define=log_debug_assert_in_release=enabled build:compile-time-options --define=path_normalization_by_default=true build:compile-time-options --define=deprecated_features=disabled -build:compile-time-options --define=tcmalloc=gperftools build:compile-time-options --define=uhv=enabled # gRPC has a lot of deprecated-enum-enum-conversion warnings with C++20 build:compile-time-options --copt=-Wno-error=deprecated-enum-enum-conversion diff --git a/bazel/BUILD b/bazel/BUILD index 05ea3ea3ccfae..69ca106bca9e3 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -227,6 +227,16 @@ config_setting( values = {"define": "tcmalloc=gperftools"}, ) +bool_flag( + name = "jemalloc", + build_setting_default = False, +) + +config_setting( + name = "jemalloc_enabled", + flag_values = {":jemalloc": "True"}, +) + config_setting( name = "disable_signal_trace", values = {"define": "signal_trace=disabled"}, diff --git a/bazel/deps.yaml b/bazel/deps.yaml index 60e15c4832e60..6fe50279172df 100644 --- a/bazel/deps.yaml +++ b/bazel/deps.yaml @@ -910,6 +910,18 @@ gperftools: license: "BSD-3-Clause" license_url: "https://github.com/gperftools/gperftools/blob/gperftools-{version}/COPYING" +jemalloc: + project_name: "jemalloc" + project_desc: "General-purpose scalable concurrent memory allocator" + project_url: "https://github.com/jemalloc/jemalloc" + release_date: "2022-05-06" + use_category: + - dataplane_core + - controlplane + cpe: "N/A" + license: "BSD-2-Clause" + license_url: "https://github.com/jemalloc/jemalloc/blob/{version}/COPYING" + grpc_httpjson_transcoding: project_name: "grpc-httpjson-transcoding" project_desc: "Library that supports transcoding so that HTTP/JSON can be converted to gRPC" diff --git a/bazel/envoy_internal.bzl b/bazel/envoy_internal.bzl index 8282fb3e4f56f..3de0fc66e0eb2 100644 --- a/bazel/envoy_internal.bzl +++ b/bazel/envoy_internal.bzl @@ -92,6 +92,7 @@ def envoy_copts(repository, test = False): _repo("//bazel:disable_tcmalloc"): ["-DABSL_MALLOC_HOOK_MMAP_DISABLE"], _repo("//bazel:debug_tcmalloc"): ["-DENVOY_MEMORY_DEBUG_ENABLED=1", "-DGPERFTOOLS_TCMALLOC"], _repo("//bazel:gperftools_tcmalloc"): ["-DGPERFTOOLS_TCMALLOC"], + _repo("//bazel:jemalloc_enabled"): ["-DJEMALLOC"], ( "@platforms//cpu:x86_64", "@platforms//cpu:aarch64", @@ -197,6 +198,7 @@ def tcmalloc_external_dep(repository): _repo("//bazel:debug_tcmalloc"), _repo("//bazel:gperftools_tcmalloc"), ): _repo("//bazel/external:gperftools"), + (_repo("//bazel:jemalloc_enabled"),): _repo("//bazel/foreign_cc:jemalloc"), "//conditions:default": _repo("//bazel:tcmalloc_lib"), }) diff --git a/bazel/envoy_library.bzl b/bazel/envoy_library.bzl index 59f59e49e1d4c..1ad14043622b7 100644 --- a/bazel/envoy_library.bzl +++ b/bazel/envoy_library.bzl @@ -30,6 +30,7 @@ def tcmalloc_external_deps(repository): _repo("//bazel:debug_tcmalloc"), _repo("//bazel:gperftools_tcmalloc"), ): [_repo("//bazel/external:gperftools")], + (_repo("//bazel:jemalloc_enabled"),): [_repo("//bazel/foreign_cc:jemalloc")], "//conditions:default": [_repo("//bazel:tcmalloc_all_libs")], }) diff --git a/bazel/foreign_cc/BUILD b/bazel/foreign_cc/BUILD index 7cd494b368ad5..f6b7b202afbba 100644 --- a/bazel/foreign_cc/BUILD +++ b/bazel/foreign_cc/BUILD @@ -446,6 +446,22 @@ envoy_cc_library( }), ) +configure_make( + name = "jemalloc", + configure_options = [ + "--with-jemalloc-prefix=", + "--disable-shared", + "--enable-static", + "--disable-doc", + "--disable-cxx", + "--disable-prof", + ], + lib_source = "@jemalloc//:all", + out_static_libs = ["libjemalloc.a"], + tags = ["skip_on_windows"], + visibility = ["//visibility:public"], +) + cc_library( name = "libcxx_msan_wrapper", visibility = ["//visibility:public"], diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 267761ef27c28..46897855db0e8 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -182,6 +182,7 @@ def envoy_dependencies(skip_targets = []): _libsxg() _tcmalloc() _gperftools() + _jemalloc() _com_github_grpc_grpc() _rules_proto_grpc() _icu() @@ -904,6 +905,12 @@ def _gperftools(): name = "gperftools", ) +def _jemalloc(): + external_http_archive( + name = "jemalloc", + build_file_content = BUILD_ALL_CONTENT, + ) + def _wamr(): external_http_archive( name = "wamr", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 70edb8a8077a7..3406f8380c3f8 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -176,6 +176,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( strip_prefix = "gperftools-{version}", urls = ["https://github.com/gperftools/gperftools/releases/download/gperftools-{version}/gperftools-{version}.tar.gz"], ), + jemalloc = dict( + version = "5.3.0", + sha256 = "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa", + strip_prefix = "jemalloc-{version}", + urls = ["https://github.com/jemalloc/jemalloc/releases/download/{version}/jemalloc-{version}.tar.bz2"], + ), com_github_grpc_grpc = dict( version = "1.76.0", sha256 = "0af37b800953130b47c075b56683ee60bdc3eda3c37fc6004193f5b569758204", diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 4c5ffa6badc79..bccdcc6b966d2 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -409,6 +409,7 @@ case $CI_TARGET in bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ --config=compile-time-options \ + --define tcmalloc=gperftools \ --define wasm=wamr \ -c fastbuild \ "${TEST_TARGETS[@]}" @@ -416,12 +417,13 @@ case $CI_TARGET in if [[ -z "$ENVOY_SKIP_CTO_WASMTIME" ]]; then exit 0 fi - echo "Building and testing with wasm=wasmtime: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}" + echo "Building and testing with wasm=wasmtime and jemalloc: and admin_functionality and admin_html disabled ${TEST_TARGETS[*]}" bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ --config=compile-time-options \ --define wasm=wasmtime \ --define admin_functionality=disabled \ + --@envoy//bazel:jemalloc=True \ -c fastbuild \ "${TEST_TARGETS[@]}" # "--define log_debug_assert_in_release=enabled" must be tested with a release build, so run only @@ -429,6 +431,7 @@ case $CI_TARGET in bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ --config=compile-time-options \ + --define tcmalloc=gperftools \ --define wasm=wasmtime \ -c opt \ @envoy//test/common/common:assert_test \ @@ -437,6 +440,7 @@ case $CI_TARGET in bazel_with_collection \ test "${BAZEL_BUILD_OPTIONS[@]}" \ --config=compile-time-options \ + --define tcmalloc=gperftools \ --define wasm=wasmtime \ -c opt \ @envoy//test/common/common:assert_test \ @@ -445,6 +449,7 @@ case $CI_TARGET in echo "Building binary with wasm=wasmtime... and logging disabled" bazel build "${BAZEL_BUILD_OPTIONS[@]}" \ --config=compile-time-options \ + --define tcmalloc=gperftools \ --define wasm=wasmtime \ --define enable_logging=disabled \ -c fastbuild \ diff --git a/source/common/memory/stats.cc b/source/common/memory/stats.cc index 31a8ea673765e..37ebaa4dddfdc 100644 --- a/source/common/memory/stats.cc +++ b/source/common/memory/stats.cc @@ -10,6 +10,8 @@ #include "tcmalloc/malloc_extension.h" #elif defined(GPERFTOOLS_TCMALLOC) #include "gperftools/malloc_extension.h" +#elif defined(JEMALLOC) +#include #endif namespace Envoy { @@ -27,6 +29,17 @@ void setMaxUnfreedMemoryBytes(uint64_t value) { max_unfreed_memory_bytes.store(value, std::memory_order_relaxed); } +#if defined(JEMALLOC) +namespace { +// Refresh jemalloc's epoch so that subsequently-read stats reflect current state. +void refreshJemallocEpoch() { + uint64_t epoch = 1; + size_t sz = sizeof(epoch); + mallctl("epoch", &epoch, &sz, &epoch, sz); +} +} // namespace +#endif + uint64_t Stats::totalCurrentlyAllocated() { #if defined(TCMALLOC) return tcmalloc::MallocExtension::GetNumericProperty("generic.current_allocated_bytes") @@ -35,6 +48,12 @@ uint64_t Stats::totalCurrentlyAllocated() { size_t value = 0; MallocExtension::instance()->GetNumericProperty("generic.current_allocated_bytes", &value); return value; +#elif defined(JEMALLOC) + refreshJemallocEpoch(); + size_t allocated = 0; + size_t sz = sizeof(allocated); + mallctl("stats.allocated", &allocated, &sz, nullptr, 0); + return allocated; #else return 0; #endif @@ -51,6 +70,12 @@ uint64_t Stats::totalCurrentlyReserved() { size_t value = 0; MallocExtension::instance()->GetNumericProperty("generic.heap_size", &value); return value; +#elif defined(JEMALLOC) + refreshJemallocEpoch(); + size_t mapped = 0; + size_t sz = sizeof(mapped); + mallctl("stats.mapped", &mapped, &sz, nullptr, 0); + return mapped; #else return 0; #endif @@ -65,6 +90,9 @@ uint64_t Stats::totalThreadCacheBytes() { MallocExtension::instance()->GetNumericProperty("tcmalloc.current_total_thread_cache_bytes", &value); return value; +#elif defined(JEMALLOC) + // jemalloc uses per-arena caches rather than per-thread caches; no direct equivalent. + return 0; #else return 0; #endif @@ -77,6 +105,13 @@ uint64_t Stats::totalPageHeapFree() { size_t value = 0; MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes", &value); return value; +#elif defined(JEMALLOC) + refreshJemallocEpoch(); + size_t active = 0, allocated = 0; + size_t sz = sizeof(size_t); + mallctl("stats.active", &active, &sz, nullptr, 0); + mallctl("stats.allocated", &allocated, &sz, nullptr, 0); + return active > allocated ? active - allocated : 0; #else return 0; #endif @@ -90,6 +125,12 @@ uint64_t Stats::totalPageHeapUnmapped() { size_t value = 0; MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes", &value); return value; +#elif defined(JEMALLOC) + refreshJemallocEpoch(); + size_t retained = 0; + size_t sz = sizeof(retained); + mallctl("stats.retained", &retained, &sz, nullptr, 0); + return retained; #else return 0; #endif @@ -102,6 +143,12 @@ uint64_t Stats::totalPhysicalBytes() { size_t value = 0; MallocExtension::instance()->GetNumericProperty("generic.total_physical_bytes", &value); return value; +#elif defined(JEMALLOC) + refreshJemallocEpoch(); + size_t resident = 0; + size_t sz = sizeof(resident); + mallctl("stats.resident", &resident, &sz, nullptr, 0); + return resident; #else return 0; #endif @@ -115,6 +162,12 @@ void Stats::dumpStatsToLog() { auto buffer = std::make_unique(buffer_size); MallocExtension::instance()->GetStats(buffer.get(), buffer_size); ENVOY_LOG_MISC(debug, "TCMalloc stats:\n{}", buffer.get()); +#elif defined(JEMALLOC) + std::string output; + malloc_stats_print( + [](void* opaque, const char* msg) { reinterpret_cast(opaque)->append(msg); }, + &output, nullptr); + ENVOY_LOG_MISC(debug, "jemalloc stats:\n{}", output); #else return; #endif @@ -129,6 +182,12 @@ absl::optional Stats::dumpStats() { MallocExtension::instance()->GetStats(buffer.data(), buffer_size); buffer.resize(strlen(buffer.c_str())); return buffer; +#elif defined(JEMALLOC) + std::string output; + malloc_stats_print( + [](void* opaque, const char* msg) { reinterpret_cast(opaque)->append(msg); }, + &output, nullptr); + return output; #else return absl::nullopt; #endif diff --git a/source/common/memory/utils.cc b/source/common/memory/utils.cc index ebab8936af4e9..cf7f5db505c06 100644 --- a/source/common/memory/utils.cc +++ b/source/common/memory/utils.cc @@ -7,6 +7,10 @@ #include "tcmalloc/malloc_extension.h" #elif defined(GPERFTOOLS_TCMALLOC) #include "gperftools/malloc_extension.h" +#elif defined(JEMALLOC) +#include + +#include #endif namespace Envoy { @@ -19,6 +23,13 @@ void Utils::releaseFreeMemory(uint64_t max_unfreed_bytes) { #elif defined(GPERFTOOLS_TCMALLOC) UNREFERENCED_PARAMETER(max_unfreed_bytes); MallocExtension::instance()->ReleaseFreeMemory(); +#elif defined(JEMALLOC) + UNREFERENCED_PARAMETER(max_unfreed_bytes); + // Purge all arenas to release dirty pages back to the OS. + // `MALLCTL_ARENAS_ALL` is jemalloc's pseudo-index for addressing all arenas at once. + char purge_cmd[32]; + snprintf(purge_cmd, sizeof(purge_cmd), "arena.%u.purge", MALLCTL_ARENAS_ALL); + mallctl(purge_cmd, nullptr, nullptr, nullptr, 0); #else UNREFERENCED_PARAMETER(max_unfreed_bytes); #endif @@ -31,7 +42,7 @@ void Utils::releaseFreeMemory(uint64_t max_unfreed_bytes) { Ref: https://github.com/envoyproxy/envoy/pull/9471#discussion_r363825985 */ void Utils::tryShrinkHeap() { -#if defined(TCMALLOC) || defined(GPERFTOOLS_TCMALLOC) +#if defined(TCMALLOC) || defined(GPERFTOOLS_TCMALLOC) || defined(JEMALLOC) auto total_physical_bytes = Stats::totalPhysicalBytes(); auto allocated_size_by_app = Stats::totalCurrentlyAllocated(); const uint64_t threshold = maxUnfreedMemoryBytes(); diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 7a8b3bcae11c1..813a6ab2aae4a 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -987,6 +987,7 @@ iteratively javascript jitter jittered +jemalloc joinable js kafka