diff --git a/README.md b/README.md index 7455d5a3a..193a82068 100644 --- a/README.md +++ b/README.md @@ -402,6 +402,25 @@ Improved thread-local storage initialization to prevent race conditions: These architectural improvements focus on eliminating race conditions, improving performance in high-throughput scenarios, and providing better debugging capabilities for the native profiling engine. +### Remote Symbolication Support (2025) + +Added support for remote symbolication to enable offloading symbol resolution from the agent to backend services: + +- **Build-ID extraction**: Automatically extracts GNU build-id from ELF binaries on Linux +- **Raw addressing information**: Stores build-id and PC offset instead of resolved symbol names +- **Remote symbolication mode**: Enable with `remotesym=true` profiler argument +- **JFR integration**: Remote frames serialized with build-id and offset for backend resolution +- **Zero encoding overhead**: Uses dedicated frame type (FRAME_NATIVE_REMOTE) for efficient serialization + +**Benefits**: +- Reduces agent overhead by eliminating local symbol resolution +- Enables centralized symbol resolution with better caching +- Supports scenarios where debug symbols are not available locally + +**Key files**: `elfBuildId.h`, `elfBuildId.cpp`, `profiler.cpp`, `flightRecorder.cpp` + +For detailed documentation, see [doc/REMOTE_SYMBOLICATION.md](doc/REMOTE_SYMBOLICATION.md). + ## Contributing 1. Fork the repository 2. Create a feature branch diff --git a/ddprof-lib/src/main/cpp/arguments.cpp b/ddprof-lib/src/main/cpp/arguments.cpp index 6dd01d032..8a9cfad3e 100644 --- a/ddprof-lib/src/main/cpp/arguments.cpp +++ b/ddprof-lib/src/main/cpp/arguments.cpp @@ -88,7 +88,10 @@ static const Multiplier UNIVERSAL[] = { // samples // generations - track surviving generations // lightweight[=BOOL] - enable lightweight profiling - events without -// stacktraces (default: true) jfr - dump events in Java +// stacktraces (default: true) +// remotesym[=BOOL] - enable remote symbolication for native frames +// (stores build-id and PC offset instead of symbol names) +// jfr - dump events in Java // Flight Recorder format interval=N - sampling interval in ns // (default: 10'000'000, i.e. 10 ms) jstackdepth=N - maximum Java stack // depth (default: 2048) safemode=BITS - disable stack recovery @@ -339,18 +342,30 @@ Error Arguments::parse(const char *args) { _enable_method_cleanup = true; } - CASE("wallsampler") + CASE("remotesym") if (value != NULL) { switch (value[0]) { - case 'j': - _wallclock_sampler = JVMTI; + case 'y': // yes + case 't': // true + _remote_symbolication = true; break; - case 'a': default: - _wallclock_sampler = ASGCT; + _remote_symbolication = false; } } + CASE("wallsampler") + if (value != NULL) { + switch (value[0]) { + case 'j': + _wallclock_sampler = JVMTI; + break; + case 'a': + default: + _wallclock_sampler = ASGCT; + } + } + DEFAULT() if (_unknown_arg == NULL) _unknown_arg = arg; diff --git a/ddprof-lib/src/main/cpp/arguments.h b/ddprof-lib/src/main/cpp/arguments.h index babfd4336..3f2542705 100644 --- a/ddprof-lib/src/main/cpp/arguments.h +++ b/ddprof-lib/src/main/cpp/arguments.h @@ -188,6 +188,7 @@ class Arguments { std::vector _context_attributes; bool _lightweight; bool _enable_method_cleanup; + bool _remote_symbolication; // Enable remote symbolication for native frames Arguments(bool persistent = false) : _buf(NULL), @@ -221,7 +222,8 @@ class Arguments { _context_attributes({}), _wallclock_sampler(ASGCT), _lightweight(false), - _enable_method_cleanup(true) {} + _enable_method_cleanup(true), + _remote_symbolication(false) {} ~Arguments(); diff --git a/ddprof-lib/src/main/cpp/codeCache.cpp b/ddprof-lib/src/main/cpp/codeCache.cpp index 1479ceb6f..b9b4cd9c6 100644 --- a/ddprof-lib/src/main/cpp/codeCache.cpp +++ b/ddprof-lib/src/main/cpp/codeCache.cpp @@ -37,6 +37,11 @@ CodeCache::CodeCache(const char *name, short lib_index, _plt_size = 0; _debug_symbols = false; + // Initialize build-id fields + _build_id = nullptr; + _build_id_len = 0; + _load_bias = 0; + memset(_imports, 0, sizeof(_imports)); _imports_patchable = imports_patchable; @@ -54,10 +59,27 @@ CodeCache::CodeCache(const CodeCache &other) { _min_address = other._min_address; _max_address = other._max_address; _text_base = other._text_base; + _image_base = other._image_base; - _imports_patchable = other._imports_patchable; _plt_offset = other._plt_offset; _plt_size = other._plt_size; + _debug_symbols = other._debug_symbols; + + // Copy build-id information + _build_id_len = other._build_id_len; + if (other._build_id != nullptr && other._build_id_len > 0) { + size_t hex_str_len = strlen(other._build_id); + _build_id = static_cast(malloc(hex_str_len + 1)); + if (_build_id != nullptr) { + strcpy(_build_id, other._build_id); + } + } else { + _build_id = nullptr; + } + _load_bias = other._load_bias; + + memset(_imports, 0, sizeof(_imports)); + _imports_patchable = other._imports_patchable; _dwarf_table_length = other._dwarf_table_length; _dwarf_table = new FrameDesc[_dwarf_table_length]; @@ -77,17 +99,34 @@ CodeCache &CodeCache::operator=(const CodeCache &other) { delete _name; delete _dwarf_table; delete _blobs; + free(_build_id); // Free existing build-id _name = NativeFunc::create(other._name, -1); _lib_index = other._lib_index; _min_address = other._min_address; _max_address = other._max_address; _text_base = other._text_base; - - _imports_patchable = other._imports_patchable; + _image_base = other._image_base; _plt_offset = other._plt_offset; _plt_size = other._plt_size; + _debug_symbols = other._debug_symbols; + + // Copy build-id information + _build_id_len = other._build_id_len; + if (other._build_id != nullptr && other._build_id_len > 0) { + size_t hex_str_len = strlen(other._build_id); + _build_id = static_cast(malloc(hex_str_len + 1)); + if (_build_id != nullptr) { + strcpy(_build_id, other._build_id); + } + } else { + _build_id = nullptr; + } + _load_bias = other._load_bias; + + memset(_imports, 0, sizeof(_imports)); + _imports_patchable = other._imports_patchable; _dwarf_table_length = other._dwarf_table_length; _dwarf_table = new FrameDesc[_dwarf_table_length]; @@ -110,6 +149,7 @@ CodeCache::~CodeCache() { NativeFunc::destroy(_name); delete[] _blobs; delete _dwarf_table; + free(_build_id); // Free build-id memory } void CodeCache::expand() { @@ -387,3 +427,23 @@ FrameDesc CodeCache::findFrameDesc(const void *pc) { return FrameDesc::default_frame; } } + +void CodeCache::setBuildId(const char* build_id, size_t build_id_len) { + // Free existing build-id if any + free(_build_id); + _build_id = nullptr; + _build_id_len = 0; + + if (build_id != nullptr && build_id_len > 0) { + // build_id is a hex string, allocate based on actual string length + size_t hex_str_len = strlen(build_id); + _build_id = static_cast(malloc(hex_str_len + 1)); + + if (_build_id != nullptr) { + // Copy the hex string + strcpy(_build_id, build_id); + // Store the original byte length (not hex string length) + _build_id_len = build_id_len; + } + } +} diff --git a/ddprof-lib/src/main/cpp/codeCache.h b/ddprof-lib/src/main/cpp/codeCache.h index a1c804b82..8a224c7d2 100644 --- a/ddprof-lib/src/main/cpp/codeCache.h +++ b/ddprof-lib/src/main/cpp/codeCache.h @@ -116,6 +116,11 @@ class CodeCache { unsigned int _plt_offset; unsigned int _plt_size; + // Build-ID and load bias for remote symbolication + char *_build_id; // GNU build-id (hex string, null if not available) + size_t _build_id_len; // Build-id length in bytes (raw, not hex string length) + uintptr_t _load_bias; // Load bias (image_base - file_base address) + void **_imports[NUM_IMPORTS][NUM_IMPORT_TYPES]; bool _imports_patchable; bool _debug_symbols; @@ -169,6 +174,19 @@ class CodeCache { void setDebugSymbols(bool debug_symbols) { _debug_symbols = debug_symbols; } + // Build-ID and remote symbolication support + const char* buildId() const { return _build_id; } + size_t buildIdLen() const { return _build_id_len; } + bool hasBuildId() const { return _build_id != nullptr; } + uintptr_t loadBias() const { return _load_bias; } + short libIndex() const { return _lib_index; } + + // Sets the build-id (hex string) and stores the original byte length + // build_id: null-terminated hex string (e.g., "abc123..." for 40-char string) + // build_id_len: original byte length before hex conversion (e.g., 20 bytes) + void setBuildId(const char* build_id, size_t build_id_len); + void setLoadBias(uintptr_t load_bias) { _load_bias = load_bias; } + void add(const void *start, int length, const char *name, bool update_bounds = false); void updateBounds(const void *start, const void *end); @@ -225,7 +243,7 @@ class CodeCacheArray { memset(_libs, 0, MAX_NATIVE_LIBS * sizeof(CodeCache *)); } - CodeCache *operator[](int index) { return _libs[index]; } + CodeCache *operator[](int index) const { return __atomic_load_n(&_libs[index], __ATOMIC_ACQUIRE); } int count() const { return __atomic_load_n(&_count, __ATOMIC_RELAXED); } @@ -247,7 +265,7 @@ class CodeCacheArray { return lib; } - size_t memoryUsage() { + size_t memoryUsage() const { return __atomic_load_n(&_used_memory, __ATOMIC_RELAXED); } }; diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp index 6de788e3f..f166bddcb 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ b/ddprof-lib/src/main/cpp/flightRecorder.cpp @@ -114,6 +114,25 @@ void Lookup::fillNativeMethodInfo(MethodInfo *mi, const char *name, } } +void Lookup::fillRemoteFrameInfo(MethodInfo *mi, const RemoteFrameInfo *rfi) { + // Store build-id in the class name field + mi->_class = _classes->lookup(rfi->build_id); + + // Store PC offset in hex format in the signature field + char offset_hex[32]; + snprintf(offset_hex, sizeof(offset_hex), "0x%lx", rfi->pc_offset); + mi->_sig = _symbols.lookup(offset_hex); + + // Use same modifiers as regular native frames (0x100 = ACC_NATIVE for consistency) + mi->_modifiers = 0x100; + // Use FRAME_NATIVE_REMOTE type to indicate remote symbolication + mi->_type = FRAME_NATIVE_REMOTE; + mi->_line_number_table = nullptr; + + // Method name indicates need for remote symbolication + mi->_name = _symbols.lookup(""); +} + void Lookup::cutArguments(char *func) { char *p = strrchr(func, ')'); if (p == NULL) @@ -322,6 +341,9 @@ MethodInfo *Lookup::resolveMethod(ASGCT_CallFrame &frame) { const char *name = (const char *)method; fillNativeMethodInfo(mi, name, Profiler::instance()->getLibraryName(name)); + } else if (frame.bci == BCI_NATIVE_FRAME_REMOTE) { + const RemoteFrameInfo *rfi = (const RemoteFrameInfo *)method; + fillRemoteFrameInfo(mi, rfi); } else { fillJavaMethodInfo(mi, method, first_time); } @@ -1036,18 +1058,23 @@ void Recording::writeNativeLibraries(Buffer *buf) { if (_recorded_lib_count < 0) return; - Profiler *profiler = Profiler::instance(); - CodeCacheArray &native_libs = profiler->_native_libs; + Libraries *libraries = Libraries::instance(); + const CodeCacheArray &native_libs = libraries->native_libs(); int native_lib_count = native_libs.count(); for (int i = _recorded_lib_count; i < native_lib_count; i++) { + CodeCache* lib = native_libs[i]; + + // Emit jdk.NativeLibrary event with extended fields (buildId and loadBias) flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - MAX_STRING_LENGTH); int start = buf->skip(5); buf->putVar64(T_NATIVE_LIBRARY); buf->putVar64(_start_ticks); - buf->putUtf8(native_libs[i]->name()); - buf->putVar64((uintptr_t)native_libs[i]->minAddress()); - buf->putVar64((uintptr_t)native_libs[i]->maxAddress()); + buf->putUtf8(lib->name()); + buf->putVar64((uintptr_t)lib->minAddress()); + buf->putVar64((uintptr_t)lib->maxAddress()); + buf->putUtf8(lib->hasBuildId() ? lib->buildId() : ""); + buf->putVar64((uintptr_t)lib->loadBias()); buf->putVar32(start, buf->offset() - start); flushIfNeeded(buf); } diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h index 4ee2b09ad..ca670b42d 100644 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ b/ddprof-lib/src/main/cpp/flightRecorder.h @@ -276,6 +276,7 @@ class Lookup { private: void fillNativeMethodInfo(MethodInfo *mi, const char *name, const char *lib_name); + void fillRemoteFrameInfo(MethodInfo *mi, const RemoteFrameInfo *rfi); void cutArguments(char *func); void fillJavaMethodInfo(MethodInfo *mi, jmethodID method, bool first_time); bool has_prefix(const char *str, const char *prefix) const { diff --git a/ddprof-lib/src/main/cpp/frame.h b/ddprof-lib/src/main/cpp/frame.h index e53212782..46a60288a 100644 --- a/ddprof-lib/src/main/cpp/frame.h +++ b/ddprof-lib/src/main/cpp/frame.h @@ -9,6 +9,7 @@ enum FrameTypeId { FRAME_CPP = 4, FRAME_KERNEL = 5, FRAME_C1_COMPILED = 6, + FRAME_NATIVE_REMOTE = 7, // Native frame with remote symbolication (build-id + pc-offset) }; class FrameType { diff --git a/ddprof-lib/src/main/cpp/jfrMetadata.cpp b/ddprof-lib/src/main/cpp/jfrMetadata.cpp index cf81ae4d4..6991a8a12 100644 --- a/ddprof-lib/src/main/cpp/jfrMetadata.cpp +++ b/ddprof-lib/src/main/cpp/jfrMetadata.cpp @@ -323,7 +323,9 @@ void JfrMetadata::initialize( << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) << field("name", T_STRING, "Name") << field("baseAddress", T_LONG, "Base Address", F_ADDRESS) - << field("topAddress", T_LONG, "Top Address", F_ADDRESS)) + << field("topAddress", T_LONG, "Top Address", F_ADDRESS) + << field("buildId", T_STRING, "GNU Build ID") + << field("loadBias", T_LONG, "Load Bias", F_ADDRESS)) << (type("profiler.Log", T_LOG, "Log Message") << category("Profiler") diff --git a/ddprof-lib/src/main/cpp/libraries.cpp b/ddprof-lib/src/main/cpp/libraries.cpp index a9b62c509..a4e40ef8e 100644 --- a/ddprof-lib/src/main/cpp/libraries.cpp +++ b/ddprof-lib/src/main/cpp/libraries.cpp @@ -1,8 +1,10 @@ #include "codeCache.h" +#include "common.h" #include "libraries.h" #include "libraryPatcher.h" #include "log.h" #include "symbols.h" +#include "symbols_linux_dd.h" #include "vmEntry.h" #include "vmStructs.h" @@ -38,6 +40,49 @@ void Libraries::updateSymbols(bool kernel_symbols) { LibraryPatcher::patch_libraries(); } +void Libraries::updateBuildIds() { +#ifdef __linux__ + int lib_count = _native_libs.count(); + TEST_LOG("updateBuildIds: processing %d libraries", lib_count); + + for (int i = 0; i < lib_count; i++) { + CodeCache* lib = _native_libs.at(i); + if (lib == nullptr || lib->hasBuildId()) { + continue; // Skip null libraries or those that already have build-id + } + + const char* lib_name = lib->name(); + if (lib_name == nullptr) { + continue; + } + + TEST_LOG("updateBuildIds: extracting build-id for %s", lib_name); + // Extract build-id from library file + size_t build_id_len; + char* build_id = ddprof::SymbolsLinux::extractBuildId(lib_name, &build_id_len); + + if (build_id != nullptr) { + // Set build-id and calculate load bias + lib->setBuildId(build_id, build_id_len); + + // Calculate load bias: difference between runtime address and file base + // For now, use image_base as the load bias base + if (lib->imageBase() != nullptr) { + lib->setLoadBias((uintptr_t)lib->imageBase()); + } + + free(build_id); // setBuildId makes its own copy + + TEST_LOG("updateBuildIds: set build-id for %s: %s", lib_name, lib->buildId()); + Log::debug("Extracted build-id for %s: %s", lib_name, lib->buildId()); + } else { + TEST_LOG("updateBuildIds: NO build-id found for %s", lib_name); + } + } + TEST_LOG("updateBuildIds: completed"); +#endif // __linux__ +} + const void *Libraries::resolveSymbol(const char *name) { char mangled_name[256]; if (strstr(name, "::") != NULL) { diff --git a/ddprof-lib/src/main/cpp/libraries.h b/ddprof-lib/src/main/cpp/libraries.h index 4c4aa132a..2257e13f1 100644 --- a/ddprof-lib/src/main/cpp/libraries.h +++ b/ddprof-lib/src/main/cpp/libraries.h @@ -12,6 +12,7 @@ class Libraries { public: Libraries() : _native_libs(), _runtime_stubs("runtime stubs") {} void updateSymbols(bool kernel_symbols); + void updateBuildIds(); // Extract build-ids for all loaded libraries const void *resolveSymbol(const char *name); // In J9 the 'libjvm' functionality is spread across multiple libraries // This function will return the 'libjvm' on non-J9 VMs and the library with the given name on J9 VMs diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 5f83a14d2..2c50dc108 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -233,17 +233,18 @@ const void *Profiler::resolveSymbol(const char *name) { } size_t len = strlen(name); - int native_lib_count = _native_libs.count(); + const CodeCacheArray& native_libs = _libs->native_libs(); + int native_lib_count = native_libs.count(); if (len > 0 && name[len - 1] == '*') { for (int i = 0; i < native_lib_count; i++) { - const void *address = _native_libs[i]->findSymbolByPrefix(name, len - 1); + const void *address = native_libs[i]->findSymbolByPrefix(name, len - 1); if (address != NULL) { return address; } } } else { for (int i = 0; i < native_lib_count; i++) { - const void *address = _native_libs[i]->findSymbol(name); + const void *address = native_libs[i]->findSymbol(name); if (address != NULL) { return address; } @@ -256,8 +257,9 @@ const void *Profiler::resolveSymbol(const char *name) { // For BCI_NATIVE_FRAME, library index is encoded ahead of the symbol name const char *Profiler::getLibraryName(const char *native_symbol) { short lib_index = NativeFunc::libIndex(native_symbol); - if (lib_index >= 0 && lib_index < _native_libs.count()) { - const char *s = _native_libs[lib_index]->name(); + const CodeCacheArray& native_libs = _libs->native_libs(); + if (lib_index >= 0 && lib_index < native_libs.count()) { + const char *s = native_libs[lib_index]->name(); if (s != NULL) { const char *p = strrchr(s, '/'); return p != NULL ? p + 1 : s; @@ -290,7 +292,7 @@ bool Profiler::isAddressInCode(const void *pc, bool include_stubs) { int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames, int event_type, int tid, StackContext *java_ctx, - bool *truncated) { + bool *truncated, int lock_index) { if (_cstack == CSTACK_NO || (event_type == BCI_ALLOC || event_type == BCI_ALLOC_OUTSIDE_TLAB) || (event_type != BCI_CPU && event_type != BCI_WALL && @@ -318,30 +320,124 @@ int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames, java_ctx, truncated); } - return convertNativeTrace(native_frames, callchain, frames); + return convertNativeTrace(native_frames, callchain, frames, lock_index); +} + +/** + * Allocate a RemoteFrameInfo from the pre-allocated pool for the given lock-strip. + * This is signal-safe (uses atomic operations, no dynamic allocation). + * + * @param lock_index The lock-strip index (0 to CONCURRENCY_LEVEL-1) + * @return Pointer to allocated RemoteFrameInfo, or nullptr if pool exhausted + */ +RemoteFrameInfo* Profiler::allocateRemoteFrameInfo(int lock_index) { + if (lock_index < 0 || lock_index >= CONCURRENCY_LEVEL) { + return nullptr; + } + + if (_remote_frame_pool[lock_index] == nullptr) { + return nullptr; // Pool not initialized + } + + // Atomic fetch-and-add to get next available slot + int slot = _remote_frame_count[lock_index].fetch_add(1, std::memory_order_relaxed); + + if (slot >= REMOTE_FRAME_POOL_SIZE) { + // Pool exhausted - fallback to symbol resolution + // Don't decrement counter as other threads may have succeeded + return nullptr; + } + + return &_remote_frame_pool[lock_index][slot]; +} + +Profiler::NativeFrameResolution Profiler::resolveNativeFrame(uintptr_t pc, int lock_index) { + if (_remote_symbolication) { + // Remote symbolication mode: store build-id and PC offset + CodeCache* lib = _libs->findLibraryByAddress((void*)pc); + TEST_LOG("Remote symbolication: pc=0x%lx, lib=%p, hasBuildId=%d", + pc, lib, lib != nullptr ? lib->hasBuildId() : -1); + if (lib != nullptr && lib->hasBuildId()) { + TEST_LOG("Using remote symbolication for lib=%s, build-id=%s", + lib->name(), lib->buildId()); + // Check if this is a marked C++ interpreter frame before using remote format + const char *method_name = nullptr; + lib->binarySearch((void*)pc, &method_name); + if (method_name != nullptr && NativeFunc::isMarked(method_name)) { + // This is C++ interpreter frame, this and later frames should be reported + // as Java frames returned by AGCT. Terminate the scan here. + return {nullptr, BCI_NATIVE_FRAME, true}; + } + + // Calculate PC offset within the library + uintptr_t offset = pc - (uintptr_t)lib->imageBase(); + + // Allocate RemoteFrameInfo from pre-allocated pool (signal-safe) + RemoteFrameInfo* rfi = allocateRemoteFrameInfo(lock_index); + if (rfi != nullptr) { + rfi->build_id = lib->buildId(); + rfi->pc_offset = offset; + rfi->lib_index = lib->libIndex(); + + return {(jmethodID)rfi, BCI_NATIVE_FRAME_REMOTE, false}; + } else { + // Pool exhausted, fallback to resolved symbol + // Need to resolve the symbol now since we didn't do it earlier + const char *fallback_name = nullptr; + lib->binarySearch((void*)pc, &fallback_name); + return {(jmethodID)fallback_name, BCI_NATIVE_FRAME, false}; + } + } else { + // Library not found or no build-id, fallback to resolved symbol + const char *method_name = findNativeMethod((void*)pc); + if (method_name != nullptr && NativeFunc::isMarked(method_name)) { + // This is C++ interpreter frame, this and later frames should be reported + // as Java frames returned by AGCT. Terminate the scan here. + return {nullptr, BCI_NATIVE_FRAME, true}; + } + return {(jmethodID)method_name, BCI_NATIVE_FRAME, false}; + } + } else { + // Traditional mode: resolve and store symbol name + const char *method_name = findNativeMethod((void*)pc); + if (method_name != nullptr && NativeFunc::isMarked(method_name)) { + // This is C++ interpreter frame, this and later frames should be reported + // as Java frames returned by AGCT. Terminate the scan here. + return {nullptr, BCI_NATIVE_FRAME, true}; + } + return {(jmethodID)method_name, BCI_NATIVE_FRAME, false}; + } +} + +Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t pc, int lock_index) { + // Direct pass-through to resolveNativeFrame with lock_index + return resolveNativeFrame(pc, lock_index); } int Profiler::convertNativeTrace(int native_frames, const void **callchain, - ASGCT_CallFrame *frames) { + ASGCT_CallFrame *frames, int lock_index) { int depth = 0; jmethodID prev_method = NULL; for (int i = 0; i < native_frames; i++) { - const char *current_method_name = findNativeMethod(callchain[i]); - if (current_method_name != NULL && - NativeFunc::isMarked(current_method_name)) { - // This is C++ interpreter frame, this and later frames should be reported - // as Java frames returned by AGCT. Terminate the scan here. + uintptr_t pc = (uintptr_t)callchain[i]; + + // Resolve native frame using shared logic + NativeFrameResolution resolution = resolveNativeFrame(pc, lock_index); + + // Check if this is a marked frame (terminate scan) + if (resolution.is_marked) { return depth; } - jmethodID current_method = (jmethodID)current_method_name; + jmethodID current_method = resolution.method_id; + int current_bci = resolution.bci; + + // Skip duplicates in LBR stack if (current_method == prev_method && _cstack == CSTACK_LBR) { - // Skip duplicates in LBR stack, where branch_stack[N].from == - // branch_stack[N+1].to prev_method = NULL; - } else { - frames[depth].bci = BCI_NATIVE_FRAME; + } else if (current_method != NULL) { + frames[depth].bci = current_bci; frames[depth].method_id = prev_method = current_method; depth++; } @@ -709,14 +805,18 @@ void Profiler::recordSample(void *ucontext, u64 counter, int tid, StackContext java_ctx = {0}; ASGCT_CallFrame *native_stop = frames + num_frames; num_frames += getNativeTrace(ucontext, native_stop, event_type, tid, - &java_ctx, &truncated); + &java_ctx, &truncated, lock_index); if (num_frames < _max_stack_depth) { int max_remaining = _max_stack_depth - num_frames; if (_features.mixed) { - num_frames += ddprof::StackWalker::walkVM(ucontext, frames + num_frames, max_remaining, _features, eventTypeFromBCI(event_type), &truncated); + int vm_start = num_frames; + int vm_frames = ddprof::StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), &truncated, lock_index); + num_frames += vm_frames; } else if (event_type == BCI_CPU || event_type == BCI_WALL) { if (_cstack >= CSTACK_VM) { - num_frames += ddprof::StackWalker::walkVM(ucontext, frames + num_frames, max_remaining, _features, eventTypeFromBCI(event_type), &truncated); + int vm_start = num_frames; + int vm_frames = ddprof::StackWalker::walkVM(ucontext, frames + vm_start, max_remaining, _features, eventTypeFromBCI(event_type), &truncated, lock_index); + num_frames += vm_frames; } else { // Async events AsyncSampleMutex mutex(ProfiledThread::currentSignalSafe()); @@ -1137,6 +1237,7 @@ Error Profiler::start(Arguments &args, bool reset) { // Validate TLS priming for signal-based profiling safety if (ProfiledThread::wasTlsPrimingAttempted() && !ProfiledThread::isTlsPrimingAvailable()) { _omit_stacktraces = args._lightweight; + _remote_symbolication = args._remote_symbolication; _event_mask = ((args._event != NULL && strcmp(args._event, EVENT_NOOP) != 0) ? EM_CPU : 0) | @@ -1158,6 +1259,7 @@ Error Profiler::start(Arguments &args, bool reset) { } } else { _omit_stacktraces = args._lightweight; + _remote_symbolication = args._remote_symbolication; _event_mask = ((args._event != NULL && strcmp(args._event, EVENT_NOOP) != 0) ? EM_CPU : 0) | @@ -1211,6 +1313,26 @@ Error Profiler::start(Arguments &args, bool reset) { } } + // Allocate/reset remote frame pools if remote symbolication is enabled + if (args._remote_symbolication) { + for (int i = 0; i < CONCURRENCY_LEVEL; i++) { + if (_remote_frame_pool[i] == nullptr) { + // First-time allocation + _remote_frame_pool[i] = (RemoteFrameInfo*)calloc(REMOTE_FRAME_POOL_SIZE, sizeof(RemoteFrameInfo)); + if (_remote_frame_pool[i] == nullptr) { + // Clean up already allocated pools + for (int j = 0; j < i; j++) { + free(_remote_frame_pool[j]); + _remote_frame_pool[j] = nullptr; + } + return Error("Not enough memory to allocate remote frame pools"); + } + } + // Reset counter for this lock-strip (handles both first-time and restart) + _remote_frame_count[i].store(0, std::memory_order_relaxed); + } + } + _features = args._features; if (VM::hotspot_version() < 8) { _features.java_anchor = 0; @@ -1262,6 +1384,11 @@ Error Profiler::start(Arguments &args, bool reset) { // Kernel symbols are useful only for perf_events without --all-user _libs->updateSymbols(_cpu_engine == &perf_events && (args._ring & RING_KERNEL)); + // Extract build-ids for remote symbolication if enabled + if (_remote_symbolication) { + _libs->updateBuildIds(); + } + enableEngines(); switchLibraryTrap(_cstack == CSTACK_DWARF); @@ -1431,10 +1558,11 @@ Error Profiler::dump(const char *path, const int length) { updateJavaThreadNames(); updateNativeThreadNames(); - Counters::set(CODECACHE_NATIVE_COUNT, _native_libs.count()); - Counters::set(CODECACHE_NATIVE_SIZE_BYTES, _native_libs.memoryUsage()); + const CodeCacheArray& native_libs = _libs->native_libs(); + Counters::set(CODECACHE_NATIVE_COUNT, native_libs.count()); + Counters::set(CODECACHE_NATIVE_SIZE_BYTES, native_libs.memoryUsage()); Counters::set(CODECACHE_RUNTIME_STUBS_SIZE_BYTES, - _native_libs.memoryUsage()); + native_libs.memoryUsage()); lockAll(); Error err = _jfr.dump(path, length); diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 141b21539..f638f8267 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -150,11 +150,16 @@ class alignas(alignof(SpinLock)) Profiler { Libraries* _libs; SpinLock _stubs_lock; CodeCache _runtime_stubs; - CodeCacheArray _native_libs; const void *_call_stub_begin; const void *_call_stub_end; u32 _num_context_attributes; bool _omit_stacktraces; + bool _remote_symbolication; // Enable remote symbolication for native frames + + // Remote symbolication frame pool (pre-allocated, signal-safe) + static const int REMOTE_FRAME_POOL_SIZE = 128; // Entries per lock-strip + RemoteFrameInfo *_remote_frame_pool[CONCURRENCY_LEVEL]; + std::atomic _remote_frame_count[CONCURRENCY_LEVEL]; // dlopen() hook support void **_dlopen_entry; @@ -174,7 +179,7 @@ class alignas(alignof(SpinLock)) Profiler { u32 getLockIndex(int tid); bool isAddressInCode(uintptr_t addr); int getNativeTrace(void *ucontext, ASGCT_CallFrame *frames, int event_type, - int tid, StackContext *java_ctx, bool *truncated); + int tid, StackContext *java_ctx, bool *truncated, int lock_index); int getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames, int max_depth, StackContext *java_ctx, bool *truncated); void fillFrameTypes(ASGCT_CallFrame *frames, int num_frames, @@ -204,7 +209,7 @@ class alignas(alignof(SpinLock)) Profiler { _notify_class_unloaded_func(NULL), _thread_filter(), _call_trace_storage(), _jfr(), _start_time(0), _epoch(0), _timer_id(NULL), _max_stack_depth(0), _safe_mode(0), _thread_events_state(JVMTI_DISABLE), - _libs(Libraries::instance()), _stubs_lock(), _runtime_stubs("[stubs]"), _native_libs(), + _libs(Libraries::instance()), _stubs_lock(), _runtime_stubs("[stubs]"), _call_stub_begin(NULL), _call_stub_end(NULL), _dlopen_entry(NULL), _num_context_attributes(0), _class_map(1), _string_label_map(2), _context_value_map(3), _cpu_engine(), _alloc_engine(), _event_mask(0), @@ -279,8 +284,19 @@ class alignas(alignof(SpinLock)) Profiler { Error dump(const char *path, const int length); void logStats(); void switchThreadEvents(jvmtiEventMode mode); + + // Result of resolving a native frame for symbolication + struct NativeFrameResolution { + jmethodID method_id; // RemoteFrameInfo* or const char* symbol name, or nullptr if marked + int bci; // BCI_NATIVE_FRAME_REMOTE or BCI_NATIVE_FRAME + bool is_marked; // true if this is a marked C++ interpreter frame (stop processing) + }; + + NativeFrameResolution resolveNativeFrame(uintptr_t pc, int lock_index); + NativeFrameResolution resolveNativeFrameForWalkVM(uintptr_t pc, int lock_index); + RemoteFrameInfo* allocateRemoteFrameInfo(int lock_index); int convertNativeTrace(int native_frames, const void **callchain, - ASGCT_CallFrame *frames); + ASGCT_CallFrame *frames, int lock_index); void recordSample(void *ucontext, u64 weight, int tid, jint event_type, u64 call_trace_id, Event *event); u64 recordJVMTISample(u64 weight, int tid, jthread thread, jint event_type, Event *event, bool deferred); diff --git a/ddprof-lib/src/main/cpp/stackWalker_dd.h b/ddprof-lib/src/main/cpp/stackWalker_dd.h index 0d88f87cc..163a8d55c 100644 --- a/ddprof-lib/src/main/cpp/stackWalker_dd.h +++ b/ddprof-lib/src/main/cpp/stackWalker_dd.h @@ -50,8 +50,8 @@ namespace ddprof { return walked; } - inline static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, StackWalkFeatures features, EventType event_type, bool* truncated) { - int walked = ::StackWalker::walkVM(ucontext, frames, max_depth + 1, features, event_type); + inline static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, StackWalkFeatures features, EventType event_type, bool* truncated, int lock_index) { + int walked = ::StackWalker::walkVM(ucontext, frames, max_depth + 1, features, event_type, lock_index); if (walked > max_depth) { *truncated = true; walked = max_depth; @@ -64,8 +64,8 @@ namespace ddprof { return walked; } - inline static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, JavaFrameAnchor* anchor, EventType event_type, bool* truncated) { - int walked = ::StackWalker::walkVM(ucontext, frames, max_depth + 1, anchor, event_type); + inline static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, JavaFrameAnchor* anchor, EventType event_type, bool* truncated, int lock_index) { + int walked = ::StackWalker::walkVM(ucontext, frames, max_depth + 1, anchor, event_type, lock_index); if (walked > max_depth) { *truncated = true; walked = max_depth; diff --git a/ddprof-lib/src/main/cpp/symbols_linux_dd.cpp b/ddprof-lib/src/main/cpp/symbols_linux_dd.cpp new file mode 100644 index 000000000..6cd9471e1 --- /dev/null +++ b/ddprof-lib/src/main/cpp/symbols_linux_dd.cpp @@ -0,0 +1,163 @@ +/* + * Copyright 2025, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifdef __linux__ + +#include "symbols_linux_dd.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// GNU build-id note constants +#define NT_GNU_BUILD_ID 3 +#define GNU_BUILD_ID_NAME "GNU" + +namespace ddprof { + +char* SymbolsLinux::extractBuildId(const char* file_path, size_t* build_id_len) { + if (!file_path || !build_id_len) { + return nullptr; + } + + int fd = open(file_path, O_RDONLY); + if (fd < 0) { + return nullptr; + } + + struct stat st; + if (fstat(fd, &st) < 0) { + close(fd); + return nullptr; + } + + void* elf_base = mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + + if (elf_base == MAP_FAILED) { + return nullptr; + } + + char* result = extractBuildIdFromMemory(elf_base, st.st_size, build_id_len); + + munmap(elf_base, st.st_size); + return result; +} + +char* SymbolsLinux::extractBuildIdFromMemory(const void* elf_base, size_t elf_size, size_t* build_id_len) { + if (!elf_base || !build_id_len || elf_size < sizeof(Elf64_Ehdr)) { + return nullptr; + } + + const Elf64_Ehdr* ehdr = static_cast(elf_base); + + // Verify ELF magic + if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) { + return nullptr; + } + + // Only handle 64-bit ELF for now + if (ehdr->e_ident[EI_CLASS] != ELFCLASS64) { + return nullptr; + } + + // Check if we have program headers + if (ehdr->e_phoff == 0 || ehdr->e_phnum == 0) { + return nullptr; + } + + // Verify program header table is within file bounds + if (ehdr->e_phoff + ehdr->e_phnum * sizeof(Elf64_Phdr) > elf_size) { + return nullptr; + } + + // Verify program header offset is properly aligned + if (ehdr->e_phoff % alignof(Elf64_Phdr) != 0) { + return nullptr; + } + + const char* base = static_cast(elf_base); + const Elf64_Phdr* phdr = reinterpret_cast(base + ehdr->e_phoff); + + // Search for PT_NOTE segments + for (int i = 0; i < ehdr->e_phnum; i++) { + if (phdr[i].p_type == PT_NOTE && phdr[i].p_filesz > 0) { + // Ensure note segment is within file bounds + if (phdr[i].p_offset + phdr[i].p_filesz > elf_size) { + continue; + } + + const void* note_data = base + phdr[i].p_offset; + const uint8_t* build_id_bytes = findBuildIdInNotes(note_data, phdr[i].p_filesz, build_id_len); + + if (build_id_bytes) { + return buildIdToHex(build_id_bytes, *build_id_len); + } + } + } + + return nullptr; +} + +const uint8_t* SymbolsLinux::findBuildIdInNotes(const void* note_data, size_t note_size, size_t* build_id_len) { + const char* data = static_cast(note_data); + size_t offset = 0; + + while (offset + sizeof(Elf64_Nhdr) < note_size) { + const Elf64_Nhdr* nhdr = reinterpret_cast(data + offset); + + // Calculate aligned sizes + size_t name_size_aligned = (nhdr->n_namesz + 3) & ~3; + size_t desc_size_aligned = (nhdr->n_descsz + 3) & ~3; + + // Check bounds + if (offset + sizeof(Elf64_Nhdr) + name_size_aligned + desc_size_aligned > note_size) { + break; + } + + // Check if this is a GNU build-id note + if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz > 0 && nhdr->n_descsz > 0) { + const char* name = data + offset + sizeof(Elf64_Nhdr); + + // Verify GNU build-id name (including null terminator) + if (nhdr->n_namesz == 4 && strncmp(name, GNU_BUILD_ID_NAME, 3) == 0 && name[3] == '\0') { + const uint8_t* desc = reinterpret_cast(data + offset + sizeof(Elf64_Nhdr) + name_size_aligned); + *build_id_len = nhdr->n_descsz; + return desc; + } + } + + offset += sizeof(Elf64_Nhdr) + name_size_aligned + desc_size_aligned; + } + + return nullptr; +} + +char* SymbolsLinux::buildIdToHex(const uint8_t* build_id_bytes, size_t byte_len) { + if (!build_id_bytes || byte_len == 0) { + return nullptr; + } + + // Allocate string for hex representation (2 chars per byte + null terminator) + char* hex_str = static_cast(malloc(byte_len * 2 + 1)); + if (!hex_str) { + return nullptr; + } + + for (size_t i = 0; i < byte_len; i++) { + snprintf(hex_str + i * 2, 3, "%02x", build_id_bytes[i]); + } + + hex_str[byte_len * 2] = '\0'; + return hex_str; +} + +} // namespace ddprof + +#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/symbols_linux_dd.h b/ddprof-lib/src/main/cpp/symbols_linux_dd.h new file mode 100644 index 000000000..0282f1e32 --- /dev/null +++ b/ddprof-lib/src/main/cpp/symbols_linux_dd.h @@ -0,0 +1,68 @@ +/* + * Copyright 2025, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _SYMBOLS_LINUX_DD_H +#define _SYMBOLS_LINUX_DD_H + +#ifdef __linux__ + +#include +#include + +namespace ddprof { + +/** + * Datadog-specific extensions to Linux symbol handling. + * Provides build-id extraction for remote symbolication support. + */ +class SymbolsLinux { +public: + /** + * Extract GNU build-id from ELF file on disk. + * Build-id is stored in .note.gnu.build-id section and provides + * unique identification for libraries/executables for remote symbolication. + * + * @param file_path Path to ELF file + * @param build_id_len Output parameter for build-id length in bytes + * @return Hex-encoded build-id string (caller must free), or NULL on error + */ + static char* extractBuildId(const char* file_path, size_t* build_id_len); + + /** + * Extract GNU build-id from ELF file already mapped in memory. + * + * @param elf_base Base address of mapped ELF file + * @param elf_size Size of mapped ELF file + * @param build_id_len Output parameter for build-id length in bytes + * @return Hex-encoded build-id string (caller must free), or NULL on error + */ + static char* extractBuildIdFromMemory(const void* elf_base, size_t elf_size, size_t* build_id_len); + +private: + /** + * Convert binary build-id to hex string. + * + * @param build_id_bytes Raw build-id bytes + * @param byte_len Length of raw build-id in bytes + * @return Hex-encoded string (caller must free) + */ + static char* buildIdToHex(const uint8_t* build_id_bytes, size_t byte_len); + + /** + * Parse ELF note section to find GNU build-id. + * + * @param note_data Start of note section data + * @param note_size Size of note section + * @param build_id_len Output parameter for build-id length + * @return Raw build-id bytes, or NULL if not found + */ + static const uint8_t* findBuildIdInNotes(const void* note_data, size_t note_size, size_t* build_id_len); +}; + +} // namespace ddprof + +#endif // __linux__ + +#endif // _SYMBOLS_LINUX_DD_H diff --git a/ddprof-lib/src/main/cpp/vmEntry.h b/ddprof-lib/src/main/cpp/vmEntry.h index 2047193d4..ed8adf2eb 100644 --- a/ddprof-lib/src/main/cpp/vmEntry.h +++ b/ddprof-lib/src/main/cpp/vmEntry.h @@ -32,6 +32,7 @@ enum ASGCT_CallFrameType { BCI_PARK = -16, // class name of the park() blocker BCI_THREAD_ID = -17, // method_id designates a thread BCI_ERROR = -18, // method_id is an error string + BCI_NATIVE_FRAME_REMOTE = -19, // method_id points to RemoteFrameInfo for remote symbolication }; // See hotspot/src/share/vm/prims/forte.cpp @@ -57,6 +58,21 @@ typedef struct { jmethodID method_id; } ASGCT_CallFrame; +/** + * Information for native frames requiring remote symbolication. + * Used when bci == BCI_NATIVE_FRAME_REMOTE. + */ +typedef struct RemoteFrameInfo { + const char* build_id; // GNU build-id for library identification (null-terminated hex string) + uintptr_t pc_offset; // PC offset within the library/module + short lib_index; // Index into CodeCache library table for fast lookup + +#ifdef __cplusplus + // Constructor for C++ convenience + RemoteFrameInfo(const char* bid, uintptr_t offset, short lib_idx) + : build_id(bid), pc_offset(offset), lib_index(lib_idx) {} +#endif +} RemoteFrameInfo; typedef struct { JNIEnv *env; diff --git a/ddprof-lib/src/test/cpp/remoteargs_ut.cpp b/ddprof-lib/src/test/cpp/remoteargs_ut.cpp new file mode 100644 index 000000000..2ba3c9af7 --- /dev/null +++ b/ddprof-lib/src/test/cpp/remoteargs_ut.cpp @@ -0,0 +1,88 @@ +/* + * Copyright 2025, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include "arguments.h" +#include "../../main/cpp/gtest_crash_handler.h" + +static constexpr char REMOTE_ARGS_TEST_NAME[] = "RemoteArgsTest"; + +class RemoteArgsGlobalSetup { +public: + RemoteArgsGlobalSetup() { + installGtestCrashHandler(); + } + ~RemoteArgsGlobalSetup() { + restoreDefaultSignalHandlers(); + } +}; + +static RemoteArgsGlobalSetup global_setup; + +class RemoteArgsTest : public ::testing::Test { +protected: + void SetUp() override { + // Test setup + } + + void TearDown() override { + // Test cleanup + } +}; + +TEST_F(RemoteArgsTest, DefaultRemoteSymbolicationDisabled) { + Arguments args; + + // Remote symbolication should be disabled by default + EXPECT_FALSE(args._remote_symbolication); +} + +TEST_F(RemoteArgsTest, EnableRemoteSymbolication) { + Arguments args; + + // Test enabling remote symbolication + Error error = args.parse("remotesym=true"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._remote_symbolication); +} + +TEST_F(RemoteArgsTest, EnableRemoteSymbolicationShort) { + Arguments args; + + // Test short form + Error error = args.parse("remotesym=y"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._remote_symbolication); +} + +TEST_F(RemoteArgsTest, DisableRemoteSymbolication) { + Arguments args; + + // Test explicitly disabling + Error error = args.parse("remotesym=false"); + EXPECT_FALSE(error); + EXPECT_FALSE(args._remote_symbolication); +} + +TEST_F(RemoteArgsTest, MultipleArgsWithRemoteSymbolication) { + Arguments args; + + // Test with multiple arguments + Error error = args.parse("event=cpu,interval=1000000,remotesym=true"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._remote_symbolication); + EXPECT_STREQ(args._event, "cpu"); + EXPECT_EQ(args._interval, 1000000); +} + +TEST_F(RemoteArgsTest, RemoteSymbolicationWithOtherFlags) { + Arguments args; + + // Test interaction with lightweight flag + Error error = args.parse("lightweight=true,remotesym=true"); + EXPECT_FALSE(error); + EXPECT_TRUE(args._lightweight); + EXPECT_TRUE(args._remote_symbolication); +} \ No newline at end of file diff --git a/ddprof-lib/src/test/cpp/remotesymbolication_ut.cpp b/ddprof-lib/src/test/cpp/remotesymbolication_ut.cpp new file mode 100644 index 000000000..243a30f6c --- /dev/null +++ b/ddprof-lib/src/test/cpp/remotesymbolication_ut.cpp @@ -0,0 +1,120 @@ +/* + * Copyright 2025, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include "symbols_linux_dd.h" +#include "vmEntry.h" +#include "../../main/cpp/gtest_crash_handler.h" + +#ifdef __linux__ + +static constexpr char REMOTE_TEST_NAME[] = "RemoteSymbolicationTest"; + +class RemoteSymbolicationGlobalSetup { +public: + RemoteSymbolicationGlobalSetup() { + installGtestCrashHandler(); + } + ~RemoteSymbolicationGlobalSetup() { + restoreDefaultSignalHandlers(); + } +}; + +static RemoteSymbolicationGlobalSetup global_setup; + +class RemoteSymbolicationTest : public ::testing::Test { +protected: + void SetUp() override { + // Test setup + } + + void TearDown() override { + // Test cleanup + } +}; + +TEST_F(RemoteSymbolicationTest, RemoteFrameInfoConstruction) { + const char* test_build_id = "deadbeefcafebabe"; + uintptr_t test_offset = 0x1234; + short test_lib_index = 5; + + RemoteFrameInfo rfi(test_build_id, test_offset, test_lib_index); + + EXPECT_STREQ(rfi.build_id, test_build_id); + EXPECT_EQ(rfi.pc_offset, test_offset); + EXPECT_EQ(rfi.lib_index, test_lib_index); +} + +TEST_F(RemoteSymbolicationTest, BciFrameTypeConstants) { + // Verify that the new BCI constant is defined + EXPECT_EQ(BCI_NATIVE_FRAME_REMOTE, -19); + + // Verify it doesn't conflict with existing constants + EXPECT_NE(BCI_NATIVE_FRAME_REMOTE, BCI_NATIVE_FRAME); + EXPECT_NE(BCI_NATIVE_FRAME_REMOTE, BCI_ERROR); + EXPECT_NE(BCI_NATIVE_FRAME_REMOTE, BCI_ALLOC); +} + +// Test build-id extraction from a minimal ELF +TEST_F(RemoteSymbolicationTest, BuildIdExtractionBasic) { + // Create a minimal ELF file with a build-id note section + // This test would be more comprehensive with a real ELF file + // For now, just test the function doesn't crash on invalid input + + size_t build_id_len = 0; + char* build_id = ddprof::SymbolsLinux::extractBuildId("/nonexistent", &build_id_len); + + // Should return null for non-existent file + EXPECT_EQ(build_id, nullptr); + EXPECT_EQ(build_id_len, 0); +} + +TEST_F(RemoteSymbolicationTest, BuildIdExtractionInvalidInput) { + size_t build_id_len = 0; + + // Test null inputs + char* build_id1 = ddprof::SymbolsLinux::extractBuildId(nullptr, &build_id_len); + EXPECT_EQ(build_id1, nullptr); + + char* build_id2 = ddprof::SymbolsLinux::extractBuildId("/some/file", nullptr); + EXPECT_EQ(build_id2, nullptr); + + // Test non-ELF file + const char* test_content = "This is not an ELF file"; + const char* temp_file = "/tmp/not_an_elf"; + + int fd = open(temp_file, O_RDWR | O_CREAT | O_TRUNC, 0600); + if (fd >= 0) { + write(fd, test_content, strlen(test_content)); + close(fd); + + char* build_id3 = ddprof::SymbolsLinux::extractBuildId(temp_file, &build_id_len); + EXPECT_EQ(build_id3, nullptr); + + unlink(temp_file); + } +} + +TEST_F(RemoteSymbolicationTest, BuildIdFromMemoryInvalidInput) { + size_t build_id_len = 0; + + // Test null pointer + char* build_id1 = ddprof::SymbolsLinux::extractBuildIdFromMemory(nullptr, 100, &build_id_len); + EXPECT_EQ(build_id1, nullptr); + + // Test invalid size + char dummy_data[10] = {0}; + char* build_id2 = ddprof::SymbolsLinux::extractBuildIdFromMemory(dummy_data, 0, &build_id_len); + EXPECT_EQ(build_id2, nullptr); + + // Test null output parameter + char* build_id3 = ddprof::SymbolsLinux::extractBuildIdFromMemory(dummy_data, 10, nullptr); + EXPECT_EQ(build_id3, nullptr); +} + +#endif // __linux__ \ No newline at end of file diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index 83a3829ea..ff82cd2ab 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -49,6 +49,7 @@ tasks.register('buildNativeJniLibrary', Exec) { args "-I${System.getenv('JAVA_HOME')}/include/linux" // Linux-specific includes args "-fPIC" args "-shared" // Build a shared library on Linux + args "-Wl,--build-id" // Embed GNU build-id for remote symbolication testing } args nativeSrcDir.listFiles()*.getAbsolutePath() // Source files args "-o", "${outputLibDir.absolutePath}/${libraryFileName}" // Output file path diff --git a/ddprof-test/src/test/cpp/remotesym.c b/ddprof-test/src/test/cpp/remotesym.c new file mode 100644 index 000000000..7f4c647bf --- /dev/null +++ b/ddprof-test/src/test/cpp/remotesym.c @@ -0,0 +1,62 @@ +/* + * Copyright 2025, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +/** + * Recursive CPU-burning function that creates a distinct call stack. + * This ensures native frames from this library appear in profiling samples. + */ +static uint64_t burn_cpu_recursive(uint64_t n, uint64_t depth) { + if (depth == 0) { + // Base case: perform actual computation + uint64_t sum = 0; + for (uint64_t i = 0; i < n; i++) { + sum += i * i; + } + return sum; + } + + // Recursive case: go deeper + return burn_cpu_recursive(n, depth - 1) + depth; +} + +/** + * Entry point for CPU-burning work. + * Called from Java to generate CPU samples with this library on the stack. + */ +JNIEXPORT jlong JNICALL +Java_com_datadoghq_profiler_RemoteSymHelper_burnCpu(JNIEnv *env, jclass clazz, jlong iterations, jint depth) { + return (jlong)burn_cpu_recursive((uint64_t)iterations, (uint32_t)depth); +} + +/** + * Additional function to create more stack depth. + */ +static uint64_t compute_fibonacci(uint32_t n) { + if (n <= 1) return n; + + uint64_t a = 0, b = 1; + for (uint32_t i = 2; i <= n; i++) { + uint64_t temp = a + b; + a = b; + b = temp; + } + return b; +} + +/** + * Another entry point with different stack signature. + */ +JNIEXPORT jlong JNICALL +Java_com_datadoghq_profiler_RemoteSymHelper_computeFibonacci(JNIEnv *env, jclass clazz, jint n) { + // Call multiple times to increase likelihood of sampling + uint64_t result = 0; + for (int i = 0; i < 1000; i++) { + result += compute_fibonacci((uint32_t)n); + } + return (jlong)result; +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/RemoteSymHelper.java b/ddprof-test/src/test/java/com/datadoghq/profiler/RemoteSymHelper.java new file mode 100644 index 000000000..af6cf86bd --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/RemoteSymHelper.java @@ -0,0 +1,35 @@ +/* + * Copyright 2025, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler; + +/** + * Helper class for remote symbolication testing. + * Provides JNI methods that burn CPU to ensure native frames appear in profiling samples. + * The native library is built with GNU build-id on Linux for remote symbolication testing. + */ +public class RemoteSymHelper { + static { + System.loadLibrary("ddproftest"); + } + + /** + * Burns CPU cycles by performing recursive computation. + * This creates a distinctive call stack that should appear in profiling samples. + * + * @param iterations Number of iterations for computation + * @param depth Recursion depth + * @return Computed result (to prevent optimization) + */ + public static native long burnCpu(long iterations, int depth); + + /** + * Computes Fibonacci numbers repeatedly to burn CPU. + * + * @param n Fibonacci number to compute + * @return Computed result (to prevent optimization) + */ + public static native long computeFibonacci(int n); +} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/RemoteSymbolicationTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/RemoteSymbolicationTest.java new file mode 100644 index 000000000..694b95193 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/RemoteSymbolicationTest.java @@ -0,0 +1,189 @@ +/* + * Copyright 2025, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler.cpu; + +import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; +import com.datadoghq.profiler.Platform; +import com.datadoghq.profiler.RemoteSymHelper; +import com.datadoghq.profiler.junit.CStack; +import com.datadoghq.profiler.junit.RetryTest; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.params.provider.ValueSource; +import org.openjdk.jmc.common.item.Attribute; +import org.openjdk.jmc.common.item.IAttribute; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemCollection; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; + +/** + * Integration test for remote symbolication feature. + * + *

Tests that when remotesym=true is enabled: + *

    + *
  • Native frames contain build-id instead of symbol names
    • + *
  • PC offsets are stored instead of symbol addresses
    • + *
  • Build-ids are valid hex strings
    • + *
+ */ +public class RemoteSymbolicationTest extends CStackAwareAbstractProfilerTest { + public RemoteSymbolicationTest(@CStack String cstack) { + super(cstack); + } + + @BeforeEach + public void checkPlatform() { + // Remote symbolication with build-id extraction is Linux-only + Assumptions.assumeTrue(Platform.isLinux(), "Remote symbolication test requires Linux"); + // Zing JVM forces cstack=no which disables native stack walking + Assumptions.assumeFalse(Platform.isZing(), "Remote symbolication test requires native stack walking (incompatible with Zing)"); + } + + @RetryTest(10) + @TestTemplate + @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) + public void testRemoteSymbolicationEnabled(@CStack String cstack) throws Exception { + try (ProfiledCode profiledCode = new ProfiledCode(profiler)) { + for (int i = 0, id = 1; i < 100; i++, id += 3) { + profiledCode.method1(id); + // Call native functions from our test library to ensure + // native frames with build-id appear in the samples + // Increased iterations to ensure profiler captures these frames + RemoteSymHelper.burnCpu(1000000, 10); + RemoteSymHelper.computeFibonacci(35); + } + stopProfiler(); + + verifyCStackSettings(); + + // First verify that our test library (libddproftest) has a build-id + // We use the extended jdk.NativeLibrary event which now includes buildId and loadBias fields + IItemCollection libraryEvents = verifyEvents("jdk.NativeLibrary"); + String testLibBuildId = null; + boolean foundTestLib = false; + + // Create attributes for the custom fields we added to jdk.NativeLibrary + IAttribute buildIdAttr = Attribute.attr("buildId", "buildId", "GNU Build ID", PLAIN_TEXT); + IAttribute nameAttr = Attribute.attr("name", "name", "Name", PLAIN_TEXT); + + for (IItemIterable libItems : libraryEvents) { + IMemberAccessor buildIdAccessor = buildIdAttr.getAccessor(libItems.getType()); + IMemberAccessor nameAccessor = nameAttr.getAccessor(libItems.getType()); + + for (IItem libItem : libItems) { + String name = nameAccessor.getMember(libItem); + String buildId = buildIdAccessor.getMember(libItem); + + System.out.println("Library: " + name + " -> build-id: " + + (buildId != null && !buildId.isEmpty() ? buildId : "")); + + // Check if this is our test library + if (name != null && name.contains("libddproftest")) { + foundTestLib = true; + testLibBuildId = buildId; + System.out.println("Found test library: " + name + " with build-id: " + buildId); + } + } + } + + // Our test library MUST be present and have a build-id + Assumptions.assumeTrue(foundTestLib, + "Test library libddproftest not found in jdk.NativeLibrary events. " + + "The test needs this library to verify remote symbolication."); + Assumptions.assumeTrue(testLibBuildId != null && !testLibBuildId.isEmpty(), + "Test library libddproftest found but has no build-id. " + + "Cannot test remote symbolication without build-id."); + + IItemCollection events = verifyEvents("datadog.ExecutionSample"); + + boolean foundTestLibFrame = false; + boolean foundTestLibRemoteFrame = false; + int sampleCount = 0; + int printCount = 0; + int testLibFrameCount = 0; + + for (IItemIterable cpuSamples : events) { + IMemberAccessor frameAccessor = + JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); + + for (IItem sample : cpuSamples) { + String stackTrace = frameAccessor.getMember(sample); + assertFalse(stackTrace.contains("jvmtiError")); + + sampleCount++; + + // Check if this sample contains frames from our test library + // In remote symbolication mode, frames will have format: .(0x) + // In fallback mode (or non-remote), they might have resolved symbols or lib names + boolean hasTestLibInStack = stackTrace.contains("burn_cpu") || + stackTrace.contains("compute_fibonacci") || + stackTrace.contains("libddproftest") || + (testLibBuildId != null && stackTrace.contains(testLibBuildId)); + + if (hasTestLibInStack) { + testLibFrameCount++; + foundTestLibFrame = true; + + // Print samples containing test lib frames for debugging + if (printCount < 5) { + System.out.println("=== Sample with test lib frame " + (printCount + 1) + " ==="); + System.out.println(stackTrace); + System.out.println("=================="); + printCount++; + } + + // In remote symbolication mode, frames from libddproftest MUST be formatted as: + // .(0x) + // They should NOT show resolved symbol names like burn_cpu or compute_fibonacci + + // If we see resolved symbol names from our test library, that's a FAILURE + if (stackTrace.contains("burn_cpu") || stackTrace.contains("compute_fibonacci")) { + fail("Found resolved symbol names (burn_cpu/compute_fibonacci) from libddproftest in stack trace. " + + "Remote symbolication should use .(0x) format instead. " + + "Stack trace:\n" + stackTrace); + } + + // Check if this sample has the expected remote symbolication format with our test lib's build-id + if (stackTrace.contains(testLibBuildId + ".")) { + foundTestLibRemoteFrame = true; + } + } + } + } + + System.out.println("Total samples analyzed: " + sampleCount); + System.out.println("Samples with test lib frames: " + testLibFrameCount); + System.out.println("Found test lib frames: " + foundTestLibFrame); + System.out.println("Found test lib remote frames: " + foundTestLibRemoteFrame); + System.out.println("Test library build-id: " + testLibBuildId); + + // We call the test library functions extensively, so we MUST see frames from it + assertTrue(foundTestLibFrame, + "No frames from libddproftest found in any samples. " + + "The test called RemoteSymHelper.burnCpu() and computeFibonacci() extensively. " + + "Analyzed " + sampleCount + " samples."); + + // Those frames MUST be in remote symbolication format (not resolved) + assertTrue(foundTestLibRemoteFrame, + "Found frames from libddproftest but they are not in remote symbolication format. " + + "Expected format: " + testLibBuildId + ".(0x). " + + "Analyzed " + testLibFrameCount + " samples with test lib frames."); + } + } + + @Override + protected String getProfilerCommand() { + return "cpu=10ms,remotesym=true"; + } +} diff --git a/doc/MODIFIER_ALLOCATION.md b/doc/MODIFIER_ALLOCATION.md new file mode 100644 index 000000000..9511082ae --- /dev/null +++ b/doc/MODIFIER_ALLOCATION.md @@ -0,0 +1,134 @@ +# Frame Type vs Modifier: Design Decision for Remote Symbolication + +## Final Solution: Use Frame Type Instead of Modifier + +After evaluating multiple approaches, **we chose to use a new frame type (`FRAME_NATIVE_REMOTE = 7`) instead of a custom modifier flag**. This eliminates: +- ✅ All varint encoding overhead +- ✅ Any potential conflicts with Java modifiers +- ✅ Ambiguity in semantics + +Remote native frames now use: +- **Modifier**: `0x0100` (ACC_NATIVE, same as regular native frames) +- **Frame Type**: `FRAME_NATIVE_REMOTE` (7) + +## Java Access Modifiers (from JVM Spec) + +The Java Virtual Machine specification defines the following access modifiers for classes, methods, and fields: + +| Modifier | Value | Applies To | Description | +|----------|-------|------------|-------------| +| ACC_PUBLIC | 0x0001 | All | Public access | +| ACC_PRIVATE | 0x0002 | Methods/Fields | Private access | +| ACC_PROTECTED | 0x0004 | Methods/Fields | Protected access | +| ACC_STATIC | 0x0008 | Methods/Fields | Static member | +| ACC_FINAL | 0x0010 | All | Final/non-overridable | +| ACC_SYNCHRONIZED | 0x0020 | Methods | Synchronized method | +| ACC_SUPER | 0x0020 | Classes | Treat superclass invokes specially | +| ACC_BRIDGE | 0x0040 | Methods | Compiler-generated bridge method | +| ACC_VOLATILE | 0x0040 | Fields | Volatile field | +| ACC_VARARGS | 0x0080 | Methods | Variable arity method | +| ACC_TRANSIENT | 0x0080 | Fields | Not serialized | +| ACC_NATIVE | 0x0100 | Methods | Native implementation | +| **ACC_INTERFACE** | **0x0200** | **Classes** | **Interface declaration** | +| ACC_ABSTRACT | 0x0400 | Classes/Methods | Abstract class/method | +| ACC_STRICT | 0x0800 | Methods | Use strict floating-point | +| ACC_SYNTHETIC | 0x1000 | All | Compiler-generated | +| ACC_ANNOTATION | 0x2000 | Classes | Annotation type | +| ACC_ENUM | 0x4000 | Classes/Fields | Enum type/constant | +| ACC_MANDATED | 0x8000 | Parameters | Implicitly declared | + +## Profiler Custom Modifiers + +For the Java profiler's internal use, we define custom modifier flags that don't conflict with Java's standard modifiers: + +| Modifier | Value | Usage | Notes | +|----------|-------|-------|-------| +| ACC_NATIVE | 0x0100 | Native frames | Reuses Java's ACC_NATIVE for consistency | +| ACC_SYNTHETIC | 0x1000 | Compiler-generated | Reuses Java's ACC_SYNTHETIC | +| ACC_BRIDGE | 0x0040 | Bridge methods | Reuses Java's ACC_BRIDGE | +| **ACC_REMOTE_SYMBOLICATION** | **0x10000** | **Remote native frames** | **Custom profiler flag (bit 16, outside Java range)** | + +## Modifier Conflict Analysis + +### Evolution of the Design + +**Version 1 (Initial)**: Used `0x200` +- ❌ **CONFLICT**: Java's `ACC_INTERFACE` (0x0200) +- Issues: Could confuse JFR parsers, clash with standard modifiers + +**Version 2**: Changed to `0x2000` +- ⚠️ **CONFLICT**: Java's `ACC_ANNOTATION` (0x2000) +- While theoretically safe for methods (annotations only apply to classes), still within Java's reserved range + +**Version 3 (Final)**: Changed to `0x10000` (bit 16) +- ✅ **NO CONFLICTS**: Completely outside Java's standard modifier range (0x0001-0x8000) +- ✅ Clean separation from JVM specification +- ✅ Future-proof against new Java modifiers + +### Why 0x10000 is the Correct Choice + +**Java Modifier Range:** +- Java uses bits 0-15 (0x0001 to 0x8000) +- Highest standard modifier: `ACC_MANDATED = 0x8000` (bit 15) + +**Custom Profiler Range:** +- Bits 16-30 available for custom flags (0x10000 to 0x40000000) +- `0x10000` (bit 16) is first bit outside Java range +- Clean power of 2, easy to test and debug + +**Benefits:** +1. **Zero theoretical conflicts** with any Java modifier (current or future) +2. **Clear separation** between JVM standard (bits 0-15) and profiler custom (bits 16+) +3. **32-bit safe**: Well within `jint` range (signed 32-bit) +4. **JFR compatible**: `_modifiers` field supports full 32-bit values +5. **Extensible**: Room for additional custom flags (0x20000, 0x40000, etc.) + +### Varint Encoding Analysis + +JFR uses LEB128 variable-length encoding for modifiers. The encoding size depends on the value: + +| Value Range | Example | Bytes | Notes | +|-------------|---------|-------|-------| +| 0x00-0x7F | 0 | 1 | Most compact | +| 0x80-0x3FFF | 0x0100 (ACC_NATIVE) | 2 | Standard native frames | +| 0x4000-0x1FFFFF | 0x1000 (ACC_SYNTHETIC) | 2 | High standard modifiers | +| 0x10000+ | 0x10000 | 3 | **+1 byte overhead!** | + +**Critical insight**: Using `0x10000` would add **1 extra byte per remote native frame**. Over millions of frames, this becomes significant! + +### Alternative Approaches Rejected + +1. **Use modifier 0x0200 (bit 9)**: + - ❌ Conflicts with ACC_INTERFACE + +2. **Use modifier 0x2000 (bit 13)**: + - ❌ Conflicts with ACC_ANNOTATION (theoretically) + - ⚠️ Would be safe in practice (annotations only for classes) + +3. **Use modifier 0x10000 (bit 16)**: + - ❌ 3-byte varint encoding vs 2-byte for regular frames + - ❌ **+1 byte overhead per frame** = significant space impact + +4. **Use a separate field**: + - ❌ Would require JFR metadata changes + - ❌ Breaks backward compatibility + +5. **Use frame type FRAME_NATIVE_REMOTE (CHOSEN)**: + - ✅ Zero encoding overhead (type already serialized) + - ✅ No modifier conflicts + - ✅ Clear semantics + +## Best Practices + +When adding custom modifiers in the future: + +1. **Check JVM Spec**: Always verify against latest JVM specification +2. **Consider Context**: Modifiers for methods vs classes vs fields +3. **Document Clearly**: Explain why the bit is safe to use +4. **Test Compatibility**: Verify JFR parsers handle custom modifiers correctly + +## References + +- Java Virtual Machine Specification (JVMS §4.1, §4.5, §4.6) +- JFR Format Specification +- Original Implementation: [elfBuildId.cpp, flightRecorder.cpp] \ No newline at end of file diff --git a/doc/REMOTE_SYMBOLICATION.md b/doc/REMOTE_SYMBOLICATION.md new file mode 100644 index 000000000..82318994c --- /dev/null +++ b/doc/REMOTE_SYMBOLICATION.md @@ -0,0 +1,222 @@ +# Remote Symbolication Implementation + +This document describes the implementation of build-id and pc/offset storage in native frames for remote symbolication in the Java profiler. + +## Overview + +The enhancement allows the Java profiler to store raw build-id and PC offset information for native frames instead of resolving symbols locally. This enables remote symbolication services to handle symbol resolution, which is especially useful for: + +- Distributed profiling scenarios where symbol files aren't available locally +- Reduced profiler overhead by deferring symbol resolution +- Better support for stripped binaries +- Centralized symbol management + +## Implementation Summary + +### 1. **Build-ID Extraction** (`symbols_linux_dd.h/cpp`) + +- **SymbolsLinux**: Utility class to extract GNU build-id from ELF files +- Supports both file-based and memory-based extraction +- Handles .note.gnu.build-id section parsing +- Returns hex-encoded build-id strings + +### 2. **Enhanced CodeCache** (`codeCache.h/cpp`) + +Added fields to store build-id information: +- `_build_id`: Hex-encoded build-id string +- `_build_id_len`: Raw build-id length in bytes +- `_load_bias`: Load bias for address calculations +- Methods: `hasBuildId()`, `buildId()`, `setBuildId()`, etc. + +### 3. **Remote Frame Information** (`vmEntry.h`) + +- **RemoteFrameInfo**: New structure containing: + - `build_id`: Library build-id + - `pc_offset`: PC offset within library + - `lib_index`: Library table index +- **BCI_NATIVE_FRAME_REMOTE**: New frame encoding (-19) + +### 4. **Enhanced Frame Collection** (`profiler.cpp`, `stackWalker_dd.h`) + +Modified frame collection to support dual modes: +- **Traditional mode**: Stores resolved symbol names (existing behavior) +- **Remote mode**: Stores RemoteFrameInfo with build-id and offset + +**Key Functions**: +- `resolveNativeFrame()`: Determines whether to use local or remote symbolication for a given PC +- `resolveNativeFrameForWalkVM()`: Helper for walkVM integration, wraps resolveNativeFrame() +- `allocateRemoteFrameInfo()`: Allocates from pre-allocated signal-safe pool (per lock-strip) +- `convertNativeTrace()`: Converts raw PCs to frames for walkFP/walkDwarf modes + +**Stack Walker Integration**: +- **walkFP/walkDwarf**: Return raw PCs → `convertNativeTrace()` → `resolveNativeFrame()` +- **walkVM/walkVMX**: Directly call `resolveNativeFrameForWalkVM(pc, lock_index)` at native frame resolution (patched via gradle/patching.gradle) + +### 5. **JFR Serialization** (`flightRecorder.cpp/h`) + +- **fillRemoteFrameInfo()**: Serializes remote frame data to JFR format +- Stores `.` in class name field (e.g., `deadbeef1234567890abcdef.`) +- Stores PC offset in signature field (e.g., `(0x1234)`) +- Uses modifier flag 0x100 (ACC_NATIVE, same as regular native frames) + +### 6. **Configuration** (`arguments.h/cpp`) + +- **remotesym[=BOOL]**: New profiler argument +- Default: disabled +- Can be enabled with `remotesym=true` or `remotesym=y` + +### 7. **Libraries Integration** (`libraries.h/cpp`) + +- **updateBuildIds()**: Extracts build-ids for all loaded libraries +- Called during profiler startup when remote symbolication is enabled +- Linux-only implementation using ELF parsing + +### 8. **Upstream Stack Walker Integration** (`gradle/patching.gradle`) + +Patches async-profiler's `stackWalker.h` and `stackWalker.cpp` to integrate remote symbolication: + +**Header Patches (stackWalker.h)**: +- Adds `lock_index` parameter to all three `walkVM` signatures (private implementation, public with features, public with anchor) +- Enables per-strip RemoteFrameInfo pool access during stack walking + +**Implementation Patches (stackWalker.cpp)**: +- Updates all `walkVM` signatures to accept and propagate `lock_index` +- **Critical patch at line 454**: Replaces `profiler->findNativeMethod(pc)` with `profiler->resolveNativeFrameForWalkVM(pc, lock_index)` +- Adds dynamic BCI selection (BCI_NATIVE_FRAME vs BCI_NATIVE_FRAME_REMOTE) +- Adds `fillFrame()` overload for void* method_id to support both symbol names and RemoteFrameInfo pointers +- Handles marked C++ interpreter frames (terminates scan if detected) + +## Usage + +### Enable Remote Symbolication + +```bash +java -agentpath:/libjavaProfiler.so=start,cpu,remotesym=true,file=profile.jfr MyApp +``` + +### Mixed Configuration + +```bash +java -agentpath:/libjavaProfiler.so=start,event=cpu,interval=1000000,remotesym=true,file=profile.jfr MyApp +``` + +## JFR Output Format + +When remote symbolication is enabled, native frames in the JFR output contain: + +- **Class Name**: `.` (e.g., `deadbeef1234567890abcdef.`) + - Build-ID hex string followed by `.` suffix for constant pool deduplication +- **Signature**: PC offset (e.g., `(0x1234)`) +- **Method Name**: The `` suffix from the class name indicates remote symbolication is needed +- **Modifier**: `0x100` (ACC_NATIVE, same as regular native frames) +- **Frame Type**: `FRAME_NATIVE_REMOTE` (7) - distinguishes from regular native frames + +## Backward Compatibility + +- **Default behavior**: No changes (remote symbolication disabled) +- **Mixed traces**: Supports both local and remote frames in same trace +- **Fallback**: Gracefully falls back to local symbolication when build-id unavailable + +## Memory Management + +- **Build-IDs**: Stored once per CodeCache, shared across frames (hex string allocated with malloc) +- **RemoteFrameInfo**: Pre-allocated pool per lock-strip (128 entries × CONCURRENCY_LEVEL strips = ~32KB total) + - Signal-safe allocation using atomic counters + - No dynamic allocation in signal handlers + - Pool resets on profiler restart +- **Automatic cleanup**: Handled by CallTrace storage lifecycle + +## Testing + +### Unit Tests +- **remotesymbolication_ut.cpp**: Tests RemoteFrameInfo structure and build-id extraction +- **remoteargs_ut.cpp**: Tests argument parsing for remote symbolication option + +### Test Coverage +- Build-ID extraction from ELF files +- Frame encoding/decoding +- Argument parsing +- Error handling for invalid inputs + +## Platform Support + +- **Linux**: Full support with ELF build-id extraction +- **macOS/Windows**: Framework in place, needs platform-specific implementation + +## Performance Considerations + +### Benefits +- **Reduced symbol resolution overhead** during profiling +- **Smaller memory footprint** for symbol tables +- **Faster profiling** with deferred symbolication + +### Costs +- **Additional build-id extraction** during startup +- **Slightly larger frame structures** for remote frames +- **Build-ID lookup overhead** during frame collection + +## Future Enhancements + +1. **macOS Support**: Implement Mach-O UUID extraction +2. **Caching**: Cache build-ids across profiler sessions +3. **Compression**: Compress build-ids in JFR output +4. **Validation**: Add runtime validation of build-id consistency +5. **Dynamic Pool Sizing**: Adjust RemoteFrameInfo pool size based on workload +6. **Native Frame Modifier Optimization**: Change native frame modifiers from `0x100` to `0x0` + - Current: All native frames use `0x100` (ACC_NATIVE) = 2-byte varint encoding + - Proposed: Use `0x0` (no modifiers) = 1-byte varint encoding + - Benefit: **Save 1 byte per native frame** across all JFR recordings + - Impact: Significant space savings for native-heavy profiles (C++ applications) + - Note: Would require coordination with JFR parsing tools + +## File Structure + +``` +ddprof-lib/src/main/cpp/ +├── symbols_linux_dd.h # Build-ID extraction interface (Linux-specific) +├── symbols_linux_dd.cpp # Build-ID extraction with bounds/alignment checks +├── vmEntry.h # Enhanced with RemoteFrameInfo and BCI constants +├── codeCache.h # Enhanced with build-id fields (cleaned up operator[]) +├── codeCache.cpp # Build-id storage implementation +├── profiler.h # Added resolveNativeFrame/ForWalkVM, RemoteFrameInfo pool +├── profiler.cpp # Remote symbolication logic and pool allocation +├── stackWalker_dd.h # DataDog wrappers with lock_index parameter +├── flightRecorder.h # Added fillRemoteFrameInfo declaration +├── flightRecorder.cpp # Remote frame JFR serialization +├── arguments.h # Added _remote_symbolication field +├── arguments.cpp # Remote symbolication argument parsing +├── libraries.h # Added updateBuildIds method +└── libraries.cpp # Build-id extraction for loaded libraries + +gradle/ +└── patching.gradle # Upstream stackWalker.h/cpp patches for remote symbolication + +ddprof-lib/src/main/cpp-external/ +├── stackWalker.h # Patched: lock_index parameter added to all walkVM signatures +└── stackWalker.cpp # Patched: resolveNativeFrameForWalkVM integration at line 454 + +ddprof-lib/src/test/cpp/ +├── remotesymbolication_ut.cpp # Unit tests for remote symbolication +└── remoteargs_ut.cpp # Unit tests for argument parsing + +ddprof-test/src/test/java/ +└── RemoteSymbolicationTest.java # Integration tests for all cstack modes +``` + +## Implementation Notes + +- **Thread Safety**: Build-ID extraction occurs during single-threaded startup +- **Signal Handler Safety**: RemoteFrameInfo uses pre-allocated pool (signal-safe, no dynamic allocation via atomic counters) +- **Error Handling**: Graceful fallback to local symbolication on failures +- **Logging**: Debug logging for build-ID extraction progress and remote symbolication usage +- **ELF Security**: + - Bounds checking for program header table (prevents reading beyond mapped region) + - Alignment verification for program header offset (prevents misaligned pointer access) + - Two-stage validation for note sections (header first, then payload) + - ELFCLASS64 verification ensures uniform 64-bit structure sizes +- **Stack Walker Integration**: + - walkFP/walkDwarf return raw PCs, converted by `convertNativeTrace()` + - walkVM/walkVMX directly call `resolveNativeFrameForWalkVM()` at native frame resolution point + - No post-processing or reverse PC lookup (eliminates broken `applyRemoteSymbolicationToVMFrames` approach) + +This implementation provides a solid foundation for remote symbolication while maintaining full backward compatibility and robust error handling. diff --git a/gradle/patching.gradle b/gradle/patching.gradle index b6aba7e36..534ec29de 100644 --- a/gradle/patching.gradle +++ b/gradle/patching.gradle @@ -188,7 +188,7 @@ ext.upstreamPatches = [ ] ], - // Stack walker patches for ASan compatibility + // Stack walker patches for ASan compatibility and remote symbolication "stackWalker.cpp": [ validations: [[contains: "StackWalker::"], [contains: "StackWalker::walkVM"]], operations: [ @@ -199,6 +199,78 @@ ext.upstreamPatches = [ find: "(int\\s+StackWalker::walkVM\\s*\\()", replace: "__attribute__((no_sanitize(\"address\"))) \$1", idempotent_check: "__attribute__((no_sanitize(\"address\"))) int StackWalker::walkVM(" + ], + [ + type: "signature_parameter_add", + name: "Add lock_index to public walkVM (features) signature", + description: "Adds lock_index parameter to public walkVM overload with features parameter", + find: "(__attribute__\\(\\(no_sanitize\\(\"address\"\\)\\)\\) int StackWalker::walkVM\\(void\\* ucontext, ASGCT_CallFrame\\* frames, int max_depth,\\s*StackWalkFeatures features, EventType event_type)\\)", + replace: "\$1, int lock_index)", + idempotent_check: "EventType event_type, int lock_index\\) \\{" + ], + [ + type: "expression_replace", + name: "Update first walkVM call in public features overload", + description: "Adds lock_index to walkVM call with callerPC/callerSP/callerFP", + find: "(return walkVM\\(&empty_ucontext, frames, max_depth, features, event_type,\\s*callerPC\\(\\), \\(uintptr_t\\)callerSP\\(\\), \\(uintptr_t\\)callerFP\\(\\))\\);", + replace: "\$1, lock_index);", + idempotent_check: "callerFP\\(\\), lock_index\\);" + ], + [ + type: "expression_replace", + name: "Update second walkVM call in public features overload", + description: "Adds lock_index to walkVM call with frame.pc/sp/fp", + find: "(return walkVM\\(ucontext, frames, max_depth, features, event_type,\\s*\\(const void\\*\\)frame\\.pc\\(\\), frame\\.sp\\(\\), frame\\.fp\\(\\))\\);", + replace: "\$1, lock_index);", + idempotent_check: "frame\\.fp\\(\\), lock_index\\);" + ], + [ + type: "signature_parameter_add", + name: "Add lock_index to public walkVM (anchor) signature", + description: "Adds lock_index parameter to public walkVM overload with anchor parameter", + find: "(__attribute__\\(\\(no_sanitize\\(\"address\"\\)\\)\\) int StackWalker::walkVM\\(void\\* ucontext, ASGCT_CallFrame\\* frames, int max_depth, JavaFrameAnchor\\* anchor, EventType event_type)\\)", + replace: "\$1, int lock_index)", + idempotent_check: "EventType event_type, int lock_index\\) \\{" + ], + [ + type: "expression_replace", + name: "Update walkVM call in public anchor overload", + description: "Adds lock_index to walkVM call from anchor overload", + find: "(return walkVM\\(ucontext, frames, max_depth, no_features, event_type, pc, sp, fp)\\);", + replace: "\$1, lock_index);", + idempotent_check: "fp, lock_index\\);" + ], + [ + type: "signature_parameter_add", + name: "Add lock_index to private walkVM implementation signature", + description: "Adds lock_index parameter to private walkVM implementation for remote symbolication pool management", + find: "(__attribute__\\(\\(no_sanitize\\(\"address\"\\)\\)\\) int StackWalker::walkVM\\(void\\* ucontext, ASGCT_CallFrame\\* frames, int max_depth,\\s*StackWalkFeatures features, EventType event_type,\\s*const void\\* pc, uintptr_t sp, uintptr_t fp)\\)", + replace: "\$1, int lock_index)", + idempotent_check: "uintptr_t fp, int lock_index\\)" + ], + [ + type: "expression_replace", + name: "Add remote symbolication support to walkVM native frame resolution", + description: "Replaces direct symbol resolution with resolveNativeFrameForWalkVM call that checks remote symbolication mode", + find: "const char\\* method_name = profiler->findNativeMethod\\(pc\\);", + replace: "// Check if remote symbolication is enabled\n Profiler::NativeFrameResolution resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)pc, lock_index);\n if (resolution.is_marked) {\n // This is a marked C++ interpreter frame, terminate scan\n break;\n }\n const char* method_name = (const char*)resolution.method_id;\n int frame_bci = resolution.bci;", + idempotent_check: "resolveNativeFrameForWalkVM" + ], + [ + type: "expression_replace", + name: "Update fillFrame call to use dynamic BCI", + description: "Updates the fillFrame call to use the dynamic frame_bci value determined by remote symbolication mode", + find: "fillFrame\\(frames\\[depth\\+\\+\\], BCI_NATIVE_FRAME, method_name\\);", + replace: "fillFrame(frames[depth++], frame_bci, (void*)method_name);", + idempotent_check: "frame_bci, \\(void\\*\\)method_name" + ], + [ + type: "method_implementation", + name: "Add fillFrame overload for void* method_id", + description: "Adds fillFrame overload that accepts void* method_id to support both symbol names and RemoteFrameInfo pointers", + find: "(static inline void fillFrame\\(ASGCT_CallFrame& frame, ASGCT_CallFrameType type, const char\\* name\\) \\{[^}]+\\})", + replace: "\$1\n\n// Overload for RemoteFrameInfo* (passed as void* to support both char* and RemoteFrameInfo*)\nstatic inline void fillFrame(ASGCT_CallFrame& frame, int bci, void* method_id_ptr) {\n frame.bci = bci;\n frame.method_id = (jmethodID)method_id_ptr;\n}", + idempotent_check: "void fillFrame\\(ASGCT_CallFrame& frame, int bci, void\\* method_id_ptr\\)" ] ] ], @@ -284,5 +356,36 @@ ext.upstreamPatches = [ idempotent_check: "uintptr_t sender_sp_baseline(" ] ] + ], + + // Stack walker header patches for remote symbolication support + "stackWalker.h": [ + validations: [[contains: "class StackWalker"], [contains: "static int walkVM"]], + operations: [ + [ + type: "signature_parameter_add", + name: "Add lock_index to private walkVM signature", + description: "Adds lock_index parameter to private walkVM implementation for remote symbolication pool management", + find: "(static int walkVM\\(void\\* ucontext, ASGCT_CallFrame\\* frames, int max_depth,\\s*StackWalkFeatures features, EventType event_type,\\s*const void\\* pc, uintptr_t sp, uintptr_t fp)\\);", + replace: "\$1, int lock_index);", + idempotent_check: "int lock_index\\);" + ], + [ + type: "signature_parameter_add", + name: "Add lock_index to public walkVM signature (features overload)", + description: "Adds lock_index parameter to public walkVM features-based overload", + find: "(static int walkVM\\(void\\* ucontext, ASGCT_CallFrame\\* frames, int max_depth, StackWalkFeatures features, EventType event_type)\\);", + replace: "\$1, int lock_index);", + idempotent_check: "EventType event_type, int lock_index\\);" + ], + [ + type: "signature_parameter_add", + name: "Add lock_index to public walkVM signature (anchor overload)", + description: "Adds lock_index parameter to public walkVM anchor-based overload", + find: "(static int walkVM\\(void\\* ucontext, ASGCT_CallFrame\\* frames, int max_depth, JavaFrameAnchor\\* anchor, EventType event_type)\\);", + replace: "\$1, int lock_index);", + idempotent_check: "JavaFrameAnchor\\* anchor, EventType event_type, int lock_index\\);" + ] + ] ] ] \ No newline at end of file