From 74158e5a8e0cd499950503be9a6ee457df1fc318 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 9 Dec 2025 15:06:32 -0500 Subject: [PATCH 01/16] Instrument native threads --- ddprof-lib/src/main/cpp/codeCache.h | 6 +- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 141 ++++++++++++------ ddprof-lib/src/main/cpp/libraries.h | 4 + ddprof-lib/src/main/cpp/utils.h | 4 +- ddprof-test/build.gradle | 56 ++++++- ddprof-test/src/test/cpp/nativethread.c | 68 +++++++++ .../nativethread/NativeThreadTest.java | 74 +++++++++ 7 files changed, 300 insertions(+), 53 deletions(-) create mode 100644 ddprof-test/src/test/cpp/nativethread.c create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java diff --git a/ddprof-lib/src/main/cpp/codeCache.h b/ddprof-lib/src/main/cpp/codeCache.h index 3398ad029..feddaeac4 100644 --- a/ddprof-lib/src/main/cpp/codeCache.h +++ b/ddprof-lib/src/main/cpp/codeCache.h @@ -226,7 +226,7 @@ class CodeCacheArray { CodeCache *operator[](int index) { return _libs[index]; } - int count() { return __atomic_load_n(&_count, __ATOMIC_ACQUIRE); } + int count() const { return __atomic_load_n(&_count, __ATOMIC_ACQUIRE); } void add(CodeCache *lib) { int index = __atomic_load_n(&_count, __ATOMIC_ACQUIRE); @@ -234,6 +234,10 @@ class CodeCacheArray { __atomic_store_n(&_count, index + 1, __ATOMIC_RELEASE); } + CodeCache* at(int index) const { + return _libs[index]; + } + long long memoryUsage() { int count = __atomic_load_n(&_count, __ATOMIC_ACQUIRE); long long totalUsage = 0; diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 9d04ad873..6a1fec1d3 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -23,6 +23,7 @@ #include "profiler.h" #include "vmStructs.h" #include +#include #include #include #include @@ -32,51 +33,105 @@ #define SIGEV_THREAD_ID 4 #endif -static inline clockid_t thread_cpu_clock(unsigned int tid) { - return ((~tid) << 3) | 6; // CPUCLOCK_SCHED | CPUCLOCK_PERTHREAD_MASK -} +typedef void* (*func_start_routine)(void*); +typedef int (*func_pthread_create)(pthread_t* thread, + const pthread_attr_t* attr, + func_start_routine start_routine, + void* arg); -static void **_pthread_entry = NULL; +// Patching libraries' pthread_create() @plt entries +typedef struct _patchEntry { + // library's @plt location + func_pthread_create* _location; + // original function + func_pthread_create _func; +} PatchEntry; -// Intercept thread creation/termination by patching libjvm's GOT entry for -// pthread_setspecific(). HotSpot puts VMThread into TLS on thread start, and -// resets on thread end. -static int pthread_setspecific_hook(pthread_key_t key, const void *value) { - if (key != static_cast(VMThread::key())) { - return pthread_setspecific(key, value); - } - if (pthread_getspecific(key) == value) { - return 0; - } +static PatchEntry* patched_entries = nullptr; +static volatile int num_of_entries = 0; + +typedef struct _startRoutineArg { + func_start_routine _func; + void* _arg; +} StartRoutineArg; - if (value != NULL) { +static void* start_routine_wrapper(void* args) { + StartRoutineArg* data = (StartRoutineArg*)args; ProfiledThread::initCurrentThread(); - int result = pthread_setspecific(key, value); - Profiler::registerThread(ProfiledThread::currentTid()); - return result; - } else { int tid = ProfiledThread::currentTid(); + Profiler::registerThread(tid); + void* result = data->_func(data->_arg); Profiler::unregisterThread(tid); ProfiledThread::release(); - return pthread_setspecific(key, value); + free(args); + return result; +} + +static int pthread_create_hook(pthread_t* thread, + const pthread_attr_t* attr, + func_start_routine start_routine, + void* arg) { + StartRoutineArg* data = (StartRoutineArg*)malloc(sizeof(StartRoutineArg)); + data->_func = start_routine; + data->_arg = arg; + return pthread_create(thread, attr, start_routine_wrapper, (void*)data); +} + +static Error patch_libraries() { + Dl_info info; + void* caller_address = __builtin_return_address(0); // Get return address of caller + + if (!dladdr(caller_address, &info)) { + return Error("Cannot resolve current library name"); + } + TEST_LOG("Profiler library name: %s\n", info.dli_fname ); + + const CodeCacheArray& native_libs = Libraries::instance()->native_libs(); + int count = native_libs.count(); + size_t size = count * sizeof(PatchEntry); + patched_entries = (PatchEntry*)malloc(size); + memset((void*)patched_entries, 0, size); + TEST_LOG("Patching libraries\n"); + + for (int index = 0; index < count; index++) { + CodeCache* lib = native_libs.at(index); + // Don't patch self + if (strcmp(lib->name(), info.dli_fname) == 0) { + continue; + } + + func_pthread_create* pthread_create_addr = (func_pthread_create*)lib->findImport(im_pthread_create); + if (pthread_create_addr != nullptr) { + TEST_LOG("Patching %s\n", lib->name()); + + patched_entries[index]._location = pthread_create_addr; + patched_entries[index]._func = (func_pthread_create)__atomic_load_n(pthread_create_addr, __ATOMIC_RELAXED); + __atomic_store_n(pthread_create_addr, (func_pthread_create)pthread_create_hook, __ATOMIC_RELAXED); + } } + // Publish everything, including patched entries + __atomic_store_n(&num_of_entries, count, __ATOMIC_SEQ_CST); + return Error::OK; } -static void **lookupThreadEntry() { - // Depending on Zing version, pthread_setspecific is called either from - // libazsys.so or from libjvm.so - if (VM::isZing()) { - CodeCache *libazsys = Libraries::instance()->findLibraryByName("libazsys"); - if (libazsys != NULL) { - void **entry = libazsys->findImport(im_pthread_setspecific); - if (entry != NULL) { - return entry; - } - } +static void unpatch_libraries() { + int count = __atomic_load_n(&num_of_entries, __ATOMIC_RELAXED); + PatchEntry* tmp = patched_entries; + patched_entries = nullptr; + __atomic_store_n(&entry_count, 0, __ATOMIC_SEQ_CST); + + for (int index = 0; index < count; index++) { + if (tmp[index]._location != nullptr) { + __atomic_store_n(tmp[index]._location, tmp[index]._func, __ATOMIC_RELAXED); + } } + __atomic_thread_fence(__ATOMIC_SEQ_CST); + free((void*)tmp); +} + - CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); - return lib != NULL ? lib->findImport(im_pthread_setspecific) : NULL; +static inline clockid_t thread_cpu_clock(unsigned int tid) { + return ((~tid) << 3) | 6; // CPUCLOCK_SCHED | CPUCLOCK_PERTHREAD_MASK } long CTimer::_interval; @@ -135,11 +190,6 @@ void CTimer::unregisterThread(int tid) { } Error CTimer::check(Arguments &args) { - if (_pthread_entry == NULL && - (_pthread_entry = lookupThreadEntry()) == NULL) { - return Error("Could not set pthread hook"); - } - timer_t timer; if (timer_create(CLOCK_THREAD_CPUTIME_ID, NULL, &timer) < 0) { return Error("Failed to create CPU timer"); @@ -153,10 +203,7 @@ Error CTimer::start(Arguments &args) { if (args._interval < 0) { return Error("interval must be positive"); } - if (_pthread_entry == NULL && - (_pthread_entry = lookupThreadEntry()) == NULL) { - return Error("Could not set pthread hook"); - } + _interval = args.cpuSamplerInterval(); _cstack = args._cstack; _signal = SIGPROF; @@ -170,9 +217,10 @@ Error CTimer::start(Arguments &args) { OS::installSignalHandler(_signal, signalHandler); - // Enable pthread hook before traversing currently running threads - __atomic_store_n(_pthread_entry, (void *)pthread_setspecific_hook, - __ATOMIC_RELEASE); + Error err = patch_libraries(); + if (err) { + return err; + } // Register all existing threads Error result = Error::OK; @@ -190,8 +238,7 @@ Error CTimer::start(Arguments &args) { } void CTimer::stop() { - __atomic_store_n(_pthread_entry, (void *)pthread_setspecific, - __ATOMIC_RELEASE); + unpatch_libraries(); for (int i = 0; i < _max_timers; i++) { unregisterThread(i); } diff --git a/ddprof-lib/src/main/cpp/libraries.h b/ddprof-lib/src/main/cpp/libraries.h index b2422c6a1..4c4aa132a 100644 --- a/ddprof-lib/src/main/cpp/libraries.h +++ b/ddprof-lib/src/main/cpp/libraries.h @@ -24,6 +24,10 @@ class Libraries { return &instance; } + const CodeCacheArray& native_libs() const { + return _native_libs; + } + // Delete copy constructor and assignment operator to prevent copies Libraries(const Libraries&) = delete; Libraries& operator=(const Libraries&) = delete; diff --git a/ddprof-lib/src/main/cpp/utils.h b/ddprof-lib/src/main/cpp/utils.h index 47d58eb96..8e8bf2090 100644 --- a/ddprof-lib/src/main/cpp/utils.h +++ b/ddprof-lib/src/main/cpp/utils.h @@ -16,12 +16,12 @@ inline bool is_aligned(const T* ptr, size_t alignment) noexcept { auto iptr = reinterpret_cast(ptr); // Check if the integer value is a multiple of the alignment - return (iptr & ~(alignment - 1) == 0); + return ((iptr & (~(alignment - 1))) == 0); } inline size_t align_down(size_t size, size_t alignment) noexcept { assert(is_power_of_2(alignment)); - return size & ~(alignment - 1); + return size & (~(alignment - 1)); } inline size_t align_up(size_t size, size_t alignment) noexcept { diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index b6ac0d579..af76ed2d5 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java' + id 'java-library' id 'application' } @@ -7,8 +8,55 @@ repositories { mavenCentral() } +// 1. Define paths and properties +def nativeSrcDir = file('src/test/cpp') +def jniHeadersDir = layout.buildDirectory.dir("generated/jni-headers").get().asFile +def outputLibDir = layout.buildDirectory.dir("libs/native").get().asFile +// Define the name of your JNI library (e.g., "ddproftest" becomes libddproftest.so/ddproftest.dll/libddproftest.dylib) +def libraryName = "ddproftest" + +// Determine OS-specific file extensions and library names +def osName = org.gradle.internal.os.OperatingSystem.current() +def libFileExtension = (os().isMacOsX() ? "dylib" : "so") +def libraryFileName = "lib${libraryName}.${libFileExtension}" + +// 2. Generate JNI headers using javac +tasks.named('compileJava') { + // Tell javac to generate the JNI headers into the specified directory + options.compilerArgs += ['-h', jniHeadersDir] +} + +// 3. Define a task to compile the native code +tasks.register('buildNativeJniLibrary', Exec) { + description 'Compiles the JNI C/C++ sources into a shared library' + group 'build' + + // Ensure Java compilation (and thus header generation) happens first + dependsOn tasks.named('compileJava') + + // Clean up previous build artifacts + doFirst { + outputLibDir.mkdirs() + } + + // Assume GCC/Clang on Linux/macOS + commandLine 'gcc' + args "-I${System.getenv('JAVA_HOME')}/include" // Standard JNI includes + if (os().isMacOsX()) { + args "-I${System.getenv('JAVA_HOME')}/include/darwin" // macOS-specific includes + args "-dynamiclib" // Build a dynamic library on macOS + } else if (os().isLinux()) { + args "-I${System.getenv('JAVA_HOME')}/include/linux" // Linux-specific includes + args "-fPIC" + args "-shared" // Build a shared library on Linux + } + args nativeSrcDir.listFiles()*.getAbsolutePath() // Source files + args "-o", "${outputLibDir.absolutePath}/${libraryFileName}" // Output file path +} + apply from: rootProject.file('common.gradle') + def addCommonTestDependencies(Configuration configuration) { configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-api:5.9.2')) configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-engine:5.9.2')) @@ -130,7 +178,7 @@ buildConfigurations.each { config -> jvmArgs '-Djdk.attach.allowAttachSelf', '-Djol.tryWithSudo=true', '-XX:ErrorFile=build/hs_err_pid%p.log', '-XX:+ResizeTLAB', - '-Xmx512m' + '-Xmx1G' } // Configure arguments for runUnwindingValidator task @@ -217,6 +265,8 @@ task unwindingReport { } tasks.withType(Test).configureEach { + dependsOn tasks.named('buildNativeJniLibrary') + // this is a shared configuration for all test tasks onlyIf { !project.hasProperty('skip-tests') @@ -229,7 +279,7 @@ tasks.withType(Test).configureEach { jvmArgs "-Dddprof_test.keep_jfrs=${keepRecordings}", '-Djdk.attach.allowAttachSelf', '-Djol.tryWithSudo=true', "-Dddprof_test.config=${config}", "-Dddprof_test.ci=${project.hasProperty('CI')}", "-Dddprof.disable_unsafe=true", '-XX:ErrorFile=build/hs_err_pid%p.log', '-XX:+ResizeTLAB', - '-Xmx512m', '-XX:OnError=/tmp/do_stuff.sh' + '-Xmx512m', '-XX:OnError=/tmp/do_stuff.sh', "-Djava.library.path=${outputLibDir.absolutePath}" def javaHome = System.getenv("JAVA_TEST_HOME") if (javaHome == null) { @@ -274,4 +324,4 @@ gradle.projectsEvaluated { testTask.dependsOn gtestTask } } -} \ No newline at end of file +} diff --git a/ddprof-test/src/test/cpp/nativethread.c b/ddprof-test/src/test/cpp/nativethread.c new file mode 100644 index 000000000..aef3df54b --- /dev/null +++ b/ddprof-test/src/test/cpp/nativethread.c @@ -0,0 +1,68 @@ +/* + * Copyright 2025, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#define MAX_PRIME 100000 + +// Burn CPU +static void do_primes() { + unsigned long i, num, primes = 0; + for (num = 1; num <= MAX_PRIME; ++num) { + for (i = 2; (i <= num) && (num % i != 0); ++i); + if (i == num) + ++primes; + } +} + +// Function to be executed by the new thread +void* thread_function(void* arg) { + do_primes(); + pthread_exit(NULL); // Terminate the thread, optionally returning a value +} + +jlong JNICALL Java_com_datadoghq_profiler_nativethread_NativeThreadTest_createNativeThread + (JNIEnv * env, jclass clz) { + + // Create a new thread + // Arguments: + // 1. &thread_id: Pointer to the pthread_t variable where the new thread's ID will be stored. + // 2. NULL: Pointer to thread attributes (using default attributes here). + // 3. thread_function: Pointer to the function the new thread will execute. + // 4. NULL: Pointer to the argument to pass to the thread_function. + pthread_t thread_id; + int result = pthread_create(&thread_id, NULL, thread_function, NULL); + + if (result != 0) { + perror("Error creating thread"); + return -1L; + } + + return (jlong) thread_id; +} + +void JNICALL Java_com_datadoghq_profiler_nativethread_NativeThreadTest_waitNativeThread + (JNIEnv * env, jclass clz, jlong threadId) { + pthread_t thread_id = (pthread_t)threadId; + // Wait for the created thread to finish + // Arguments: + // 1. thread_id: The ID of the thread to wait for. + // 2. NULL: Pointer to store the return value of the joined thread (not used here). + pthread_join(thread_id, NULL); +} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java new file mode 100644 index 000000000..ac71912d4 --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2025, Datadog, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datadoghq.profiler.nativethread; + +import com.datadoghq.profiler.AbstractProfilerTest; +import org.junitpioneer.jupiter.RetryingTest; +import org.openjdk.jmc.common.item.IItem; +import org.openjdk.jmc.common.item.IItemIterable; +import org.openjdk.jmc.common.item.IMemberAccessor; +import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; + +import java.util.HashMap; +import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +public class NativeThreadTest extends AbstractProfilerTest { + + static { + System.loadLibrary("ddproftest"); + } + + @Override + protected String getProfilerCommand() { + return "cpu=1ms"; + } + + @RetryingTest(3) + public void test() { + long[] threads = new long[8]; + for (int index = 0; index < threads.length; index++) { + threads[index] = createNativeThread(); + } + + for (int index = 0; index < threads.length; index++) { + waitNativeThread(threads[index]); + } + stopProfiler(); + int count = 0; + boolean stacktrace_printed = false; + for (IItemIterable cpuSamples : verifyEvents("datadog.ExecutionSample")) { + IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); + IMemberAccessor modeAccessor = THREAD_EXECUTION_MODE.getAccessor(cpuSamples.getType()); + for (IItem item : cpuSamples) { + String stacktrace = stacktraceAccessor.getMember(item); + if (stacktrace.indexOf("do_primes()") != -1) { + if (!stacktrace_printed) { + stacktrace_printed = true; + System.out.println("Native thread stack:"); + System.out.println(stacktrace); + } + count++; + } + } + } + assertTrue(count > 0, "no native thread sample"); + } + + private static native long createNativeThread(); + private static native void waitNativeThread(long threadId); +} From 07e480d5e63d7b9f607feec3915b96886dbc7be6 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 9 Dec 2025 15:25:53 -0500 Subject: [PATCH 02/16] Fix --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 6a1fec1d3..3ccd2fae1 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -118,7 +118,7 @@ static void unpatch_libraries() { int count = __atomic_load_n(&num_of_entries, __ATOMIC_RELAXED); PatchEntry* tmp = patched_entries; patched_entries = nullptr; - __atomic_store_n(&entry_count, 0, __ATOMIC_SEQ_CST); + __atomic_store_n(&num_of_entries, 0, __ATOMIC_SEQ_CST); for (int index = 0; index < count; index++) { if (tmp[index]._location != nullptr) { From db6afef44aff8d1a6b85705db78489c26c46b0ba Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 9 Dec 2025 20:15:13 -0500 Subject: [PATCH 03/16] J9 stack walk --- .../com/datadoghq/profiler/nativethread/NativeThreadTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java index ac71912d4..1e13407be 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java @@ -35,7 +35,7 @@ public class NativeThreadTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - return "cpu=1ms"; + return "cpu=1ms,cstack=fp"; } @RetryingTest(3) From ba406616fb9cd2f201fef6653bd32c32a33555c8 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Wed, 10 Dec 2025 08:35:58 -0500 Subject: [PATCH 04/16] Cleanup --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 3ccd2fae1..4dd317cc2 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -84,16 +84,16 @@ static Error patch_libraries() { if (!dladdr(caller_address, &info)) { return Error("Cannot resolve current library name"); } - TEST_LOG("Profiler library name: %s\n", info.dli_fname ); + TEST_LOG("Profiler library name: %s", info.dli_fname ); const CodeCacheArray& native_libs = Libraries::instance()->native_libs(); - int count = native_libs.count(); + int count = 0; size_t size = count * sizeof(PatchEntry); patched_entries = (PatchEntry*)malloc(size); memset((void*)patched_entries, 0, size); - TEST_LOG("Patching libraries\n"); + TEST_LOG("Patching libraries"); - for (int index = 0; index < count; index++) { + for (int index = 0; index < native_libs.count(); index++) { CodeCache* lib = native_libs.at(index); // Don't patch self if (strcmp(lib->name(), info.dli_fname) == 0) { @@ -102,11 +102,12 @@ static Error patch_libraries() { func_pthread_create* pthread_create_addr = (func_pthread_create*)lib->findImport(im_pthread_create); if (pthread_create_addr != nullptr) { - TEST_LOG("Patching %s\n", lib->name()); + TEST_LOG("Patching %s", lib->name()); - patched_entries[index]._location = pthread_create_addr; - patched_entries[index]._func = (func_pthread_create)__atomic_load_n(pthread_create_addr, __ATOMIC_RELAXED); + patched_entries[count]._location = pthread_create_addr; + patched_entries[count]._func = (func_pthread_create)__atomic_load_n(pthread_create_addr, __ATOMIC_RELAXED); __atomic_store_n(pthread_create_addr, (func_pthread_create)pthread_create_hook, __ATOMIC_RELAXED); + count++; } } // Publish everything, including patched entries From 3ac62231b5775fedc7d93c55f779ebba4606d27d Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Wed, 10 Dec 2025 09:11:13 -0500 Subject: [PATCH 05/16] Fix bug --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 4dd317cc2..f2e629434 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -88,12 +88,13 @@ static Error patch_libraries() { const CodeCacheArray& native_libs = Libraries::instance()->native_libs(); int count = 0; - size_t size = count * sizeof(PatchEntry); + int num_of_libs = native_libs.count(); + size_t size = num_of_libs * sizeof(PatchEntry); patched_entries = (PatchEntry*)malloc(size); memset((void*)patched_entries, 0, size); TEST_LOG("Patching libraries"); - for (int index = 0; index < native_libs.count(); index++) { + for (int index = 0; index < num_of_libs; index++) { CodeCache* lib = native_libs.at(index); // Don't patch self if (strcmp(lib->name(), info.dli_fname) == 0) { From 0bdded8456d4068e3b40566c997e5f4e82e38e0c Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Wed, 10 Dec 2025 20:18:57 -0500 Subject: [PATCH 06/16] Set cstack=dwarf for NativeThreadTest (J9) --- .../com/datadoghq/profiler/nativethread/NativeThreadTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java index 1e13407be..b7118d446 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java @@ -35,7 +35,7 @@ public class NativeThreadTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - return "cpu=1ms,cstack=fp"; + return "cpu=1ms,cstack=dwarf"; } @RetryingTest(3) From 48354c241ed63f3a48491638c25b884904f090ce Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 08:38:58 -0500 Subject: [PATCH 07/16] Hotspot/Zing only --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 62 +++++++++++++++++-- .../nativethread/NativeThreadTest.java | 2 +- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index f2e629434..d0eeedbcb 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -33,6 +33,8 @@ #define SIGEV_THREAD_ID 4 #endif +typedef int (*func_pthread_setspecific)(pthread_key_t key, const void *value); + typedef void* (*func_start_routine)(void*); typedef int (*func_pthread_create)(pthread_t* thread, const pthread_attr_t* attr, @@ -42,9 +44,9 @@ typedef int (*func_pthread_create)(pthread_t* thread, // Patching libraries' pthread_create() @plt entries typedef struct _patchEntry { // library's @plt location - func_pthread_create* _location; + void** _location; // original function - func_pthread_create _func; + void* _func; } PatchEntry; static PatchEntry* patched_entries = nullptr; @@ -78,6 +80,14 @@ static int pthread_create_hook(pthread_t* thread, } static Error patch_libraries() { + if (VM::isHotspot() || VM::isZing()) { + return patch_libraries_for_hotspot_and_zing(); + } else { + return patch_libraries_for_J9(); + } +} + +static Error patch_libraries_for_hotspot_and_zing() { Dl_info info; void* caller_address = __builtin_return_address(0); // Get return address of caller @@ -101,13 +111,13 @@ static Error patch_libraries() { continue; } - func_pthread_create* pthread_create_addr = (func_pthread_create*)lib->findImport(im_pthread_create); + void** pthread_create_location = (void*)lib->findImport(im_pthread_create); if (pthread_create_addr != nullptr) { TEST_LOG("Patching %s", lib->name()); - patched_entries[count]._location = pthread_create_addr; - patched_entries[count]._func = (func_pthread_create)__atomic_load_n(pthread_create_addr, __ATOMIC_RELAXED); - __atomic_store_n(pthread_create_addr, (func_pthread_create)pthread_create_hook, __ATOMIC_RELAXED); + patched_entries[count]._location = pthread_create_location; + patched_entries[count]._func = (void*)__atomic_load_n(pthread_create_location, __ATOMIC_RELAXED); + __atomic_store_n(pthread_create_addr, (void*)pthread_create_hook, __ATOMIC_RELAXED); count++; } } @@ -116,6 +126,46 @@ static Error patch_libraries() { return Error::OK; } +static Error patch_libraries_for_J9() { + CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); + return Error("Cannot find J9 library to patch"); + void** func_location = lib->findImport(im_pthread_setspecific); + + patched_entries = (PatchEntry*)malloc(sizeof(PatchEntry)); + patched_entries[0]._location = func_location; + patched_entries[0]._func = (void*)__atomic_load_n(func_location, __ATOMIC_RELAXED); + __atomic_store_n(func_location, (void*)pthread_setspecific_hook, __ATOMIC_RELAXED); + + // Publish everything, including patched entries + __atomic_store_n(&num_of_entries, 1, __ATOMIC_SEQ_CST); + return Error::OK; +} + +// Intercept thread creation/termination by patching libjvm's GOT entry for +// pthread_setspecific(). HotSpot puts VMThread into TLS on thread start, and +// resets on thread end. +static int pthread_setspecific_hook(pthread_key_t key, const void *value) { + if (key != static_cast(VMThread::key())) { + return pthread_setspecific(key, value); + } + if (pthread_getspecific(key) == value) { + return 0; + } + + if (value != NULL) { + ProfiledThread::initCurrentThread(); + int result = pthread_setspecific(key, value); + Profiler::registerThread(ProfiledThread::currentTid()); + return result; + } else { + int tid = ProfiledThread::currentTid(); + Profiler::unregisterThread(tid); + ProfiledThread::release(); + return pthread_setspecific(key, value); + } +} + + static void unpatch_libraries() { int count = __atomic_load_n(&num_of_entries, __ATOMIC_RELAXED); PatchEntry* tmp = patched_entries; diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java index b7118d446..ac71912d4 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java @@ -35,7 +35,7 @@ public class NativeThreadTest extends AbstractProfilerTest { @Override protected String getProfilerCommand() { - return "cpu=1ms,cstack=dwarf"; + return "cpu=1ms"; } @RetryingTest(3) From 617ab9a68891d2a35af082943bbe37c95fdc29ac Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 08:47:39 -0500 Subject: [PATCH 08/16] Fix --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 49 ++++++++++++------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index d0eeedbcb..c8b4c564a 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -79,15 +79,7 @@ static int pthread_create_hook(pthread_t* thread, return pthread_create(thread, attr, start_routine_wrapper, (void*)data); } -static Error patch_libraries() { - if (VM::isHotspot() || VM::isZing()) { - return patch_libraries_for_hotspot_and_zing(); - } else { - return patch_libraries_for_J9(); - } -} - -static Error patch_libraries_for_hotspot_and_zing() { +static Error patch_libraries_for_hotspot_or_zing() { Dl_info info; void* caller_address = __builtin_return_address(0); // Get return address of caller @@ -111,7 +103,7 @@ static Error patch_libraries_for_hotspot_and_zing() { continue; } - void** pthread_create_location = (void*)lib->findImport(im_pthread_create); + void** pthread_create_location = (void**)lib->findImport(im_pthread_create); if (pthread_create_addr != nullptr) { TEST_LOG("Patching %s", lib->name()); @@ -126,21 +118,6 @@ static Error patch_libraries_for_hotspot_and_zing() { return Error::OK; } -static Error patch_libraries_for_J9() { - CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); - return Error("Cannot find J9 library to patch"); - void** func_location = lib->findImport(im_pthread_setspecific); - - patched_entries = (PatchEntry*)malloc(sizeof(PatchEntry)); - patched_entries[0]._location = func_location; - patched_entries[0]._func = (void*)__atomic_load_n(func_location, __ATOMIC_RELAXED); - __atomic_store_n(func_location, (void*)pthread_setspecific_hook, __ATOMIC_RELAXED); - - // Publish everything, including patched entries - __atomic_store_n(&num_of_entries, 1, __ATOMIC_SEQ_CST); - return Error::OK; -} - // Intercept thread creation/termination by patching libjvm's GOT entry for // pthread_setspecific(). HotSpot puts VMThread into TLS on thread start, and // resets on thread end. @@ -165,6 +142,28 @@ static int pthread_setspecific_hook(pthread_key_t key, const void *value) { } } +static Error patch_libraries_for_J9() { + CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); + return Error("Cannot find J9 library to patch"); + void** func_location = lib->findImport(im_pthread_setspecific); + + patched_entries = (PatchEntry*)malloc(sizeof(PatchEntry)); + patched_entries[0]._location = func_location; + patched_entries[0]._func = (void*)__atomic_load_n(func_location, __ATOMIC_RELAXED); + __atomic_store_n(func_location, (void*)pthread_setspecific_hook, __ATOMIC_RELAXED); + + // Publish everything, including patched entries + __atomic_store_n(&num_of_entries, 1, __ATOMIC_SEQ_CST); + return Error::OK; +} + +static Error patch_libraries() { + if (VM::isHotspot() || VM::isZing()) { + return patch_libraries_for_hotspot_or_zing(); + } else { + return patch_libraries_for_J9(); + } +} static void unpatch_libraries() { int count = __atomic_load_n(&num_of_entries, __ATOMIC_RELAXED); From b445304df191b6f062c87bef9939bd5328c21f8e Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 08:48:51 -0500 Subject: [PATCH 09/16] More fix --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index c8b4c564a..8ce34c3a6 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -104,12 +104,12 @@ static Error patch_libraries_for_hotspot_or_zing() { } void** pthread_create_location = (void**)lib->findImport(im_pthread_create); - if (pthread_create_addr != nullptr) { + if (pthread_create_location != nullptr) { TEST_LOG("Patching %s", lib->name()); patched_entries[count]._location = pthread_create_location; patched_entries[count]._func = (void*)__atomic_load_n(pthread_create_location, __ATOMIC_RELAXED); - __atomic_store_n(pthread_create_addr, (void*)pthread_create_hook, __ATOMIC_RELAXED); + __atomic_store_n(pthread_create_location, (void*)pthread_create_hook, __ATOMIC_RELAXED); count++; } } From 77aaedbba04cc789c646942a6a7ff9198f794bf9 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 08:54:52 -0500 Subject: [PATCH 10/16] Exclude J9 from NativeThreadTest --- .../datadoghq/profiler/nativethread/NativeThreadTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java index ac71912d4..4a99e3f42 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java @@ -16,6 +16,8 @@ package com.datadoghq.profiler.nativethread; import com.datadoghq.profiler.AbstractProfilerTest; +import com.datadoghq.profiler.Platform; + import org.junitpioneer.jupiter.RetryingTest; import org.openjdk.jmc.common.item.IItem; import org.openjdk.jmc.common.item.IItemIterable; @@ -40,6 +42,10 @@ protected String getProfilerCommand() { @RetryingTest(3) public void test() { + // Exclude J9 for now + if (Platform.isJ9()) { + return; + } long[] threads = new long[8]; for (int index = 0; index < threads.length; index++) { threads[index] = createNativeThread(); From 60d835009b39bf735b8d921ff533e2f336e28cfa Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 09:17:31 -0500 Subject: [PATCH 11/16] Fix --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 8ce34c3a6..ec72a4100 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -144,16 +144,19 @@ static int pthread_setspecific_hook(pthread_key_t key, const void *value) { static Error patch_libraries_for_J9() { CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); - return Error("Cannot find J9 library to patch"); + if (lib == nullptr) { + return Error("Cannot find J9 library to patch"); + } void** func_location = lib->findImport(im_pthread_setspecific); - - patched_entries = (PatchEntry*)malloc(sizeof(PatchEntry)); - patched_entries[0]._location = func_location; - patched_entries[0]._func = (void*)__atomic_load_n(func_location, __ATOMIC_RELAXED); - __atomic_store_n(func_location, (void*)pthread_setspecific_hook, __ATOMIC_RELAXED); - - // Publish everything, including patched entries - __atomic_store_n(&num_of_entries, 1, __ATOMIC_SEQ_CST); + if (func_location != nullptr) { + patched_entries = (PatchEntry*)malloc(sizeof(PatchEntry)); + patched_entries[0]._location = func_location; + patched_entries[0]._func = (void*)__atomic_load_n(func_location, __ATOMIC_RELAXED); + __atomic_store_n(func_location, (void*)pthread_setspecific_hook, __ATOMIC_RELAXED); + + // Publish everything, including patched entries + __atomic_store_n(&num_of_entries, 1, __ATOMIC_SEQ_CST); + } return Error::OK; } @@ -177,7 +180,9 @@ static void unpatch_libraries() { } } __atomic_thread_fence(__ATOMIC_SEQ_CST); - free((void*)tmp); + if (tmp != nullptr) { + free((void*)tmp); + } } From 8cdabce30f6a9413ce05c1a0a33436c94a1b0a9c Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 09:42:00 -0500 Subject: [PATCH 12/16] Handle musl --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index ec72a4100..64f899691 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -142,10 +142,10 @@ static int pthread_setspecific_hook(pthread_key_t key, const void *value) { } } -static Error patch_libraries_for_J9() { +static Error patch_libraries_for_J9_or_musl() { CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); if (lib == nullptr) { - return Error("Cannot find J9 library to patch"); + lib = VMStructs::libjvm(); } void** func_location = lib->findImport(im_pthread_setspecific); if (func_location != nullptr) { @@ -161,10 +161,10 @@ static Error patch_libraries_for_J9() { } static Error patch_libraries() { - if (VM::isHotspot() || VM::isZing()) { + if ((VM::isHotspot() || VM::isZing()) && !OS::isMusl()) { return patch_libraries_for_hotspot_or_zing(); } else { - return patch_libraries_for_J9(); + return patch_libraries_for_J9_or_musl(); } } From 36fe9b122ab7d0beb1839e813725b4842fe309de Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 10:02:44 -0500 Subject: [PATCH 13/16] Exclude Musl from NativeThreadTest --- .../com/datadoghq/profiler/nativethread/NativeThreadTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java index 4a99e3f42..5ab8b3df2 100644 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java @@ -43,7 +43,7 @@ protected String getProfilerCommand() { @RetryingTest(3) public void test() { // Exclude J9 for now - if (Platform.isJ9()) { + if (Platform.isJ9() || Platform.isMusl()) { return; } long[] threads = new long[8]; From 5d46345b32cb44f24bc2729df52fd60c35df51b0 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 11 Dec 2025 11:30:14 -0500 Subject: [PATCH 14/16] cleanup --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 64f899691..a857cea36 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -33,15 +33,9 @@ #define SIGEV_THREAD_ID 4 #endif -typedef int (*func_pthread_setspecific)(pthread_key_t key, const void *value); - typedef void* (*func_start_routine)(void*); -typedef int (*func_pthread_create)(pthread_t* thread, - const pthread_attr_t* attr, - func_start_routine start_routine, - void* arg); -// Patching libraries' pthread_create() @plt entries +// Patch libraries' @plt entries typedef struct _patchEntry { // library's @plt location void** _location; From 7774a2c126166042ecccb054c48e343ece9cb893 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 12 Dec 2025 08:25:15 -0500 Subject: [PATCH 15/16] Remove duplicated check --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index a857cea36..a4c9e35a3 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -138,9 +138,6 @@ static int pthread_setspecific_hook(pthread_key_t key, const void *value) { static Error patch_libraries_for_J9_or_musl() { CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); - if (lib == nullptr) { - lib = VMStructs::libjvm(); - } void** func_location = lib->findImport(im_pthread_setspecific); if (func_location != nullptr) { patched_entries = (PatchEntry*)malloc(sizeof(PatchEntry)); From d76aba980057798b103af096377237be62f50999 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 15 Dec 2025 08:28:56 -0500 Subject: [PATCH 16/16] Restore test parameter --- ddprof-test/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle index af76ed2d5..abbfecdd7 100644 --- a/ddprof-test/build.gradle +++ b/ddprof-test/build.gradle @@ -178,7 +178,7 @@ buildConfigurations.each { config -> jvmArgs '-Djdk.attach.allowAttachSelf', '-Djol.tryWithSudo=true', '-XX:ErrorFile=build/hs_err_pid%p.log', '-XX:+ResizeTLAB', - '-Xmx1G' + '-Xmx512m' } // Configure arguments for runUnwindingValidator task