From 33dbbe4d497ef5d0499459edb444d256878142b4 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Thu, 27 Jan 2022 21:08:25 +0000 Subject: [PATCH 01/22] Replace "NULL" and "0" with "nullptr". PiperOrigin-RevId: 424689000 --- .github/workflows/ci.yml | 25 ------------------------- src/entry.cc | 6 +++--- src/profiler.cc | 4 ++-- 3 files changed, 5 insertions(+), 30 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 79cbc290b..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: CI - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - build-amd64: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - run: ./build.sh -m amd64 - - build-alpine: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - run: ./build.sh -m alpine diff --git a/src/entry.cc b/src/entry.cc index 24c20fd6f..f71c32a18 100644 --- a/src/entry.cc +++ b/src/entry.cc @@ -100,7 +100,7 @@ void CreateJMethodIDsForClass(jvmtiEnv *jvmti, jclass klass) { // JVMTI_ERROR_CLASS_NOT_PREPARED is okay because some classes may // be loaded but not prepared at this point. google::javaprofiler::JvmtiScopedPtr ksig(jvmti); - JVMTI_ERROR((jvmti->GetClassSignature(klass, ksig.GetRef(), NULL))); + JVMTI_ERROR((jvmti->GetClassSignature(klass, ksig.GetRef(), nullptr))); LOG(ERROR) << "Failed to create method IDs for methods in class " << ksig.Get() << " with error " << e; } @@ -146,7 +146,7 @@ void JNICALL OnVMDeath(jvmtiEnv *jvmti_env, JNIEnv *jni_env) { LOG(INFO) << "On VM death"; worker->Stop(); delete worker; - worker = NULL; + worker = nullptr; if (google::javaprofiler::HeapMonitor::Enabled()) { google::javaprofiler::HeapMonitor::Disable(); @@ -231,7 +231,7 @@ static bool RegisterJvmti(jvmtiEnv *jvmti) { // Events are enumerated in jvmstatagent.h for (int i = 0; i < events.size(); i++) { JVMTI_ERROR_1( - (jvmti->SetEventNotificationMode(JVMTI_ENABLE, events[i], NULL)), + (jvmti->SetEventNotificationMode(JVMTI_ENABLE, events[i], nullptr)), false); } diff --git a/src/profiler.cc b/src/profiler.cc index 2cc1bbf9f..69cbe791f 100644 --- a/src/profiler.cc +++ b/src/profiler.cc @@ -158,7 +158,7 @@ bool SignalHandler::SetSigprofInterval(int64_t period_usec) { timer.it_interval.tv_sec = 0; timer.it_interval.tv_usec = period_usec; timer.it_value = timer.it_interval; - if (setitimer(ITIMER_PROF, &timer, 0) == -1) { + if (setitimer(ITIMER_PROF, &timer, nullptr) == -1) { LOG(ERROR) << "Scheduling profiler interval failed with error " << errno; return false; } @@ -168,7 +168,7 @@ bool SignalHandler::SetSigprofInterval(int64_t period_usec) { struct sigaction SignalHandler::SetAction(void (*action)(int, siginfo_t *, void *)) { struct sigaction sa; - sa.sa_handler = NULL; + sa.sa_handler = nullptr; sa.sa_sigaction = action; sa.sa_flags = SA_RESTART | SA_SIGINFO; From 1be226ad3585dc11703bbd99239cf5008d0db1c7 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Fri, 4 Feb 2022 06:29:52 +0000 Subject: [PATCH 02/22] Add contention and threads profile types to throttler. Note: The Java agent currently doesn't support exporting contention, thread profiles. PiperOrigin-RevId: 426318492 --- src/throttler.h | 3 +++ src/throttler_api.cc | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/throttler.h b/src/throttler.h index aa9bbcc9b..93de22497 100644 --- a/src/throttler.h +++ b/src/throttler.h @@ -26,6 +26,9 @@ namespace profiler { constexpr char kTypeCPU[] = "cpu"; constexpr char kTypeWall[] = "wall"; constexpr char kTypeHeap[] = "heap"; +// contention and threads are not supported in Java but are needed here for C++ +constexpr char kTypeContention[] = "contention"; +constexpr char kTypeThreads[] = "threads"; // Iterator-like abstraction used to guide a profiling loop comprising of // waiting for when the next profile may be collected and saving its data once diff --git a/src/throttler_api.cc b/src/throttler_api.cc index e3e6bf824..c6ede7377 100644 --- a/src/throttler_api.cc +++ b/src/throttler_api.cc @@ -341,6 +341,12 @@ std::string APIThrottler::ProfileType() { return kTypeWall; case api::HEAP: return kTypeHeap; + // contention and threads are not supported in Java but are needed here + // for C++ + case api::CONTENTION: + return kTypeContention; + case api::THREADS: + return kTypeThreads; default: const std::string& pt_name = api::ProfileType_Name(pt); LOG(ERROR) << "Unsupported profile type " << pt_name; From aab0dd1bac7edd2ca33bc4c8891fbff5d8071f94 Mon Sep 17 00:00:00 2001 From: nolanmar Date: Fri, 6 May 2022 23:24:53 +0000 Subject: [PATCH 03/22] fix: relax service name regexp to allow service name to start with number With this change, profiling agents can be run with App Engine applications whose service names start with number. PiperOrigin-RevId: 447093837 --- src/throttler_api.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/throttler_api.cc b/src/throttler_api.cc index c6ede7377..6abe99165 100644 --- a/src/throttler_api.cc +++ b/src/throttler_api.cc @@ -178,7 +178,7 @@ bool InitializeDeployment(CloudEnv* env, const std::string& labels, if (!IsValidServiceName(service)) { LOG(ERROR) << "Deployment service name '" << service - << "' does not match pattern '^[a-z]([-a-z0-9_.]{0,253}[a-z0-9])?$'"; + << "' does not match pattern '^[a-z0-9]([-a-z0-9_.]{0,253}[a-z0-9])?$'"; return false; } d->set_target(service); @@ -227,12 +227,12 @@ bool AddProfileLabels(api::Profile* p, const std::string& labels) { } // namespace // Returns true if the service name matches the regex -// "^[a-z]([-a-z0-9_.]{0,253}[a-z0-9])?$", and false otherwise. +// "^[a-z0-9]([-a-z0-9_.]{0,253}[a-z0-9])?$", and false otherwise. bool IsValidServiceName(std::string s) { if (s.length() < 1 || s.length() > 255) { return false; } - if (s[0] < 'a' || s[0] > 'z') { + if ((s[0] < 'a' || s[0] > 'z') && (s[0] < '0' || s[0] > '9')) { return false; } if ((s.back() < 'a' || s.back() > 'z') && From 596bf18c35dc426b1152f3329ed97e847f3afa5f Mon Sep 17 00:00:00 2001 From: dthomson Date: Mon, 27 Jun 2022 19:09:42 +0000 Subject: [PATCH 04/22] Allow use of AsyncGetCallTrace for TLAB heap sampling, which is faster than JVMTI stack traces. PiperOrigin-RevId: 457536381 --- src/entry.cc | 5 +- third_party/javaprofiler/accessors.h | 18 ++-- third_party/javaprofiler/heap_sampler.cc | 93 +++++++++++++++----- third_party/javaprofiler/heap_sampler.h | 20 +++-- third_party/javaprofiler/profile_test_lib.cc | 19 ++++ third_party/javaprofiler/profile_test_lib.h | 2 + 6 files changed, 121 insertions(+), 36 deletions(-) diff --git a/src/entry.cc b/src/entry.cc index f71c32a18..3d0d17dcd 100644 --- a/src/entry.cc +++ b/src/entry.cc @@ -122,8 +122,11 @@ void JNICALL OnVMInit(jvmtiEnv *jvmti, JNIEnv *jni_env, jthread thread) { } if (FLAGS_cprof_enable_heap_sampling) { + // TODO: Allow using the JVM's stack tracer with a flag once + // we can get the current context in a cloud build. google::javaprofiler::HeapMonitor::Enable( - jvmti, jni_env, FLAGS_cprof_heap_sampling_interval); + jvmti, jni_env, FLAGS_cprof_heap_sampling_interval, + false /* use_jvm_trace */); } worker->Start(jni_env); diff --git a/third_party/javaprofiler/accessors.h b/third_party/javaprofiler/accessors.h index 004bfe945..ef100d1e3 100644 --- a/third_party/javaprofiler/accessors.h +++ b/third_party/javaprofiler/accessors.h @@ -69,17 +69,25 @@ class Accessors { template static inline FunctionType GetJvmFunction(const char *function_name) { - // get handle to library - static void *handle = dlopen("libjvm.so", RTLD_LAZY); - if (handle == NULL) { - return NULL; + static void *handle = GetJvmLibHandle(); + if (handle == nullptr) { + return nullptr; } - // get address of function, return null if not found + // Get address of function, return null if not found return bit_cast(dlsym(handle, function_name)); } private: + static void *GetJvmLibHandle() { + void *handle = dlopen("libjvm.so", RTLD_LAZY); + if (handle != nullptr) { + return handle; + } + + return dlopen(nullptr, RTLD_LAZY); + } + // This is subtle and potentially dangerous, read this carefully. // // In glibc, TLS access is not signal-async-safe, as they can call malloc for diff --git a/third_party/javaprofiler/heap_sampler.cc b/third_party/javaprofiler/heap_sampler.cc index 37d131950..09a91c7af 100644 --- a/third_party/javaprofiler/heap_sampler.cc +++ b/third_party/javaprofiler/heap_sampler.cc @@ -14,14 +14,23 @@ * limitations under the License. */ +#include +#include +#include + +#include "third_party/javaprofiler/accessors.h" #include "third_party/javaprofiler/heap_sampler.h" #include "third_party/javaprofiler/profile_proto_builder.h" +#include "third_party/javaprofiler/stacktrace_decls.h" +#include "third_party/javaprofiler/stacktraces.h" namespace { -std::vector +using google::javaprofiler::JVMPI_CallFrame; + +std::vector TransformFrames(jvmtiFrameInfo *stack_frames, int count) { - std::vector frames(count); + std::vector frames(count); for (int i = 0; i < count; i++) { // Note that technically this is not the line number; it is the location but @@ -34,6 +43,32 @@ TransformFrames(jvmtiFrameInfo *stack_frames, int count) { return frames; } +std::unique_ptr> GetTrace(JNIEnv *jni) { + std::unique_ptr> trace_result = nullptr; + +// Cannot use the fast stacktrace for standalone builds as their standard +// libraries do not support getcontext. + + return trace_result; +} + +std::unique_ptr> GetTraceUsingJvmti( + JNIEnv *jni, jvmtiEnv *jvmti, jthread thread) { + const int kMaxFrames = 128; + jint count = 0; + jvmtiFrameInfo stack_frames[kMaxFrames]; + + jvmtiError err = + jvmti->GetStackTrace(thread, 0, kMaxFrames, stack_frames, &count); + + if (err != JVMTI_ERROR_NONE || count <= 0) { + return nullptr; + } + + return std::unique_ptr>( + new std::vector(TransformFrames(stack_frames, count))); +} + extern "C" JNIEXPORT void SampledObjectAlloc(jvmtiEnv *jvmti_env, JNIEnv *jni_env, jthread thread, jobject object, @@ -50,8 +85,10 @@ extern "C" JNIEXPORT void GarbageCollectionFinish(jvmtiEnv *jvmti_env) { namespace google { namespace javaprofiler { + std::atomic HeapMonitor::jvmti_; std::atomic HeapMonitor::sampling_interval_; +std::atomic HeapMonitor::use_jvm_trace_; HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, int max_garbage_size) @@ -62,28 +99,19 @@ HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, } void HeapEventStorage::Add(JNIEnv *jni, jthread thread, jobject object, - jclass klass, jlong size) { - const int kMaxFrames = 128; - jint count = 0; - jvmtiFrameInfo stack_frames[kMaxFrames]; - jvmtiError err = - jvmti_->GetStackTrace(thread, 0, kMaxFrames, stack_frames, &count); - - if (err == JVMTI_ERROR_NONE && count > 0) { - auto frames = TransformFrames(stack_frames, count); - - jweak weak_ref = jni->NewWeakGlobalRef(object); - if (jni->ExceptionCheck()) { - LOG(WARNING) << "Failed to create NewWeakGlobalRef, skipping heap sample"; - return; - } + jclass klass, jlong size, + const std::vector &&frames) { + jweak weak_ref = jni->NewWeakGlobalRef(object); + if (jni->ExceptionCheck()) { + LOG(WARNING) << "Failed to create NewWeakGlobalRef, skipping heap sample"; + return; + } - HeapObjectTrace live_object(weak_ref, size, std::move(frames)); + HeapObjectTrace live_object(weak_ref, size, std::move(frames)); - // Only now lock and get things done quickly. - std::lock_guard lock(storage_lock_); - newly_allocated_objects_.push_back(std::move(live_object)); - } + // Only now lock and get things done quickly. + std::lock_guard lock(storage_lock_); + newly_allocated_objects_.push_back(std::move(live_object)); } void HeapEventStorage::AddToGarbage(HeapObjectTrace &&obj) { @@ -266,6 +294,19 @@ bool HeapMonitor::Supported(jvmtiEnv *jvmti) { #endif } +void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, + jclass object_klass, jlong size) { + auto trace = use_jvm_trace_.load() + ? GetTrace(jni_env) + : GetTraceUsingJvmti(jni_env, jvmti_.load(), thread); + if (trace == nullptr) { + return; + } + + GetInstance()->storage_.Add(jni_env, thread, object, object_klass, size, + std::move(*trace)); +} + void HeapMonitor::AddCallback(jvmtiEventCallbacks *callbacks) { #ifdef ENABLE_HEAP_SAMPLING callbacks->SampledObjectAlloc = &SampledObjectAlloc; @@ -274,7 +315,8 @@ void HeapMonitor::AddCallback(jvmtiEventCallbacks *callbacks) { } // Currently, we enable once and forget about it. -bool HeapMonitor::Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval) { +bool HeapMonitor::Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval, + bool use_jvm_trace) { #ifdef ENABLE_HEAP_SAMPLING if (!Supported(jvmti)) { LOG(WARNING) << "Heap sampling is not supported by the JVM, disabling the " @@ -304,6 +346,7 @@ bool HeapMonitor::Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval) { jvmti_.store(jvmti); sampling_interval_.store(sampling_interval); + use_jvm_trace_.store(use_jvm_trace); if (!GetInstance()->CreateGCWaitingThread(jvmti, jni)) { return false; @@ -328,6 +371,10 @@ bool HeapMonitor::Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval) { return false; } + if (use_jvm_trace) { + Asgct::SetAsgct(Accessors::GetJvmFunction("AsyncGetCallTrace")); + } + return true; #else return false; diff --git a/third_party/javaprofiler/heap_sampler.h b/third_party/javaprofiler/heap_sampler.h index bd830cefe..b5997f11c 100644 --- a/third_party/javaprofiler/heap_sampler.h +++ b/third_party/javaprofiler/heap_sampler.h @@ -20,13 +20,13 @@ #include #include +#include #include // NOLINT #include #include #include // NOLINT #include -#include "third_party/javaprofiler/globals.h" #include "third_party/javaprofiler/profile_proto_builder.h" namespace google { @@ -45,7 +45,7 @@ class HeapEventStorage { // Adds an object to the storage system. void Add(JNIEnv *jni, jthread thread, jobject object, jclass klass, - jlong size); + jlong size, const std::vector &&frames); // Returns a perftools::profiles::Profile with the objects stored via // calls to the Add method. @@ -190,7 +190,8 @@ class HeapEventStorage { // Due to the JVMTI callback, everything here is static. class HeapMonitor { public: - static bool Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval); + static bool Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval, + bool use_jvm_trace); static void Disable(); static bool Enabled() { return jvmti_ != nullptr; } @@ -210,9 +211,7 @@ class HeapMonitor { JNIEnv* env, bool force_gc); static void AddSample(JNIEnv *jni_env, jthread thread, jobject object, - jclass object_klass, jlong size) { - GetInstance()->storage_.Add(jni_env, thread, object, object_klass, size); - } + jclass object_klass, jlong size); static void AddCallback(jvmtiEventCallbacks *callbacks); @@ -229,7 +228,8 @@ class HeapMonitor { HeapMonitor &operator=(const HeapMonitor &) = delete; private: - HeapMonitor() : storage_(jvmti_.load()) {} + HeapMonitor() : storage_(jvmti_.load(), GetFrameCache()) { + } // We construct the heap_monitor at the first call to GetInstance, so ensure // Enable was called at least once before to initialize jvmti_. @@ -238,6 +238,11 @@ class HeapMonitor { return &heap_monitor; } + ProfileFrameCache *GetFrameCache() { + ProfileFrameCache *cache = nullptr; + return cache; + } + static bool Supported(jvmtiEnv* jvmti); enum class GcEvent { @@ -259,6 +264,7 @@ class HeapMonitor { static std::atomic jvmti_; static std::atomic sampling_interval_; + static std::atomic use_jvm_trace_; std::list gc_notify_events_; std::condition_variable gc_waiting_cv_; diff --git a/third_party/javaprofiler/profile_test_lib.cc b/third_party/javaprofiler/profile_test_lib.cc index 701dc69ed..f41c6f996 100644 --- a/third_party/javaprofiler/profile_test_lib.cc +++ b/third_party/javaprofiler/profile_test_lib.cc @@ -177,6 +177,25 @@ static jvmtiError GetStackTrace(jvmtiEnv* env, jthread thread, jint start_depth, return JVMTI_ERROR_NONE; } +std::vector CreateStackTrace(int i) { + std::vector frames; + + switch (i) { + case 0: + frames.push_back({30, reinterpret_cast(1)}); + frames.push_back({64, reinterpret_cast(2)}); + break; + case 1: + frames.push_back({128, reinterpret_cast(3)}); + break; + default: + // Add nothing + ; + } + + return frames; +} + static jvmtiError ForceGarbageCollection(jvmtiEnv* env) { return JVMTI_ERROR_NONE; } diff --git a/third_party/javaprofiler/profile_test_lib.h b/third_party/javaprofiler/profile_test_lib.h index 1f0655700..c0251e49e 100644 --- a/third_party/javaprofiler/profile_test_lib.h +++ b/third_party/javaprofiler/profile_test_lib.h @@ -76,6 +76,8 @@ class JvmProfileTestLib { static std::atomic method_name_call_count; }; +std::vector CreateStackTrace(int i); + } // namespace javaprofiler } // namespace google From 49cb69e40227205070e5f0aad7c99f267bef1d61 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Mon, 27 Jun 2022 20:03:32 +0000 Subject: [PATCH 05/22] Update roots.pem to align with the current public set PiperOrigin-RevId: 457548031 --- src/pem_roots.cc | 291 +++++++++++++++++++++++------------------------ 1 file changed, 145 insertions(+), 146 deletions(-) diff --git a/src/pem_roots.cc b/src/pem_roots.cc index 7a3cf8711..5512b5b3c 100644 --- a/src/pem_roots.cc +++ b/src/pem_roots.cc @@ -806,152 +806,6 @@ dEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5Cuf ReYNnyicsbkqWletNw+vHX/bvZ8= -----END CERTIFICATE----- -# Operating CA: Google Trust Services LLC -# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 -# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign ECC Root CA - R4 -# Label: "GlobalSign ECC Root CA - R4" -# Serial: 14367148294922964480859022125800977897474 -# MD5 Fingerprint: 20:f0:27:68:d1:7e:a0:9d:0e:e6:2a:ca:df:5c:89:8e -# SHA1 Fingerprint: 69:69:56:2e:40:80:f4:24:a1:e7:19:9f:14:ba:f3:ee:58:ab:6a:bb -# SHA256 Fingerprint: be:c9:49:11:c2:95:56:76:db:6c:0a:55:09:86:d7:6e:3b:a0:05:66:7c:44:2c:97:62:b4:fb:b7:73:de:22:8c ------BEGIN CERTIFICATE----- -MIIB4TCCAYegAwIBAgIRKjikHJYKBN5CsiilC+g0mAIwCgYIKoZIzj0EAwIwUDEk -MCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBDQSAtIFI0MRMwEQYDVQQKEwpH -bG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWduMB4XDTEyMTExMzAwMDAwMFoX -DTM4MDExOTAzMTQwN1owUDEkMCIGA1UECxMbR2xvYmFsU2lnbiBFQ0MgUm9vdCBD -QSAtIFI0MRMwEQYDVQQKEwpHbG9iYWxTaWduMRMwEQYDVQQDEwpHbG9iYWxTaWdu -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEuMZ5049sJQ6fLjkZHAOkrprlOQcJ -FspjsbmG+IpXwVfOQvpzofdlQv8ewQCybnMO/8ch5RikqtlxP6jUuc6MHaNCMEAw -DgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFFSwe61F -uOJAf/sKbvu+M8k8o4TVMAoGCCqGSM49BAMCA0gAMEUCIQDckqGgE6bPA7DmxCGX -kPoUVy0D7O48027KqGx2vKLeuwIgJ6iFJzWbVsaj8kfSt24bAgAXqmemFZHe+pTs -ewv4n4Q= ------END CERTIFICATE----- - -# Operating CA: Google Trust Services LLC -# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R1 -# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R1 -# Label: "GTS Root R1" -# Serial: 6e:47:a9:c5:4b:47:0c:0d:ec:33:d0:89:b9:1c:f4:e1 -# MD5 Fingerprint: 82:1A:EF:D4:D2:4A:F2:9F:E2:3D:97:06:14:70:72:85 -# SHA1 Fingerprint: E1:C9:50:E6:EF:22:F8:4C:56:45:72:8B:92:20:60:D7:D5:A7:A3:E8 -# SHA256 Fingerprint: 2A:57:54:71:E3:13:40:BC:21:58:1C:BD:2C:F1:3E:15:84:63:20:3E:CE:94:BC:F9:D3:CC:19:6B:F0:9A:54:72 ------BEGIN CERTIFICATE----- -MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH -MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM -QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy -MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl -cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM -f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX -mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 -zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P -fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc -vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 -Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp -zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO -Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW -k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ -DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF -lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV -HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW -Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 -d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z -XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR -gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 -d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv -J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg -DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM -+SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy -F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 -SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws -E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl ------END CERTIFICATE----- - -# Operating CA: Google Trust Services LLC -# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R2 -# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R2 -# Label: "GTS Root R2" -# Serial: 6e:47:a9:c6:5a:b3:e7:20:c5:30:9a:3f:68:52:f2:6f -# MD5 Fingerprint: 44:ED:9A:0E:A4:09:3B:00:F2:AE:4C:A3:C6:61:B0:8B -# SHA1 Fingerprint: D2:73:96:2A:2A:5E:39:9F:73:3F:E1:C7:1E:64:3F:03:38:34:FC:4D -# SHA256 Fingerprint: C4:5D:7B:B0:8E:6D:67:E6:2E:42:35:11:0B:56:4E:5F:78:FD:92:EF:05:8C:84:0A:EA:4E:64:55:D7:58:5C:60 ------BEGIN CERTIFICATE----- -MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH -MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM -QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy -MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl -cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB -AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv -CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg -GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu -XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd -re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu -PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 -mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K -8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj -x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR -nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 -kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok -twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV -HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp -8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT -vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT -z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA -pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb -pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB -R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R -RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk -0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC -5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF -izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn -yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC ------END CERTIFICATE----- - -# Operating CA: Google Trust Services LLC -# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R3 -# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R3 -# Label: "GTS Root R3" -# Serial: 6e:47:a9:c7:6c:a9:73:24:40:89:0f:03:55:dd:8d:1d -# MD5 Fingerprint: 1A:79:5B:6B:04:52:9C:5D:C7:74:33:1B:25:9A:F9:25 -# SHA1 Fingerprint: 30:D4:24:6F:07:FF:DB:91:89:8A:0B:E9:49:66:11:EB:8C:5E:46:E5 -# SHA256 Fingerprint: 15:D5:B8:77:46:19:EA:7D:54:CE:1C:A6:D0:B0:C4:03:E0:37:A9:17:F1:31:E8:A0:4E:1E:6B:7A:71:BA:BC:E5 ------BEGIN CERTIFICATE----- -MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw -CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU -MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw -MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp -Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA -IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout -736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A -DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud -DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk -fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA -njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd ------END CERTIFICATE----- - -# Operating CA: Google Trust Services LLC -# Issuer: C=US, O=Google Trust Services LLC, CN=GTS Root R4 -# Subject: C=US, O=Google Trust Services LLC, CN=GTS Root R4 -# Label: "GTS Root R4" -# Serial: 6e:47:a9:c8:8b:94:b6:e8:bb:3b:2a:d8:a2:b2:c1:99 -# MD5 Fingerprint: 5D:B6:6A:C4:60:17:24:6A:1A:99:A8:4B:EE:5E:B4:26 -# SHA1 Fingerprint: 2A:1D:60:27:D9:4A:B1:0A:1C:4D:91:5C:CD:33:A0:CB:3E:2D:54:CB -# SHA256 Fingerprint: 71:CC:A5:39:1F:9E:79:4B:04:80:25:30:B3:63:E1:21:DA:8A:30:43:BB:26:66:2F:EA:4D:CA:7F:C9:51:A4:BD ------BEGIN CERTIFICATE----- -MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw -CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU -MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw -MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp -Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA -IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu -hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l -xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud -DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 -CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx -sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== ------END CERTIFICATE----- - # Operating CA: Sectigo # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited @@ -1155,6 +1009,151 @@ VXyNWQKV3WKdwrnuWih0hKWbt5DHDAff9Yk2dDLWKMGwsAvgnEzDHNb842m1R0aB L6KCq9NjRHDEjf8tM7qtj3u1cIiuPhnPQCjY/MiQu12ZIvVS5ljFH4gxQ+6IHdfG jjxDah2nGN59PRbxYvnKkKj9 -----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Subject: C = US, O = Google Trust Services LLC, CN = GTS Root R1 +# Issuer: C = US, O = Google Trust Services LLC, CN = GTS Root R1 +# Label: "GTS Root R1" +# Serial: 0203E5936F31B01349886BA217 +# MD5 Fingerprint: 05:FE:D0:BF:71:A8:A3:76:63:DA:01:E0:D8:52:DC:40 +# SHA1 Fingerprint: E5:8C:1C:C4:91:3B:38:63:4B:E9:10:6E:E3:AD:8E:6B:9D:D9:81:4A +# SHA256 Fingerprint: D9:47:43:2A:BD:E7:B7:FA:90:FC:2E:6B:59:10:1B:12:80:E0:E1:C7:E4:E4:0F:A3:C6:88:7F:FF:57:A7:F4:CF +-----BEGIN CERTIFICATE----- +MIIFVzCCAz+gAwIBAgINAgPlk28xsBNJiGuiFzANBgkqhkiG9w0BAQwFADBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaMf/vo +27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vXmX7w +Cl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7zUjw +TcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0Pfybl +qAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtcvfaH +szVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4Zor8 +Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUspzBmk +MiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOORc92 +wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYWk70p +aDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+DVrN +VjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgFlQID +AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBAJ+qQibb +C5u+/x6Wki4+omVKapi6Ist9wTrYggoGxval3sBOh2Z5ofmmWJyq+bXmYOfg6LEe +QkEzCzc9zolwFcq1JKjPa7XSQCGYzyI0zzvFIoTgxQ6KfF2I5DUkzps+GlQebtuy +h6f88/qBVRRiClmpIgUxPoLW7ttXNLwzldMXG+gnoot7TiYaelpkttGsN/H9oPM4 +7HLwEXWdyzRSjeZ2axfG34arJ45JK3VmgRAhpuo+9K4l/3wV3s6MJT/KYnAK9y8J +ZgfIPxz88NtFMN9iiMG1D53Dn0reWVlHxYciNuaCp+0KueIHoI17eko8cdLiA6Ef +MgfdG+RCzgwARWGAtQsgWSl4vflVy2PFPEz0tv/bal8xa5meLMFrUKTX5hgUvYU/ +Z6tGn6D/Qqc6f1zLXbBwHSs09dR2CQzreExZBfMzQsNhFRAbd03OIozUhfJFfbdT +6u9AWpQKXCBfTkBdYiJ23//OYb2MI3jSNwLgjt7RETeJ9r/tSQdirpLsQBqvFAnZ +0E6yove+7u7Y/9waLd64NnHi/Hm3lCXRSHNboTXns5lndcEZOitHTtNCjv0xyBZm +2tIMPNuzjsmhDYAPexZ3FL//2wmUspO8IFgV6dtxQ/PeEMMA3KgqlbbC1j+Qa3bb +bP6MvPJwNQzcmRk13NfIRmPVNnGuV/u3gm3c +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Subject: C = US, O = Google Trust Services LLC, CN = GTS Root R2 +# Issuer: C = US, O = Google Trust Services LLC, CN = GTS Root R2 +# Label: "GTS Root R2" +# Serial: 0203E5AEC58D04251AAB1125AA +# MD5 Fingerprint=1E:39:C0:53:E6:1E:29:82:0B:CA:52:55:36:5D:57:DC +# SHA1 Fingerprint=9A:44:49:76:32:DB:DE:FA:D0:BC:FB:5A:7B:17:BD:9E:56:09:24:94 +# SHA256 Fingerprint=8D:25:CD:97:22:9D:BF:70:35:6B:DA:4E:B3:CC:73:40:31:E2:4C:F0:0F:AF:CF:D3:2D:C7:6E:B5:84:1C:7E:A8 +-----BEGIN CERTIFICATE----- +MIIFVzCCAz+gAwIBAgINAgPlrsWNBCUaqxElqjANBgkqhkiG9w0BAQwFADBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEBAQUA +A4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3LvCvpt +nfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3KgGjSY +6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9BuXvAu +MC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOdre7k +RXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXuPuWg +f9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1mKPV ++3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K8Yzo +dDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqjx5RW +Ir9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsRnTKa +G73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0kzCq +gc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9OktwID +AQABo0IwQDAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBAB/Kzt3H +vqGf2SdMC9wXmBFqiN495nFWcrKeGk6c1SuYJF2ba3uwM4IJvd8lRuqYnrYb/oM8 +0mJhwQTtzuDFycgTE1XnqGOtjHsB/ncw4c5omwX4Eu55MaBBRTUoCnGkJE+M3DyC +B19m3H0Q/gxhswWV7uGugQ+o+MePTagjAiZrHYNSVc61LwDKgEDg4XSsYPWHgJ2u +NmSRXbBoGOqKYcl3qJfEycel/FVL8/B/uWU9J2jQzGv6U53hkRrJXRqWbTKH7QMg +yALOWr7Z6v2yTcQvG99fevX4i8buMTolUVVnjWQye+mew4K6Ki3pHrTgSAai/Gev +HyICc/sgCq+dVEuhzf9gR7A/Xe8bVr2XIZYtCtFenTgCR2y59PYjJbigapordwj6 +xLEokCZYCDzifqrXPW+6MYgKBesntaFJ7qBFVHvmJ2WZICGoo7z7GJa7Um8M7YNR +TOlZ4iBgxcJlkoKM8xAfDoqXvneCbT+PHV28SSe9zE8P4c52hgQjxcCMElv924Sg +JPFI/2R80L5cFtHvma3AH/vLrrw4IgYmZNralw4/KBVEqE8AyvCazM90arQ+POuV +7LXTWtiBmelDGDfrs7vRWGJB82bSj6p4lVQgw1oudCvV0b4YacCs1aTPObpRhANl +6WLAYv7YTVWW4tAR+kg0Eeye7QUd5MjWHYbL +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Subject: C = US, O = Google Trust Services LLC, CN = GTS Root R3 +# Issuer: C = US, O = Google Trust Services LLC, CN = GTS Root R3 +# Label: "GTS Root R3" +# Serial: 0203E5B882EB20F825276D3D66 +# MD5 Fingerprint: 3E:E7:9D:58:02:94:46:51:94:E5:E0:22:4A:8B:E7:73 +# SHA1 Fingerprint: ED:E5:71:80:2B:C8:92:B9:5B:83:3C:D2:32:68:3F:09:CD:A0:1E:46 +# SHA256 Fingerprint: 34:D8:A7:3E:E2:08:D9:BC:DB:0D:95:65:20:93:4B:4E:40:E6:94:82:59:6E:8B:6F:73:C8:42:6B:01:0A:6F:48 +-----BEGIN CERTIFICATE----- +MIICCTCCAY6gAwIBAgINAgPluILrIPglJ209ZjAKBggqhkjOPQQDAzBHMQswCQYD +VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG +A1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw +WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz +IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQAIgNi +AAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout736G +jOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2ADDL2 +4CejQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEA9uEglRR7 +VKOQFhG/hMjqb2sXnh5GmCCbn9MN2azTL818+FsuVbu/3ZL3pAzcMeGiAjEA/Jdm +ZuVDFhOD3cffL74UOO0BzrEXGhF16b0DjyZ+hOXJYKaV11RZt+cRLInUue4X +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Subject: C = US, O = Google Trust Services LLC, CN = GTS Root R4 +# Issuer: C = US, O = Google Trust Services LLC, CN = GTS Root R4 +# Label: "GTS Root R4" +# Serial: 0203E5C068EF631A9C72905052 +# MD5 Fingerprint=43:96:83:77:19:4D:76:B3:9D:65:52:E4:1D:22:A5:E8 +# SHA1 Fingerprint=77:D3:03:67:B5:E0:0C:15:F6:0C:38:61:DF:7C:E1:3B:92:46:4D:47 +# SHA256 Fingerprint=34:9D:FA:40:58:C5:E2:63:12:3B:39:8A:E7:95:57:3C:4E:13:13:C8:3F:E6:8F:93:55:6C:D5:E8:03:1B:3C:7D +-----BEGIN CERTIFICATE----- +MIICCTCCAY6gAwIBAgINAgPlwGjvYxqccpBQUjAKBggqhkjOPQQDAzBHMQswCQYD +VQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEUMBIG +A1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAwMDAw +WjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2Vz +IExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQAIgNi +AATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzuhXyi +QHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/lxKvR +HYqjQjBAMA4GA1UdDwEB/wQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW +BBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNpADBmAjEA6ED/g94D +9J+uHXqnLrmvT/aDHQ4thQEd0dlq7A/Cr8deVl5c1RxYIigL9zC2L7F8AjEA8GE8 +p/SgguMh1YQdc4acLa/KNJvxn7kjNuK8YAOdgLOaVsjh4rsUecrNIdSUtUlD +-----END CERTIFICATE----- + +# Operating CA: Google Trust Services LLC +# Subject: OU = GlobalSign ECC Root CA - R4, O = GlobalSign, CN = GlobalSign +# Issuer: OU = GlobalSign ECC Root CA - R4, O = GlobalSign, CN = GlobalSign +# Label: "GlobalSign R4" +# Serial: 0203E57EF53F93FDA50921B2A6 +# MD5 Fingerprint: 26:29:F8:6D:E1:88:BF:A2:65:7F:AA:C4:CD:0F:7F:FC +# SHA1 Fingerprint: 6B:A0:B0:98:E1:71:EF:5A:AD:FE:48:15:80:77:10:F4:BD:6F:0B:28 +# SHA256 Fingerprint: B0:85:D7:0B:96:4F:19:1A:73:E4:AF:0D:54:AE:7A:0E:07:AA:FD:AF:9B:71:DD:08:62:13:8A:B7:32:5A:24:A2 +-----BEGIN CERTIFICATE----- +MIIB3DCCAYOgAwIBAgINAgPlfvU/k/2lCSGypjAKBggqhkjOPQQDAjBQMSQwIgYD +VQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0gUjQxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTIxMTEzMDAwMDAwWhcNMzgw +MTE5MDMxNDA3WjBQMSQwIgYDVQQLExtHbG9iYWxTaWduIEVDQyBSb290IENBIC0g +UjQxEzARBgNVBAoTCkdsb2JhbFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wWTAT +BgcqhkjOPQIBBggqhkjOPQMBBwNCAAS4xnnTj2wlDp8uORkcA6SumuU5BwkWymOx +uYb4ilfBV85C+nOh92VC/x7BALJucw7/xyHlGKSq2XE/qNS5zowdo0IwQDAOBgNV +HQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUVLB7rUW44kB/ ++wpu+74zyTyjhNUwCgYIKoZIzj0EAwIDRwAwRAIgIk90crlgr/HmnKAWBVBfw147 +bmF0774BxL4YSFlhgjICICadVGNA3jdgUM/I2O2dgq43mLyjj0xMqTQrbO/7lZsm +-----END CERTIFICATE----- )|"; // end kPemRootCerts } // namespace profiler From e09f17fa3260e02e5c4cc4073c4b875647adecbb Mon Sep 17 00:00:00 2001 From: nolanmar Date: Thu, 7 Jul 2022 22:13:30 +0000 Subject: [PATCH 06/22] chore: update CONTRIBUTING.md Unfortunately, this repo is not set up in a way that allows it to accept contributions from open source. Updating CONTRIBUTING.md to reflect this. PiperOrigin-RevId: 459613579 --- CONTRIBUTING.md | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 939e5341e..977af8bb8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,28 +1 @@ -# How to Contribute - -We'd love to accept your patches and contributions to this project. There are -just a few small guidelines you need to follow. - -## Contributor License Agreement - -Contributions to this project must be accompanied by a Contributor License -Agreement. You (or your employer) retain the copyright to your contribution; -this simply gives us permission to use and redistribute your contributions as -part of the project. Head over to to see -your current agreements on file or to sign a new one. - -You generally only need to submit a CLA once, so if you've already submitted one -(even if it was for a different project), you probably don't need to do it -again. - -## Code reviews - -All submissions, including submissions by project members, require review. We -use GitHub pull requests for this purpose. Consult -[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more -information on using pull requests. - -## Community Guidelines - -This project follows [Google's Open Source Community -Guidelines](https://opensource.google.com/conduct/). +This repo cannot currently accept open source contributions. From b4f90d5a5f83cdcb372dfe3d3aa37f389b292dc5 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Fri, 22 Jul 2022 18:42:39 +0000 Subject: [PATCH 07/22] Prepare for requiring default_sample_type to always be present. PiperOrigin-RevId: 462669300 --- third_party/javaprofiler/profile_proto_builder.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/third_party/javaprofiler/profile_proto_builder.cc b/third_party/javaprofiler/profile_proto_builder.cc index 9f764406d..bcfeff9d6 100644 --- a/third_party/javaprofiler/profile_proto_builder.cc +++ b/third_party/javaprofiler/profile_proto_builder.cc @@ -43,6 +43,9 @@ ProfileProtoBuilder::ProfileProtoBuilder(JNIEnv *jni_env, jvmtiEnv *jvmti_env, location_builder_(&builder_) { AddSampleType(count_type); AddSampleType(metric_type); + builder_.mutable_profile()->set_default_sample_type( + builder_.StringId(metric_type.type.c_str())); + SetPeriodType(metric_type); } From a68992c3325731368a62b56be99fdcf3d992e9ed Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Sat, 23 Jul 2022 03:43:44 +0000 Subject: [PATCH 08/22] Prepare for requiring default_sample_type to always be present. PiperOrigin-RevId: 462758425 --- src/proto.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/proto.cc b/src/proto.cc index 6dcd905c9..6c64d8ee3 100644 --- a/src/proto.cc +++ b/src/proto.cc @@ -231,6 +231,8 @@ void ProfileProtoBuilder::Populate( sample_type->set_type(builder_.StringId(profile_type)); sample_type->set_unit(builder_.StringId("nanoseconds")); + profile->set_default_sample_type(builder_.StringId(profile_type)); + profile->set_duration_nanos(duration_ns); for (const auto &trace : traces) { From 5c8cca31c819303a6a9be8728bae2be4447cf15e Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Tue, 2 Aug 2022 19:59:00 +0000 Subject: [PATCH 09/22] Prepare for requiring default_sample_type to always be present. PiperOrigin-RevId: 464865590 --- .../javaprofiler/profile_proto_builder.h | 2 ++ .../perftools/profiles/proto/builder.cc | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/third_party/javaprofiler/profile_proto_builder.h b/third_party/javaprofiler/profile_proto_builder.h index ec29c5f4a..75c7bb070 100644 --- a/third_party/javaprofiler/profile_proto_builder.h +++ b/third_party/javaprofiler/profile_proto_builder.h @@ -441,6 +441,8 @@ class ContentionProfileProtoBuilder : public ProfileProtoBuilder { ProfileProtoBuilder::SampleType("delay", "microseconds")) { builder_.mutable_profile()->set_duration_nanos(duration_nanos); builder_.mutable_profile()->set_period(sampling_rate); + builder_.mutable_profile()->set_default_sample_type( + builder_.StringId("delay")); } std::unique_ptr CreateProto() { diff --git a/third_party/perftools/profiles/proto/builder.cc b/third_party/perftools/profiles/proto/builder.cc index 428cf93d7..da1f6173d 100644 --- a/third_party/perftools/profiles/proto/builder.cc +++ b/third_party/perftools/profiles/proto/builder.cc @@ -192,6 +192,26 @@ bool Builder::CheckValid(const Profile &profile) { return false; } + const int default_sample_type = profile.default_sample_type(); + if (default_sample_type <= 0 || + default_sample_type >= profile.string_table_size()) { + LOG(ERROR) << "No default sample type specified"; + return false; + } + + std::unordered_set value_types; + for (const auto &sample_type : profile.sample_type()) { + if (!value_types.insert(sample_type.type()).second) { + LOG(ERROR) << "Duplicate sample_type specified"; + return false; + } + } + + if (value_types.count(default_sample_type) == 0) { + LOG(ERROR) << "Default sample type not found"; + return false; + } + for (const auto &sample : profile.sample()) { if (sample.value_size() != sample_type_len) { LOG(ERROR) << "Found sample with " << sample.value_size() From f1be5e412528686ed06bc94dc25fa2f05029a295 Mon Sep 17 00:00:00 2001 From: dthomson Date: Wed, 3 Aug 2022 18:47:09 +0000 Subject: [PATCH 10/22] Don't attempt to skip C++->Java JNI frames in profile traces. PiperOrigin-RevId: 465109242 --- .../javaprofiler/profile_proto_builder.cc | 22 +++------- .../javaprofiler/profile_proto_builder.h | 43 +------------------ 2 files changed, 8 insertions(+), 57 deletions(-) diff --git a/third_party/javaprofiler/profile_proto_builder.cc b/third_party/javaprofiler/profile_proto_builder.cc index bcfeff9d6..e34094995 100644 --- a/third_party/javaprofiler/profile_proto_builder.cc +++ b/third_party/javaprofiler/profile_proto_builder.cc @@ -181,15 +181,13 @@ void ProfileProtoBuilder::AddTrace(const ProfileStackTrace &profile_trace, const JVMPI_CallTrace *trace = trace_and_labels.trace; int first_frame = SkipTopNativeFrames(*trace); - StackState stack_state; - for (int i = first_frame; i < trace->num_frames; ++i) { auto &jvm_frame = trace->frames[i]; if (jvm_frame.lineno == kNativeFrameLineNum) { - AddNativeInfo(jvm_frame, profile, sample, &stack_state); + AddNativeInfo(jvm_frame, profile, sample); } else { - AddJavaInfo(jvm_frame, profile, sample, &stack_state); + AddJavaInfo(jvm_frame, profile, sample); } } } @@ -197,10 +195,7 @@ void ProfileProtoBuilder::AddTrace(const ProfileStackTrace &profile_trace, void ProfileProtoBuilder::AddJavaInfo( const JVMPI_CallFrame &jvm_frame, perftools::profiles::Profile *profile, - perftools::profiles::Sample *sample, - StackState *stack_state) { - stack_state->JavaFrame(); - + perftools::profiles::Sample *sample) { if (!jvm_frame.method_id) { perftools::profiles::Location *location = location_builder_.LocationFor( "", "[Unknown method]", "", 0); @@ -265,8 +260,7 @@ MethodInfo *ProfileProtoBuilder::Method(jmethodID method_id) { void ProfileProtoBuilder::AddNativeInfo(const JVMPI_CallFrame &jvm_frame, perftools::profiles::Profile *profile, - perftools::profiles::Sample *sample, - StackState *stack_state) { + perftools::profiles::Sample *sample) { if (!native_cache_) { perftools::profiles::Location *location = location_builder_.LocationFor( "", "[Unknown non-Java frame]", "", 0); @@ -285,12 +279,8 @@ void ProfileProtoBuilder::AddNativeInfo(const JVMPI_CallFrame &jvm_frame, &location_builder_); - stack_state->NativeFrame(function_name); - - if (!stack_state->SkipFrame()) { - location->set_address(reinterpret_cast(jvm_frame.method_id)); - sample->add_location_id(location->id()); - } + location->set_address(reinterpret_cast(jvm_frame.method_id)); + sample->add_location_id(location->id()); } void ContentionProfileProtoBuilder::MultiplyBySamplingRate() { diff --git a/third_party/javaprofiler/profile_proto_builder.h b/third_party/javaprofiler/profile_proto_builder.h index 75c7bb070..8e49cfcce 100644 --- a/third_party/javaprofiler/profile_proto_builder.h +++ b/third_party/javaprofiler/profile_proto_builder.h @@ -288,43 +288,6 @@ class ProfileProtoBuilder { int64 sampling_rate_ = 0; private: - // Track progress through a stack as we traverse it, in order to determine - // how processing should proceed based on the context of a frame. - class StackState { - public: - StackState() { - } - - // Notify the state that we are visiting a Java frame. - void JavaFrame() { - in_jni_helpers_ = false; - } - - // Notify the state that we are visiting a native frame. - void NativeFrame(const std::string &function_name) { - if (StartsWith(function_name, "JavaCalls::call_helper")) { - in_jni_helpers_ = true; - } - } - - // Should we skip the current frame? - bool SkipFrame() const { - return in_jni_helpers_; - } - - private: - // We don't add native frames that are just "helper" native code for - // dispatching to JNI code. We determine this by detecting a native - // JavaCalls::call_helper frame, then skipping until the we see a Java - // frame again. - // TODO: Support a "complete detail" mode to override this. - bool in_jni_helpers_ = false; - - static bool StartsWith(const std::string &s, const std::string &prefix) { - return s.find(prefix) == 0; - } - }; - void AddSampleType(const SampleType &sample_type); void SetPeriodType(const SampleType &metric_type); void InitSampleValues(perftools::profiles::Sample *sample, int64 count, @@ -334,12 +297,10 @@ class ProfileProtoBuilder { void AddTrace(const ProfileStackTrace &trace, int32 count); void AddJavaInfo(const JVMPI_CallFrame &jvm_frame, perftools::profiles::Profile *profile, - perftools::profiles::Sample *sample, - StackState *stack_state); + perftools::profiles::Sample *sample); void AddNativeInfo(const JVMPI_CallFrame &jvm_frame, perftools::profiles::Profile *profile, - perftools::profiles::Sample *sample, - StackState *stack_state); + perftools::profiles::Sample *sample); void UnsampleMetrics(); MethodInfo *Method(jmethodID id); From 47d6e250416b43269095c23ba7e8b5bd0ed01b5b Mon Sep 17 00:00:00 2001 From: dthomson Date: Mon, 15 Aug 2022 18:33:09 +0000 Subject: [PATCH 11/22] Subtract one from native addresses in profiles. PiperOrigin-RevId: 467720579 --- .../javaprofiler/profile_proto_builder.cc | 36 ++++++++++++++----- .../javaprofiler/profile_proto_builder.h | 4 ++- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/third_party/javaprofiler/profile_proto_builder.cc b/third_party/javaprofiler/profile_proto_builder.cc index e34094995..87e7469ab 100644 --- a/third_party/javaprofiler/profile_proto_builder.cc +++ b/third_party/javaprofiler/profile_proto_builder.cc @@ -75,7 +75,7 @@ void ProfileProtoBuilder::AddTraces(const ProfileStackTrace *traces, void ProfileProtoBuilder::AddArtificialTrace(const std::string &name, int count, int sampling_rate) { perftools::profiles::Location *location = location_builder_.LocationFor( - name, name, "", -1); + name, name, "", -1, 0); auto profile = builder_.mutable_profile(); auto sample = profile->add_sample(); @@ -198,7 +198,7 @@ void ProfileProtoBuilder::AddJavaInfo( perftools::profiles::Sample *sample) { if (!jvm_frame.method_id) { perftools::profiles::Location *location = location_builder_.LocationFor( - "", "[Unknown method]", "", 0); + "", "[Unknown method]", "", 0, 0); sample->add_location_id(location->id()); return; } @@ -208,7 +208,7 @@ void ProfileProtoBuilder::AddJavaInfo( } int64 ProfileProtoBuilder::Location(MethodInfo *method, - const JVMPI_CallFrame &frame) { + const JVMPI_CallFrame &frame) { // lineno is actually the BCI of the frame. int bci = frame.lineno; @@ -225,7 +225,8 @@ int64 ProfileProtoBuilder::Location(MethodInfo *method, location_builder_.LocationFor(method->ClassName(), method->MethodName(), method->FileName(), - line_number); + line_number, + 0); method->AddLocation(bci, location->id()); return location->id(); @@ -262,8 +263,9 @@ void ProfileProtoBuilder::AddNativeInfo(const JVMPI_CallFrame &jvm_frame, perftools::profiles::Profile *profile, perftools::profiles::Sample *sample) { if (!native_cache_) { + int64_t address = reinterpret_cast(jvm_frame.method_id); perftools::profiles::Location *location = location_builder_.LocationFor( - "", "[Unknown non-Java frame]", "", 0); + "", "[Unknown non-Java frame]", "", 0, address); sample->add_location_id(location->id()); return; } @@ -279,7 +281,6 @@ void ProfileProtoBuilder::AddNativeInfo(const JVMPI_CallFrame &jvm_frame, &location_builder_); - location->set_address(reinterpret_cast(jvm_frame.method_id)); sample->add_location_id(location->id()); } @@ -310,6 +311,7 @@ size_t LocationBuilder::LocationInfoHash::operator()( h = 31U * h + hash_string(info.function_name); h = 31U * h + hash_string(info.file_name); h = 31U * h + hash_int(info.line_number); + h = 31U * h + hash_int(info.address); return h; } @@ -319,15 +321,30 @@ bool LocationBuilder::LocationInfoEquals::operator()( return info1.class_name == info2.class_name && info1.function_name == info2.function_name && info1.file_name == info2.file_name && - info1.line_number == info2.line_number; + info1.line_number == info2.line_number && + info1.address == info2.address; } perftools::profiles::Location *LocationBuilder::LocationFor( const std::string &class_name, const std::string &function_name, - const std::string &file_name, int line_number) { + const std::string &file_name, int line_number, int64_t address) { auto profile = builder_->mutable_profile(); - LocationInfo info{ class_name, function_name, file_name, line_number }; + // Adjust address by -1, so that the address is within the call instruction + // instead of at the start of the return instruction, as per + // go/new-z-page#ret-addr-fixup + // + // We use 0 (NULL) to indicate the absence of a valid address, and so + // do not do this where it would be invalid i.e. leading to -1 + address = (address > 0) ? address - 1 : 0; + + LocationInfo info { + class_name, + function_name, + file_name, + line_number, + address + }; if (locations_.count(info) > 0) { return locations_.find(info)->second; @@ -336,6 +353,7 @@ perftools::profiles::Location *LocationBuilder::LocationFor( auto location_id = profile->location_size() + 1; perftools::profiles::Location *location = profile->add_location(); location->set_id(location_id); + location->set_address(address); locations_[info] = location; diff --git a/third_party/javaprofiler/profile_proto_builder.h b/third_party/javaprofiler/profile_proto_builder.h index 8e49cfcce..edc9ffc41 100644 --- a/third_party/javaprofiler/profile_proto_builder.h +++ b/third_party/javaprofiler/profile_proto_builder.h @@ -168,7 +168,8 @@ class LocationBuilder { perftools::profiles::Location *LocationFor(const std::string &class_name, const std::string &function_name, const std::string &file_name, - int line_number); + int line_number, + int64_t address); private: struct LocationInfo { @@ -176,6 +177,7 @@ class LocationBuilder { std::string function_name; std::string file_name; int line_number; + int64_t address; }; struct LocationInfoHash { From 4fa940a5b46a9035e11d6260c7df4a3edaff3c86 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Mon, 22 Aug 2022 02:14:55 +0000 Subject: [PATCH 12/22] Enable `HeapMonitor` to register, store, and invoke allocation instrumentation functions for TLAB. PiperOrigin-RevId: 469077486 --- third_party/javaprofiler/heap_sampler.cc | 57 ++++++++++++++++++++++++ third_party/javaprofiler/heap_sampler.h | 18 ++++++++ 2 files changed, 75 insertions(+) diff --git a/third_party/javaprofiler/heap_sampler.cc b/third_party/javaprofiler/heap_sampler.cc index 09a91c7af..7b5cd6914 100644 --- a/third_party/javaprofiler/heap_sampler.cc +++ b/third_party/javaprofiler/heap_sampler.cc @@ -69,12 +69,46 @@ std::unique_ptr> GetTraceUsingJvmti( new std::vector(TransformFrames(stack_frames, count))); } +static jstring GetClassName(JNIEnv *jni_env, jobject object, + jclass object_klass) { + jclass cls = jni_env->FindClass("java/lang/Class"); + jmethodID get_name_id = + jni_env->GetMethodID(cls, "getName", "()Ljava/lang/String;"); + jstring name_obj = static_cast( + jni_env->CallObjectMethod(object_klass, get_name_id)); + return name_obj; +} + +static jlong GetThreadId(JNIEnv *jni_env, jthread thread) { + jclass thread_class = jni_env->FindClass("java/lang/Thread"); + jmethodID get_id_method_id = jni_env->GetMethodID(thread_class, + "getId", "()J"); + jlong thread_id = jni_env->CallLongMethod(thread, get_id_method_id); + return thread_id; +} + extern "C" JNIEXPORT void SampledObjectAlloc(jvmtiEnv *jvmti_env, JNIEnv *jni_env, jthread thread, jobject object, jclass object_klass, jlong size) { google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, object_klass, size); + if (!google::javaprofiler::HeapMonitor::HasAllocationInstrumentation()) { + return; + } + jstring class_name = GetClassName(jni_env, object, object_klass); + const char* name_str = jni_env->GetStringUTFChars(class_name, NULL); + int name_len = strlen(name_str); + + jlong thread_id = GetThreadId(jni_env, thread); + + jbyte *name_bytes = const_cast( + reinterpret_cast(name_str)); + + // Invoke all functions in HeapMonitor::alloc_inst_functions_ + google::javaprofiler::HeapMonitor::InvokeAllocationInstrumentationFunctions( + thread_id, name_bytes, name_len, size, 0); + jni_env->ReleaseStringUTFChars(class_name, name_str); } extern "C" JNIEXPORT void GarbageCollectionFinish(jvmtiEnv *jvmti_env) { @@ -89,6 +123,8 @@ namespace javaprofiler { std::atomic HeapMonitor::jvmti_; std::atomic HeapMonitor::sampling_interval_; std::atomic HeapMonitor::use_jvm_trace_; +std::vector + HeapMonitor::alloc_inst_functions_; HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, int max_garbage_size) @@ -307,6 +343,27 @@ void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, std::move(*trace)); } +void HeapMonitor::InvokeAllocationInstrumentationFunctions(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext) { + std::vector fn_list = + GetInstance()->alloc_inst_functions_; + for (auto fn : fn_list) { + fn(thread_id, name, name_length, size, gcontext); + } +} + +void HeapMonitor::AddAllocationInstrumentation( + AllocationInstrumentationFunction fn) { + GetInstance()->alloc_inst_functions_.push_back(fn); +} + +bool HeapMonitor::HasAllocationInstrumentation() { + return !GetInstance()->alloc_inst_functions_.empty(); +} + void HeapMonitor::AddCallback(jvmtiEventCallbacks *callbacks) { #ifdef ENABLE_HEAP_SAMPLING callbacks->SampledObjectAlloc = &SampledObjectAlloc; diff --git a/third_party/javaprofiler/heap_sampler.h b/third_party/javaprofiler/heap_sampler.h index b5997f11c..97b76d507 100644 --- a/third_party/javaprofiler/heap_sampler.h +++ b/third_party/javaprofiler/heap_sampler.h @@ -29,6 +29,12 @@ #include "third_party/javaprofiler/profile_proto_builder.h" +typedef void (*AllocationInstrumentationFunction)(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext); + namespace google { namespace javaprofiler { @@ -213,6 +219,17 @@ class HeapMonitor { static void AddSample(JNIEnv *jni_env, jthread thread, jobject object, jclass object_klass, jlong size); + static void InvokeAllocationInstrumentationFunctions(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext); + + static void AddAllocationInstrumentation( + AllocationInstrumentationFunction fn); + + static bool HasAllocationInstrumentation(); + static void AddCallback(jvmtiEventCallbacks *callbacks); static void NotifyGCWaitingThread() { @@ -259,6 +276,7 @@ class HeapMonitor { void CompactData(JNIEnv* jni_env); + static std::vector alloc_inst_functions_; static std::unique_ptr EmptyHeapProfile( JNIEnv *jni_env); From 7df96633d4955dd9037896963a55f0e204ed8181 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Fri, 2 Sep 2022 16:13:25 +0000 Subject: [PATCH 13/22] Enable `HeapMonitor` and `HeapEventStorage` to register, store, and invoke garbage instrumentation functions for TLAB. PiperOrigin-RevId: 471816315 --- third_party/javaprofiler/heap_sampler.cc | 59 ++++++++++++++++++------ third_party/javaprofiler/heap_sampler.h | 47 +++++++++++++++++-- 2 files changed, 87 insertions(+), 19 deletions(-) diff --git a/third_party/javaprofiler/heap_sampler.cc b/third_party/javaprofiler/heap_sampler.cc index 7b5cd6914..a0e55e23f 100644 --- a/third_party/javaprofiler/heap_sampler.cc +++ b/third_party/javaprofiler/heap_sampler.cc @@ -91,20 +91,24 @@ extern "C" JNIEXPORT void SampledObjectAlloc(jvmtiEnv *jvmti_env, JNIEnv *jni_env, jthread thread, jobject object, jclass object_klass, jlong size) { - google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, - object_klass, size); - if (!google::javaprofiler::HeapMonitor::HasAllocationInstrumentation()) { + if (!google::javaprofiler::HeapMonitor::HasAllocationInstrumentation() && + !google::javaprofiler::HeapMonitor::HasGarbageInstrumentation()) { + google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, + object_klass, size, nullptr, 0, + 0); return; } jstring class_name = GetClassName(jni_env, object, object_klass); - const char* name_str = jni_env->GetStringUTFChars(class_name, NULL); + const char *name_str = jni_env->GetStringUTFChars(class_name, NULL); int name_len = strlen(name_str); + jbyte *name_bytes = const_cast( + reinterpret_cast(name_str)); jlong thread_id = GetThreadId(jni_env, thread); - jbyte *name_bytes = const_cast( - reinterpret_cast(name_str)); - + google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, + object_klass, size, name_bytes, + name_len, thread_id); // Invoke all functions in HeapMonitor::alloc_inst_functions_ google::javaprofiler::HeapMonitor::InvokeAllocationInstrumentationFunctions( thread_id, name_bytes, name_len, size, 0); @@ -125,6 +129,8 @@ std::atomic HeapMonitor::sampling_interval_; std::atomic HeapMonitor::use_jvm_trace_; std::vector HeapMonitor::alloc_inst_functions_; +std::vector + HeapMonitor::gc_inst_functions_; HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, int max_garbage_size) @@ -136,14 +142,16 @@ HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, void HeapEventStorage::Add(JNIEnv *jni, jthread thread, jobject object, jclass klass, jlong size, - const std::vector &&frames) { + const std::vector &&frames, + jbyte *name, jint name_len, jlong thread_id) { jweak weak_ref = jni->NewWeakGlobalRef(object); if (jni->ExceptionCheck()) { LOG(WARNING) << "Failed to create NewWeakGlobalRef, skipping heap sample"; return; } - HeapObjectTrace live_object(weak_ref, size, std::move(frames)); + HeapObjectTrace live_object(weak_ref, size, std::move(frames), name, name_len, + thread_id); // Only now lock and get things done quickly. std::lock_guard lock(storage_lock_); @@ -166,6 +174,11 @@ void HeapEventStorage::MoveLiveObjects( if (elem.IsLive(env)) { still_live_objects->push_back(std::move(elem)); } else { + HeapMonitor::InvokeGarbageInstrumentationFunctions(elem.ThreadId(), + elem.Name(), + elem.NameLength(), + elem.Size(), + 0); elem.DeleteWeakReference(env); AddToGarbage(std::move(elem)); } @@ -331,7 +344,8 @@ bool HeapMonitor::Supported(jvmtiEnv *jvmti) { } void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, - jclass object_klass, jlong size) { + jclass object_klass, jlong size, jbyte *name, + jint name_len, jlong thread_id) { auto trace = use_jvm_trace_.load() ? GetTrace(jni_env) : GetTraceUsingJvmti(jni_env, jvmti_.load(), thread); @@ -340,7 +354,7 @@ void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, } GetInstance()->storage_.Add(jni_env, thread, object, object_klass, size, - std::move(*trace)); + std::move(*trace), name, name_len, thread_id); } void HeapMonitor::InvokeAllocationInstrumentationFunctions(jlong thread_id, @@ -348,9 +362,7 @@ void HeapMonitor::InvokeAllocationInstrumentationFunctions(jlong thread_id, int name_length, int size, jlong gcontext) { - std::vector fn_list = - GetInstance()->alloc_inst_functions_; - for (auto fn : fn_list) { + for (auto fn : GetInstance()->alloc_inst_functions_) { fn(thread_id, name, name_length, size, gcontext); } } @@ -364,6 +376,25 @@ bool HeapMonitor::HasAllocationInstrumentation() { return !GetInstance()->alloc_inst_functions_.empty(); } +bool HeapMonitor::HasGarbageInstrumentation() { + return !GetInstance()->gc_inst_functions_.empty(); +} + +void HeapMonitor::InvokeGarbageInstrumentationFunctions(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext) { + for (auto fn : GetInstance()->gc_inst_functions_) { + fn(thread_id, name, name_length, size, gcontext); + } +} + +void HeapMonitor::AddGarbageInstrumentation( + GarbageInstrumentationFunction fn) { + GetInstance()->gc_inst_functions_.push_back(fn); +} + void HeapMonitor::AddCallback(jvmtiEventCallbacks *callbacks) { #ifdef ENABLE_HEAP_SAMPLING callbacks->SampledObjectAlloc = &SampledObjectAlloc; diff --git a/third_party/javaprofiler/heap_sampler.h b/third_party/javaprofiler/heap_sampler.h index 97b76d507..6e029bc47 100644 --- a/third_party/javaprofiler/heap_sampler.h +++ b/third_party/javaprofiler/heap_sampler.h @@ -35,6 +35,12 @@ typedef void (*AllocationInstrumentationFunction)(jlong thread_id, int size, jlong gcontext); +typedef void (*GarbageInstrumentationFunction)(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext); + namespace google { namespace javaprofiler { @@ -51,7 +57,8 @@ class HeapEventStorage { // Adds an object to the storage system. void Add(JNIEnv *jni, jthread thread, jobject object, jclass klass, - jlong size, const std::vector &&frames); + jlong size, const std::vector &&frames, jbyte *name, + jint name_len, jlong thread_id); // Returns a perftools::profiles::Profile with the objects stored via // calls to the Add method. @@ -91,8 +98,10 @@ class HeapEventStorage { // This object owns the jweak object parameter. It is freed when the object // is sent to the garbage list, and the object is set to nullptr. HeapObjectTrace(jweak object, jlong size, - const std::vector &&frames) - : object_(object), size_(size), frames_(std::move(frames)) {} + const std::vector &&frames, jbyte *name, + int name_length, jlong thread_id) + : object_(object), size_(size), frames_(std::move(frames)), + name_(name), name_length_(name_length), thread_id_(thread_id) {} HeapObjectTrace(jweak object, jlong size, const std::vector &frames) @@ -114,6 +123,18 @@ class HeapEventStorage { return size_; } + jbyte *Name() const { + return name_; + } + + int NameLength() const { + return name_length_; + } + + jlong ThreadId() const { + return thread_id_; + } + void DeleteWeakReference(JNIEnv* env) { env->DeleteWeakGlobalRef(object_); object_ = nullptr; @@ -135,6 +156,9 @@ class HeapEventStorage { jweak object_; int size_; std::vector frames_; + jbyte *name_; + int name_length_; + jlong thread_id_; }; // Helper for creating a google::javaprofiler::ProfileStackTrace array @@ -152,7 +176,8 @@ class HeapEventStorage { private: int curr_trace_ = 0; std::size_t objects_size_; - std::unique_ptr stack_trace_data_; + std::unique_ptr + stack_trace_data_; std::unique_ptr call_trace_data_; }; @@ -217,7 +242,8 @@ class HeapMonitor { JNIEnv* env, bool force_gc); static void AddSample(JNIEnv *jni_env, jthread thread, jobject object, - jclass object_klass, jlong size); + jclass object_klass, jlong size, jbyte *name, + jint name_len, jlong thread_id); static void InvokeAllocationInstrumentationFunctions(jlong thread_id, jbyte *name, @@ -230,6 +256,16 @@ class HeapMonitor { static bool HasAllocationInstrumentation(); + static void InvokeGarbageInstrumentationFunctions(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext); + + static void AddGarbageInstrumentation(GarbageInstrumentationFunction fn); + + static bool HasGarbageInstrumentation(); + static void AddCallback(jvmtiEventCallbacks *callbacks); static void NotifyGCWaitingThread() { @@ -277,6 +313,7 @@ class HeapMonitor { void CompactData(JNIEnv* jni_env); static std::vector alloc_inst_functions_; + static std::vector gc_inst_functions_; static std::unique_ptr EmptyHeapProfile( JNIEnv *jni_env); From dc8f15784ee40c9d41d8f218464e7609881251b2 Mon Sep 17 00:00:00 2001 From: dthomson Date: Fri, 2 Sep 2022 18:22:18 +0000 Subject: [PATCH 14/22] Enable `HeapMonitor` and `HeapEventStorage` to register, store, and invoke garbage instrumentation functions for TLAB. PiperOrigin-RevId: 471845478 --- third_party/javaprofiler/heap_sampler.cc | 59 ++++++------------------ third_party/javaprofiler/heap_sampler.h | 47 ++----------------- 2 files changed, 19 insertions(+), 87 deletions(-) diff --git a/third_party/javaprofiler/heap_sampler.cc b/third_party/javaprofiler/heap_sampler.cc index a0e55e23f..7b5cd6914 100644 --- a/third_party/javaprofiler/heap_sampler.cc +++ b/third_party/javaprofiler/heap_sampler.cc @@ -91,24 +91,20 @@ extern "C" JNIEXPORT void SampledObjectAlloc(jvmtiEnv *jvmti_env, JNIEnv *jni_env, jthread thread, jobject object, jclass object_klass, jlong size) { - if (!google::javaprofiler::HeapMonitor::HasAllocationInstrumentation() && - !google::javaprofiler::HeapMonitor::HasGarbageInstrumentation()) { - google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, - object_klass, size, nullptr, 0, - 0); + google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, + object_klass, size); + if (!google::javaprofiler::HeapMonitor::HasAllocationInstrumentation()) { return; } jstring class_name = GetClassName(jni_env, object, object_klass); - const char *name_str = jni_env->GetStringUTFChars(class_name, NULL); + const char* name_str = jni_env->GetStringUTFChars(class_name, NULL); int name_len = strlen(name_str); - jbyte *name_bytes = const_cast( - reinterpret_cast(name_str)); jlong thread_id = GetThreadId(jni_env, thread); - google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, - object_klass, size, name_bytes, - name_len, thread_id); + jbyte *name_bytes = const_cast( + reinterpret_cast(name_str)); + // Invoke all functions in HeapMonitor::alloc_inst_functions_ google::javaprofiler::HeapMonitor::InvokeAllocationInstrumentationFunctions( thread_id, name_bytes, name_len, size, 0); @@ -129,8 +125,6 @@ std::atomic HeapMonitor::sampling_interval_; std::atomic HeapMonitor::use_jvm_trace_; std::vector HeapMonitor::alloc_inst_functions_; -std::vector - HeapMonitor::gc_inst_functions_; HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, int max_garbage_size) @@ -142,16 +136,14 @@ HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, void HeapEventStorage::Add(JNIEnv *jni, jthread thread, jobject object, jclass klass, jlong size, - const std::vector &&frames, - jbyte *name, jint name_len, jlong thread_id) { + const std::vector &&frames) { jweak weak_ref = jni->NewWeakGlobalRef(object); if (jni->ExceptionCheck()) { LOG(WARNING) << "Failed to create NewWeakGlobalRef, skipping heap sample"; return; } - HeapObjectTrace live_object(weak_ref, size, std::move(frames), name, name_len, - thread_id); + HeapObjectTrace live_object(weak_ref, size, std::move(frames)); // Only now lock and get things done quickly. std::lock_guard lock(storage_lock_); @@ -174,11 +166,6 @@ void HeapEventStorage::MoveLiveObjects( if (elem.IsLive(env)) { still_live_objects->push_back(std::move(elem)); } else { - HeapMonitor::InvokeGarbageInstrumentationFunctions(elem.ThreadId(), - elem.Name(), - elem.NameLength(), - elem.Size(), - 0); elem.DeleteWeakReference(env); AddToGarbage(std::move(elem)); } @@ -344,8 +331,7 @@ bool HeapMonitor::Supported(jvmtiEnv *jvmti) { } void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, - jclass object_klass, jlong size, jbyte *name, - jint name_len, jlong thread_id) { + jclass object_klass, jlong size) { auto trace = use_jvm_trace_.load() ? GetTrace(jni_env) : GetTraceUsingJvmti(jni_env, jvmti_.load(), thread); @@ -354,7 +340,7 @@ void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, } GetInstance()->storage_.Add(jni_env, thread, object, object_klass, size, - std::move(*trace), name, name_len, thread_id); + std::move(*trace)); } void HeapMonitor::InvokeAllocationInstrumentationFunctions(jlong thread_id, @@ -362,7 +348,9 @@ void HeapMonitor::InvokeAllocationInstrumentationFunctions(jlong thread_id, int name_length, int size, jlong gcontext) { - for (auto fn : GetInstance()->alloc_inst_functions_) { + std::vector fn_list = + GetInstance()->alloc_inst_functions_; + for (auto fn : fn_list) { fn(thread_id, name, name_length, size, gcontext); } } @@ -376,25 +364,6 @@ bool HeapMonitor::HasAllocationInstrumentation() { return !GetInstance()->alloc_inst_functions_.empty(); } -bool HeapMonitor::HasGarbageInstrumentation() { - return !GetInstance()->gc_inst_functions_.empty(); -} - -void HeapMonitor::InvokeGarbageInstrumentationFunctions(jlong thread_id, - jbyte *name, - int name_length, - int size, - jlong gcontext) { - for (auto fn : GetInstance()->gc_inst_functions_) { - fn(thread_id, name, name_length, size, gcontext); - } -} - -void HeapMonitor::AddGarbageInstrumentation( - GarbageInstrumentationFunction fn) { - GetInstance()->gc_inst_functions_.push_back(fn); -} - void HeapMonitor::AddCallback(jvmtiEventCallbacks *callbacks) { #ifdef ENABLE_HEAP_SAMPLING callbacks->SampledObjectAlloc = &SampledObjectAlloc; diff --git a/third_party/javaprofiler/heap_sampler.h b/third_party/javaprofiler/heap_sampler.h index 6e029bc47..97b76d507 100644 --- a/third_party/javaprofiler/heap_sampler.h +++ b/third_party/javaprofiler/heap_sampler.h @@ -35,12 +35,6 @@ typedef void (*AllocationInstrumentationFunction)(jlong thread_id, int size, jlong gcontext); -typedef void (*GarbageInstrumentationFunction)(jlong thread_id, - jbyte *name, - int name_length, - int size, - jlong gcontext); - namespace google { namespace javaprofiler { @@ -57,8 +51,7 @@ class HeapEventStorage { // Adds an object to the storage system. void Add(JNIEnv *jni, jthread thread, jobject object, jclass klass, - jlong size, const std::vector &&frames, jbyte *name, - jint name_len, jlong thread_id); + jlong size, const std::vector &&frames); // Returns a perftools::profiles::Profile with the objects stored via // calls to the Add method. @@ -98,10 +91,8 @@ class HeapEventStorage { // This object owns the jweak object parameter. It is freed when the object // is sent to the garbage list, and the object is set to nullptr. HeapObjectTrace(jweak object, jlong size, - const std::vector &&frames, jbyte *name, - int name_length, jlong thread_id) - : object_(object), size_(size), frames_(std::move(frames)), - name_(name), name_length_(name_length), thread_id_(thread_id) {} + const std::vector &&frames) + : object_(object), size_(size), frames_(std::move(frames)) {} HeapObjectTrace(jweak object, jlong size, const std::vector &frames) @@ -123,18 +114,6 @@ class HeapEventStorage { return size_; } - jbyte *Name() const { - return name_; - } - - int NameLength() const { - return name_length_; - } - - jlong ThreadId() const { - return thread_id_; - } - void DeleteWeakReference(JNIEnv* env) { env->DeleteWeakGlobalRef(object_); object_ = nullptr; @@ -156,9 +135,6 @@ class HeapEventStorage { jweak object_; int size_; std::vector frames_; - jbyte *name_; - int name_length_; - jlong thread_id_; }; // Helper for creating a google::javaprofiler::ProfileStackTrace array @@ -176,8 +152,7 @@ class HeapEventStorage { private: int curr_trace_ = 0; std::size_t objects_size_; - std::unique_ptr - stack_trace_data_; + std::unique_ptr stack_trace_data_; std::unique_ptr call_trace_data_; }; @@ -242,8 +217,7 @@ class HeapMonitor { JNIEnv* env, bool force_gc); static void AddSample(JNIEnv *jni_env, jthread thread, jobject object, - jclass object_klass, jlong size, jbyte *name, - jint name_len, jlong thread_id); + jclass object_klass, jlong size); static void InvokeAllocationInstrumentationFunctions(jlong thread_id, jbyte *name, @@ -256,16 +230,6 @@ class HeapMonitor { static bool HasAllocationInstrumentation(); - static void InvokeGarbageInstrumentationFunctions(jlong thread_id, - jbyte *name, - int name_length, - int size, - jlong gcontext); - - static void AddGarbageInstrumentation(GarbageInstrumentationFunction fn); - - static bool HasGarbageInstrumentation(); - static void AddCallback(jvmtiEventCallbacks *callbacks); static void NotifyGCWaitingThread() { @@ -313,7 +277,6 @@ class HeapMonitor { void CompactData(JNIEnv* jni_env); static std::vector alloc_inst_functions_; - static std::vector gc_inst_functions_; static std::unique_ptr EmptyHeapProfile( JNIEnv *jni_env); From 5f9e12cf337a66e6fcb31716d9e86d9caa780040 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Wed, 14 Sep 2022 00:06:55 +0000 Subject: [PATCH 15/22] Enable `HeapMonitor` and `HeapEventStorage` to register, store, and invoke garbage instrumentation functions for TLAB. PiperOrigin-RevId: 474160995 --- third_party/javaprofiler/heap_sampler.cc | 59 ++++++++++++++++++------ third_party/javaprofiler/heap_sampler.h | 47 +++++++++++++++++-- 2 files changed, 87 insertions(+), 19 deletions(-) diff --git a/third_party/javaprofiler/heap_sampler.cc b/third_party/javaprofiler/heap_sampler.cc index 7b5cd6914..a0e55e23f 100644 --- a/third_party/javaprofiler/heap_sampler.cc +++ b/third_party/javaprofiler/heap_sampler.cc @@ -91,20 +91,24 @@ extern "C" JNIEXPORT void SampledObjectAlloc(jvmtiEnv *jvmti_env, JNIEnv *jni_env, jthread thread, jobject object, jclass object_klass, jlong size) { - google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, - object_klass, size); - if (!google::javaprofiler::HeapMonitor::HasAllocationInstrumentation()) { + if (!google::javaprofiler::HeapMonitor::HasAllocationInstrumentation() && + !google::javaprofiler::HeapMonitor::HasGarbageInstrumentation()) { + google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, + object_klass, size, nullptr, 0, + 0); return; } jstring class_name = GetClassName(jni_env, object, object_klass); - const char* name_str = jni_env->GetStringUTFChars(class_name, NULL); + const char *name_str = jni_env->GetStringUTFChars(class_name, NULL); int name_len = strlen(name_str); + jbyte *name_bytes = const_cast( + reinterpret_cast(name_str)); jlong thread_id = GetThreadId(jni_env, thread); - jbyte *name_bytes = const_cast( - reinterpret_cast(name_str)); - + google::javaprofiler::HeapMonitor::AddSample(jni_env, thread, object, + object_klass, size, name_bytes, + name_len, thread_id); // Invoke all functions in HeapMonitor::alloc_inst_functions_ google::javaprofiler::HeapMonitor::InvokeAllocationInstrumentationFunctions( thread_id, name_bytes, name_len, size, 0); @@ -125,6 +129,8 @@ std::atomic HeapMonitor::sampling_interval_; std::atomic HeapMonitor::use_jvm_trace_; std::vector HeapMonitor::alloc_inst_functions_; +std::vector + HeapMonitor::gc_inst_functions_; HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, int max_garbage_size) @@ -136,14 +142,16 @@ HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, void HeapEventStorage::Add(JNIEnv *jni, jthread thread, jobject object, jclass klass, jlong size, - const std::vector &&frames) { + const std::vector &&frames, + jbyte *name, jint name_len, jlong thread_id) { jweak weak_ref = jni->NewWeakGlobalRef(object); if (jni->ExceptionCheck()) { LOG(WARNING) << "Failed to create NewWeakGlobalRef, skipping heap sample"; return; } - HeapObjectTrace live_object(weak_ref, size, std::move(frames)); + HeapObjectTrace live_object(weak_ref, size, std::move(frames), name, name_len, + thread_id); // Only now lock and get things done quickly. std::lock_guard lock(storage_lock_); @@ -166,6 +174,11 @@ void HeapEventStorage::MoveLiveObjects( if (elem.IsLive(env)) { still_live_objects->push_back(std::move(elem)); } else { + HeapMonitor::InvokeGarbageInstrumentationFunctions(elem.ThreadId(), + elem.Name(), + elem.NameLength(), + elem.Size(), + 0); elem.DeleteWeakReference(env); AddToGarbage(std::move(elem)); } @@ -331,7 +344,8 @@ bool HeapMonitor::Supported(jvmtiEnv *jvmti) { } void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, - jclass object_klass, jlong size) { + jclass object_klass, jlong size, jbyte *name, + jint name_len, jlong thread_id) { auto trace = use_jvm_trace_.load() ? GetTrace(jni_env) : GetTraceUsingJvmti(jni_env, jvmti_.load(), thread); @@ -340,7 +354,7 @@ void HeapMonitor::AddSample(JNIEnv *jni_env, jthread thread, jobject object, } GetInstance()->storage_.Add(jni_env, thread, object, object_klass, size, - std::move(*trace)); + std::move(*trace), name, name_len, thread_id); } void HeapMonitor::InvokeAllocationInstrumentationFunctions(jlong thread_id, @@ -348,9 +362,7 @@ void HeapMonitor::InvokeAllocationInstrumentationFunctions(jlong thread_id, int name_length, int size, jlong gcontext) { - std::vector fn_list = - GetInstance()->alloc_inst_functions_; - for (auto fn : fn_list) { + for (auto fn : GetInstance()->alloc_inst_functions_) { fn(thread_id, name, name_length, size, gcontext); } } @@ -364,6 +376,25 @@ bool HeapMonitor::HasAllocationInstrumentation() { return !GetInstance()->alloc_inst_functions_.empty(); } +bool HeapMonitor::HasGarbageInstrumentation() { + return !GetInstance()->gc_inst_functions_.empty(); +} + +void HeapMonitor::InvokeGarbageInstrumentationFunctions(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext) { + for (auto fn : GetInstance()->gc_inst_functions_) { + fn(thread_id, name, name_length, size, gcontext); + } +} + +void HeapMonitor::AddGarbageInstrumentation( + GarbageInstrumentationFunction fn) { + GetInstance()->gc_inst_functions_.push_back(fn); +} + void HeapMonitor::AddCallback(jvmtiEventCallbacks *callbacks) { #ifdef ENABLE_HEAP_SAMPLING callbacks->SampledObjectAlloc = &SampledObjectAlloc; diff --git a/third_party/javaprofiler/heap_sampler.h b/third_party/javaprofiler/heap_sampler.h index 97b76d507..6e029bc47 100644 --- a/third_party/javaprofiler/heap_sampler.h +++ b/third_party/javaprofiler/heap_sampler.h @@ -35,6 +35,12 @@ typedef void (*AllocationInstrumentationFunction)(jlong thread_id, int size, jlong gcontext); +typedef void (*GarbageInstrumentationFunction)(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext); + namespace google { namespace javaprofiler { @@ -51,7 +57,8 @@ class HeapEventStorage { // Adds an object to the storage system. void Add(JNIEnv *jni, jthread thread, jobject object, jclass klass, - jlong size, const std::vector &&frames); + jlong size, const std::vector &&frames, jbyte *name, + jint name_len, jlong thread_id); // Returns a perftools::profiles::Profile with the objects stored via // calls to the Add method. @@ -91,8 +98,10 @@ class HeapEventStorage { // This object owns the jweak object parameter. It is freed when the object // is sent to the garbage list, and the object is set to nullptr. HeapObjectTrace(jweak object, jlong size, - const std::vector &&frames) - : object_(object), size_(size), frames_(std::move(frames)) {} + const std::vector &&frames, jbyte *name, + int name_length, jlong thread_id) + : object_(object), size_(size), frames_(std::move(frames)), + name_(name), name_length_(name_length), thread_id_(thread_id) {} HeapObjectTrace(jweak object, jlong size, const std::vector &frames) @@ -114,6 +123,18 @@ class HeapEventStorage { return size_; } + jbyte *Name() const { + return name_; + } + + int NameLength() const { + return name_length_; + } + + jlong ThreadId() const { + return thread_id_; + } + void DeleteWeakReference(JNIEnv* env) { env->DeleteWeakGlobalRef(object_); object_ = nullptr; @@ -135,6 +156,9 @@ class HeapEventStorage { jweak object_; int size_; std::vector frames_; + jbyte *name_; + int name_length_; + jlong thread_id_; }; // Helper for creating a google::javaprofiler::ProfileStackTrace array @@ -152,7 +176,8 @@ class HeapEventStorage { private: int curr_trace_ = 0; std::size_t objects_size_; - std::unique_ptr stack_trace_data_; + std::unique_ptr + stack_trace_data_; std::unique_ptr call_trace_data_; }; @@ -217,7 +242,8 @@ class HeapMonitor { JNIEnv* env, bool force_gc); static void AddSample(JNIEnv *jni_env, jthread thread, jobject object, - jclass object_klass, jlong size); + jclass object_klass, jlong size, jbyte *name, + jint name_len, jlong thread_id); static void InvokeAllocationInstrumentationFunctions(jlong thread_id, jbyte *name, @@ -230,6 +256,16 @@ class HeapMonitor { static bool HasAllocationInstrumentation(); + static void InvokeGarbageInstrumentationFunctions(jlong thread_id, + jbyte *name, + int name_length, + int size, + jlong gcontext); + + static void AddGarbageInstrumentation(GarbageInstrumentationFunction fn); + + static bool HasGarbageInstrumentation(); + static void AddCallback(jvmtiEventCallbacks *callbacks); static void NotifyGCWaitingThread() { @@ -277,6 +313,7 @@ class HeapMonitor { void CompactData(JNIEnv* jni_env); static std::vector alloc_inst_functions_; + static std::vector gc_inst_functions_; static std::unique_ptr EmptyHeapProfile( JNIEnv *jni_env); From 4d4d81d969c987773e87df08a1015bff019b13d3 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Wed, 28 Sep 2022 16:23:39 +0000 Subject: [PATCH 16/22] fix: Enable test-only builds on ARM64 PiperOrigin-RevId: 477471509 --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index ec5ccbd9c..97099397f 100755 --- a/build.sh +++ b/build.sh @@ -38,7 +38,7 @@ while getopts ":d:m:v:" opt; do m) TARGET=$OPTARG if [[ "${TARGET}" == "arm64" ]]; then - if [[ "${MACHINE_TYPE}" != "aarch64" ]] || [[ "${MACHINE_TYPE}" != "arm64" ]]; then + if [[ "${MACHINE_TYPE}" != "aarch64" ]] && [[ "${MACHINE_TYPE}" != "arm64" ]]; then echo "-m arm64 is supported only when running on an ARM64 system." exit; fi From 911084cb037822b52afe59bafeca60e908bfe3f0 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Fri, 28 Oct 2022 20:26:35 +0000 Subject: [PATCH 17/22] Consolidate constant for stack size limit. PiperOrigin-RevId: 484607327 --- third_party/javaprofiler/heap_sampler.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/javaprofiler/heap_sampler.cc b/third_party/javaprofiler/heap_sampler.cc index a0e55e23f..e51cb736a 100644 --- a/third_party/javaprofiler/heap_sampler.cc +++ b/third_party/javaprofiler/heap_sampler.cc @@ -54,12 +54,12 @@ std::unique_ptr> GetTrace(JNIEnv *jni) { std::unique_ptr> GetTraceUsingJvmti( JNIEnv *jni, jvmtiEnv *jvmti, jthread thread) { - const int kMaxFrames = 128; jint count = 0; - jvmtiFrameInfo stack_frames[kMaxFrames]; + jvmtiFrameInfo stack_frames[google::javaprofiler::kMaxFramesToCapture]; jvmtiError err = - jvmti->GetStackTrace(thread, 0, kMaxFrames, stack_frames, &count); + jvmti->GetStackTrace(thread, 0, google::javaprofiler::kMaxFramesToCapture, + stack_frames, &count); if (err != JVMTI_ERROR_NONE || count <= 0) { return nullptr; From 6c44a3ce42c354310a7142f1940082daaf7a44c9 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Mon, 14 Nov 2022 20:03:54 +0000 Subject: [PATCH 18/22] Fill Function.start_line for Java profiles. PiperOrigin-RevId: 488432046 --- third_party/javaprofiler/display.cc | 4 ++++ third_party/javaprofiler/method_info.h | 8 +++++-- .../javaprofiler/profile_proto_builder.cc | 22 +++++++++++-------- .../javaprofiler/profile_proto_builder.h | 1 + 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/third_party/javaprofiler/display.cc b/third_party/javaprofiler/display.cc index 86b867880..bbdc37834 100644 --- a/third_party/javaprofiler/display.cc +++ b/third_party/javaprofiler/display.cc @@ -174,6 +174,10 @@ jint GetLineNumber(jvmtiEnv *jvmti, jmethodID method, jlocation location) { if (entry_count == 1) { return table_ptr[0].line_number; } + if (location == 0) { + // Return method first line. + return table_ptr[0].line_number; + } jlocation last_location = table_ptr[0].start_location; for (int l = 1; l < entry_count; l++) { diff --git a/third_party/javaprofiler/method_info.h b/third_party/javaprofiler/method_info.h index a66f81a5b..b3c91742e 100644 --- a/third_party/javaprofiler/method_info.h +++ b/third_party/javaprofiler/method_info.h @@ -34,10 +34,11 @@ class MethodInfo { public: // Constructor providing all the information regarding a method. MethodInfo(const std::string &method_name, const std::string &class_name, - const std::string &file_name) + const std::string &file_name, const int &start_line) : method_name_(method_name), class_name_(class_name), - file_name_(file_name) {} + file_name_(file_name), + start_line_(start_line) {} // An invalid location Id. static const int64 kInvalidLocationId = 0; @@ -57,10 +58,13 @@ class MethodInfo { const std::string &FileName() const { return file_name_; } + const int& StartLine() const { return start_line_; } + private: std::string method_name_; std::string class_name_; std::string file_name_; + int start_line_; // Cache of jlocation results. std::unordered_map locations_; diff --git a/third_party/javaprofiler/profile_proto_builder.cc b/third_party/javaprofiler/profile_proto_builder.cc index 87e7469ab..4ebacbc3d 100644 --- a/third_party/javaprofiler/profile_proto_builder.cc +++ b/third_party/javaprofiler/profile_proto_builder.cc @@ -75,7 +75,7 @@ void ProfileProtoBuilder::AddTraces(const ProfileStackTrace *traces, void ProfileProtoBuilder::AddArtificialTrace(const std::string &name, int count, int sampling_rate) { perftools::profiles::Location *location = location_builder_.LocationFor( - name, name, "", -1, 0); + name, name, "", 0, -1, 0); auto profile = builder_.mutable_profile(); auto sample = profile->add_sample(); @@ -198,7 +198,7 @@ void ProfileProtoBuilder::AddJavaInfo( perftools::profiles::Sample *sample) { if (!jvm_frame.method_id) { perftools::profiles::Location *location = location_builder_.LocationFor( - "", "[Unknown method]", "", 0, 0); + "", "[Unknown method]", "", 0, 0, 0); sample->add_location_id(location->id()); return; } @@ -225,6 +225,7 @@ int64 ProfileProtoBuilder::Location(MethodInfo *method, location_builder_.LocationFor(method->ClassName(), method->MethodName(), method->FileName(), + method->StartLine(), line_number, 0); @@ -242,17 +243,18 @@ MethodInfo *ProfileProtoBuilder::Method(jmethodID method_id) { std::string class_name; std::string method_name; std::string signature; + int start_line = 0; - // Ignore lineno since we pass nullptr anyway. + // Set lineno to zero to get method first line. JVMPI_CallFrame jvm_frame = { 0, method_id }; GetStackFrameElements(jni_env_, jvmti_env_, jvm_frame, &file_name, - &class_name, &method_name, &signature, nullptr); + &class_name, &method_name, &signature, &start_line); FixMethodParameters(&signature); std::string full_method_name = class_name + "." + method_name + signature; std::unique_ptr unique_method( - new MethodInfo(full_method_name, class_name, file_name)); + new MethodInfo(full_method_name, class_name, file_name, start_line)); auto method_ptr = unique_method.get(); methods_[method_id] = std::move(unique_method); @@ -265,7 +267,7 @@ void ProfileProtoBuilder::AddNativeInfo(const JVMPI_CallFrame &jvm_frame, if (!native_cache_) { int64_t address = reinterpret_cast(jvm_frame.method_id); perftools::profiles::Location *location = location_builder_.LocationFor( - "", "[Unknown non-Java frame]", "", 0, address); + "", "[Unknown non-Java frame]", "", 0, 0, address); sample->add_location_id(location->id()); return; } @@ -327,7 +329,8 @@ bool LocationBuilder::LocationInfoEquals::operator()( perftools::profiles::Location *LocationBuilder::LocationFor( const std::string &class_name, const std::string &function_name, - const std::string &file_name, int line_number, int64_t address) { + const std::string &file_name, int start_line, int line_number, + int64_t address) { auto profile = builder_->mutable_profile(); // Adjust address by -1, so that the address is within the call instruction @@ -361,8 +364,9 @@ perftools::profiles::Location *LocationBuilder::LocationFor( std::string simplified_name = function_name; SimplifyFunctionName(&simplified_name); - auto function_id = builder_->FunctionId( - simplified_name.c_str(), function_name.c_str(), file_name.c_str(), 0); + auto function_id = + builder_->FunctionId(simplified_name.c_str(), function_name.c_str(), + file_name.c_str(), start_line); line->set_function_id(function_id); line->set_line(line_number); diff --git a/third_party/javaprofiler/profile_proto_builder.h b/third_party/javaprofiler/profile_proto_builder.h index edc9ffc41..5814a6d0e 100644 --- a/third_party/javaprofiler/profile_proto_builder.h +++ b/third_party/javaprofiler/profile_proto_builder.h @@ -168,6 +168,7 @@ class LocationBuilder { perftools::profiles::Location *LocationFor(const std::string &class_name, const std::string &function_name, const std::string &file_name, + int start_line, int line_number, int64_t address); From 153e5b6884afaecc510a338e6a3bee83fd1de4bf Mon Sep 17 00:00:00 2001 From: aalexand Date: Tue, 29 Nov 2022 21:53:47 +0000 Subject: [PATCH 19/22] Make Builder::MarshalToFile output overwritable so that repetitive profiling succeeds. PiperOrigin-RevId: 491728975 --- third_party/perftools/profiles/proto/builder.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/third_party/perftools/profiles/proto/builder.cc b/third_party/perftools/profiles/proto/builder.cc index da1f6173d..62e253bfb 100644 --- a/third_party/perftools/profiles/proto/builder.cc +++ b/third_party/perftools/profiles/proto/builder.cc @@ -121,8 +121,9 @@ bool Builder::MarshalToFile(const Profile &profile, int fd) { bool Builder::MarshalToFile(const Profile &profile, const char *filename) { int fd; - while ((fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0444)) < 0 && - errno == EINTR) {} + while ((fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0666)) < 0 && + errno == EINTR) { + } if (fd == -1) { PLOG(ERROR) << "Failed to open file " << filename; return false; From 6127512e73c02feaa01fdcaebabd3d26297413a8 Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Sat, 10 Dec 2022 04:27:16 +0000 Subject: [PATCH 20/22] Fix go install error PiperOrigin-RevId: 494325378 --- Dockerfile | 7 +++++-- Dockerfile.alpine | 7 +++++-- Makefile | 5 +++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 50c6af32a..138a8c453 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,7 @@ RUN git clone --depth=1 -b curl-7_69_1 https://github.com/curl/curl.git /tmp/cur ./buildconf && \ ./configure --disable-ldap --disable-shared --without-libssh2 \ --without-librtmp --without-libidn --enable-static \ + --without-libidn2 \ --with-pic --with-ssl=/usr/local/ssl/ && \ make -j && make install && \ cd ~ && rm -rf /tmp/curl @@ -86,13 +87,15 @@ RUN mkdir /tmp/glog && cd /tmp/glog && \ # process of gRPC puts the OpenSSL static object files into the gRPC archive # which causes link errors later when the agent is linked with the static # OpenSSL library itself. +# Limit the number of threads used by make, as unlimited threads causes +# memory exhausted error on the Kokoro VM. RUN git clone --depth=1 --recursive -b v1.28.1 https://github.com/grpc/grpc.git /tmp/grpc && \ cd /tmp/grpc && \ cd third_party/protobuf && \ ./autogen.sh && \ ./configure --with-pic CXXFLAGS="$(pkg-config --cflags protobuf)" LIBS="$(pkg-config --libs protobuf)" LDFLAGS="-Wl,--no-as-needed" && \ - make -j && make install && ldconfig && \ + make -j4 && make install && ldconfig && \ cd ../.. && \ - CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make -j CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 && \ + CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make -j4 CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 && \ CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 install && \ rm -rf /tmp/grpc diff --git a/Dockerfile.alpine b/Dockerfile.alpine index b4922a962..202537a91 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -59,6 +59,7 @@ RUN git clone --depth=1 -b curl-7_69_1 https://github.com/curl/curl.git /tmp/cur ./buildconf && \ ./configure --disable-ldap --disable-shared --without-libssh2 \ --without-librtmp --without-libidn --enable-static \ + --without-libidn2 \ --with-pic --with-ssl=/usr/local/ssl/ && \ make -j && make install && \ cd ~ && rm -rf /tmp/curl @@ -85,13 +86,15 @@ RUN mkdir /tmp/glog && cd /tmp/glog && \ # process of gRPC puts the OpenSSL static object files into the gRPC archive # which causes link errors later when the agent is linked with the static # OpenSSL library itself. +# Limit the number of threads used by make, as unlimited threads causes +# memory exhausted error on the Kokoro VM. RUN git clone --depth=1 --recursive -b v1.28.1 https://github.com/grpc/grpc.git /tmp/grpc && \ cd /tmp/grpc && \ cd third_party/protobuf && \ ./autogen.sh && \ ./configure --with-pic CXXFLAGS="$(pkg-config --cflags protobuf)" LIBS="$(pkg-config --libs protobuf)" LDFLAGS="-Wl,--no-as-needed" && \ - make -j && make install && ldconfig / && \ + make -j4 && make install && ldconfig / && \ cd ../.. && \ - CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make -j CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 && \ + CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make -j4 CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 && \ CPPFLAGS="-I /usr/local/ssl/include" LDFLAGS="-L /usr/local/ssl/lib/ -Wl,--no-as-needed" make CONFIG=opt EMBED_OPENSSL=false V=1 HAS_SYSTEM_OPENSSL_NPN=0 install && \ rm -rf /tmp/grpc diff --git a/Makefile b/Makefile index ce3b49955..15b6e6d33 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,7 @@ PROFILER_API_SOURCES = \ $(GENFILES_PATH)/google/api/annotations.pb.cc \ $(GENFILES_PATH)/google/api/client.pb.cc \ $(GENFILES_PATH)/google/api/http.pb.cc \ + $(GENFILES_PATH)/google/api/launch_stage.pb.cc \ $(GENFILES_PATH)/google/devtools/cloudprofiler/v2/profiler.grpc.pb.cc \ $(GENFILES_PATH)/google/devtools/cloudprofiler/v2/profiler.pb.cc \ $(GENFILES_PATH)/google/protobuf/duration.pb.cc \ @@ -231,6 +232,10 @@ $(GENFILES_PATH)/%annotations.pb.h $(GENFILES_PATH)/%annotations.pb.cc : third_p mkdir -p $(dir $@) $(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) --cpp_out=$(GENFILES_PATH) $< +$(GENFILES_PATH)/%launch_stage.pb.h $(GENFILES_PATH)/%launch_stage.pb.cc : third_party/googleapis/%launch_stage.proto + mkdir -p $(dir $@) + $(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) --cpp_out=$(GENFILES_PATH) $< + $(GENFILES_PATH)/%client.pb.h $(GENFILES_PATH)/%client.pb.cc : third_party/googleapis/%client.proto mkdir -p $(dir $@) $(PROTOC) -Ithird_party/googleapis -I$(PROTOBUF_INCLUDE_PATH) --cpp_out=$(GENFILES_PATH) $< From 645c79bf8299d5eb9bad412236b2709343570e4a Mon Sep 17 00:00:00 2001 From: cloud-profiler-team Date: Mon, 12 Dec 2022 18:02:08 +0000 Subject: [PATCH 21/22] Internal change. PiperOrigin-RevId: 494758590 --- src/throttler_api.cc | 12 ++++++------ src/throttler_api.h | 2 -- src/worker.cc | 1 - 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/throttler_api.cc b/src/throttler_api.cc index 6abe99165..12832ebd9 100644 --- a/src/throttler_api.cc +++ b/src/throttler_api.cc @@ -14,6 +14,12 @@ #include "src/throttler_api.h" +#include +#include +#include +#include +#include + #include #include // NOLINT #include @@ -26,12 +32,6 @@ #include "google/devtools/cloudprofiler/v2/profiler.grpc.pb.h" #include "google/protobuf/duration.pb.h" // NOLINT #include "google/rpc/error_details.pb.h" // NOLINT -#include "grpc/grpc_security.h" -#include "grpc/support/log.h" -#include "grpc/support/string_util.h" -#include "grpcpp/create_channel.h" -#include "grpcpp/security/credentials.h" -#include "grpcpp/support/channel_arguments.h" #include "third_party/javaprofiler/clock.h" // API curated profiling configuration. diff --git a/src/throttler_api.h b/src/throttler_api.h index c4ee41314..9083beb3f 100644 --- a/src/throttler_api.h +++ b/src/throttler_api.h @@ -30,8 +30,6 @@ #include "src/cloud_env.h" #include "src/throttler.h" #include "google/devtools/cloudprofiler/v2/profiler.grpc.pb.h" -#include "grpcpp/client_context.h" -#include "grpcpp/support/status.h" namespace cloud { namespace profiler { diff --git a/src/worker.cc b/src/worker.cc index 08213d796..9349838a5 100644 --- a/src/worker.cc +++ b/src/worker.cc @@ -18,7 +18,6 @@ #include "src/profiler.h" #include "src/throttler_api.h" #include "src/throttler_timed.h" -#include "google/devtools/cloudprofiler/v2/profiler.grpc.pb.h" #include "third_party/javaprofiler/heap_sampler.h" DEFINE_bool(cprof_enabled, true, From 93ac6e09ed476f1474e5d0e3ebb179c7b907cfc0 Mon Sep 17 00:00:00 2001 From: dthomson Date: Wed, 14 Dec 2022 23:44:37 +0000 Subject: [PATCH 22/22] Fix TSAN errors in the TLAB-based heap profiler PiperOrigin-RevId: 495437976 --- third_party/javaprofiler/heap_sampler.cc | 60 ++++++-- third_party/javaprofiler/heap_sampler.h | 172 ++++++++++++----------- 2 files changed, 138 insertions(+), 94 deletions(-) diff --git a/third_party/javaprofiler/heap_sampler.cc b/third_party/javaprofiler/heap_sampler.cc index e51cb736a..8dc91dd50 100644 --- a/third_party/javaprofiler/heap_sampler.cc +++ b/third_party/javaprofiler/heap_sampler.cc @@ -124,6 +124,17 @@ extern "C" JNIEXPORT void GarbageCollectionFinish(jvmtiEnv *jvmti_env) { namespace google { namespace javaprofiler { +const HeapEventStorage::GcCallback HeapMonitor::gc_callback_ = + [](const HeapObjectTrace &elem) { + HeapMonitor::InvokeGarbageInstrumentationFunctions(elem.ThreadId(), + elem.Name(), + elem.NameLength(), + elem.Size(), + 0); +}; + +std::atomic HeapMonitor::heap_monitor_; + std::atomic HeapMonitor::jvmti_; std::atomic HeapMonitor::sampling_interval_; std::atomic HeapMonitor::use_jvm_trace_; @@ -132,12 +143,14 @@ std::vector std::vector HeapMonitor::gc_inst_functions_; -HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache, - int max_garbage_size) +HeapEventStorage::HeapEventStorage(jvmtiEnv *jvmti, + ProfileFrameCache *cache, + int max_garbage_size, + GcCallback gc_callback) : peak_profile_size_(0), max_garbage_size_(max_garbage_size), cur_garbage_pos_(0), - jvmti_(jvmti), cache_(cache) { + jvmti_(jvmti), cache_(cache), gc_callback_(gc_callback) { } void HeapEventStorage::Add(JNIEnv *jni, jthread thread, jobject object, @@ -174,11 +187,7 @@ void HeapEventStorage::MoveLiveObjects( if (elem.IsLive(env)) { still_live_objects->push_back(std::move(elem)); } else { - HeapMonitor::InvokeGarbageInstrumentationFunctions(elem.ThreadId(), - elem.Name(), - elem.NameLength(), - elem.Size(), - 0); + gc_callback_(elem); elem.DeleteWeakReference(env); AddToGarbage(std::move(elem)); } @@ -231,7 +240,7 @@ HeapEventStorage::StackTraceArrayBuilder::StackTraceArrayBuilder( } void HeapEventStorage::StackTraceArrayBuilder::AddTrace( - HeapEventStorage::HeapObjectTrace &object) { + HeapObjectTrace &object) { std::vector &frames = object.Frames(); call_trace_data_[curr_trace_] = {nullptr, @@ -395,6 +404,22 @@ void HeapMonitor::AddGarbageInstrumentation( GetInstance()->gc_inst_functions_.push_back(fn); } +void HeapMonitor::ShutdownGCWaitingThread() { + GetInstance()->NotifyGCWaitingThreadInternal(GcEvent::SHUTDOWN); + GetInstance()->WaitForShutdown(); +} + +void HeapMonitor::WaitForShutdown() { + // Here we wait for the GC thread to process the shutdown event. + // We can't use JNI to call "Thread.join" on the thread as FindClass + // crashes if called in VM shutdown. + + std::unique_lock lock(gc_waiting_mutex_); + + // If we are woken up without having been notified, just go back to sleep. + gc_waiting_cv_.wait(lock, [this] { return gc_thread_shutdown; } ); +} + void HeapMonitor::AddCallback(jvmtiEventCallbacks *callbacks) { #ifdef ENABLE_HEAP_SAMPLING callbacks->SampledObjectAlloc = &SampledObjectAlloc; @@ -412,6 +437,16 @@ bool HeapMonitor::Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval, return false; } + jvmti_.store(jvmti); + sampling_interval_.store(sampling_interval); + use_jvm_trace_.store(use_jvm_trace); + + // Ensure this is really a singleton i.e. don't recreate it if sampling is + // re-enabled. + if (heap_monitor_ == nullptr) { + heap_monitor_.store(new HeapMonitor()); + } + jvmtiCapabilities caps; memset(&caps, 0, sizeof(caps)); // Get line numbers, sample events, and filename for the tests. @@ -432,10 +467,6 @@ bool HeapMonitor::Enable(jvmtiEnv *jvmti, JNIEnv* jni, int sampling_interval, return false; } - jvmti_.store(jvmti); - sampling_interval_.store(sampling_interval); - use_jvm_trace_.store(use_jvm_trace); - if (!GetInstance()->CreateGCWaitingThread(jvmti, jni)) { return false; } @@ -563,6 +594,9 @@ void HeapMonitor::GCWaitingThreadRun(JNIEnv* jni_env) { // Was the heap monitor disabled? if (event == GcEvent::SHUTDOWN) { + std::unique_lock lock(gc_waiting_mutex_); + gc_thread_shutdown = true; + gc_waiting_cv_.notify_all(); break; } diff --git a/third_party/javaprofiler/heap_sampler.h b/third_party/javaprofiler/heap_sampler.h index 6e029bc47..48eafc177 100644 --- a/third_party/javaprofiler/heap_sampler.h +++ b/third_party/javaprofiler/heap_sampler.h @@ -22,6 +22,7 @@ #include #include // NOLINT +#include #include #include #include // NOLINT @@ -44,12 +45,86 @@ typedef void (*GarbageInstrumentationFunction)(jlong thread_id, namespace google { namespace javaprofiler { +// A sampled heap object, defined by the object, its size, and the stack +// frame. +class HeapObjectTrace { + public: + // This object owns the jweak object parameter. It is freed when the object + // is sent to the garbage list, and the object is set to nullptr. + HeapObjectTrace(jweak object, jlong size, + const std::vector &&frames, jbyte *name, + int name_length, jlong thread_id) + : object_(object), size_(size), frames_(std::move(frames)), + name_(name), name_length_(name_length), thread_id_(thread_id) {} + + HeapObjectTrace(jweak object, jlong size, + const std::vector &frames) + : object_(object), size_(size), frames_(frames) {} + + // Allow moving. + HeapObjectTrace(HeapObjectTrace&& o) = default; + HeapObjectTrace& operator=(HeapObjectTrace&& o) = default; + + // No copying allowed. + HeapObjectTrace(const HeapObjectTrace& o) = delete; + HeapObjectTrace& operator=(const HeapObjectTrace& o) = delete; + + std::vector &Frames() { + return frames_; + } + + int Size() const { + return size_; + } + + jbyte *Name() const { + return name_; + } + + int NameLength() const { + return name_length_; + } + + jlong ThreadId() const { + return thread_id_; + } + + void DeleteWeakReference(JNIEnv* env) { + env->DeleteWeakGlobalRef(object_); + object_ = nullptr; + } + + bool IsLive(JNIEnv *env) { + // When GC collects the object, the object represented by the weak + // reference will be considered as the same object as NULL. + return !env->IsSameObject(object_, NULL); + } + + // Make copying an explicit operation for the one case we need it (adding + // to the peak heapz storage) + HeapObjectTrace Copy() { + return HeapObjectTrace(object_, size_, frames_); + } + + private: + jweak object_; + int size_; + std::vector frames_; + jbyte *name_; + int name_length_; + jlong thread_id_; +}; + // Storage for the sampled heap objects recorded from the heap sampling JVMTI // API callbacks. class HeapEventStorage { public: - HeapEventStorage(jvmtiEnv *jvmti, ProfileFrameCache *cache = nullptr, - int max_garbage_size = 200); + typedef std::function GcCallback; + + HeapEventStorage(jvmtiEnv *jvmti, + ProfileFrameCache *cache = nullptr, + int max_garbage_size = 200, + GcCallback gc_callback = [](const HeapObjectTrace &t){}); // TODO: establish correct shutdown sequence: how do we ensure that // things are not going to go awfully wrong at shutdown, is it this class' job @@ -91,83 +166,13 @@ class HeapEventStorage { HeapEventStorage& operator=(const HeapEventStorage&) = delete; private: - // A sampled heap object, defined by the object, its size, and the stack - // frame. - class HeapObjectTrace { - public: - // This object owns the jweak object parameter. It is freed when the object - // is sent to the garbage list, and the object is set to nullptr. - HeapObjectTrace(jweak object, jlong size, - const std::vector &&frames, jbyte *name, - int name_length, jlong thread_id) - : object_(object), size_(size), frames_(std::move(frames)), - name_(name), name_length_(name_length), thread_id_(thread_id) {} - - HeapObjectTrace(jweak object, jlong size, - const std::vector &frames) - : object_(object), size_(size), frames_(frames) {} - - // Allow moving. - HeapObjectTrace(HeapObjectTrace&& o) = default; - HeapObjectTrace& operator=(HeapObjectTrace&& o) = default; - - // No copying allowed. - HeapObjectTrace(const HeapObjectTrace& o) = delete; - HeapObjectTrace& operator=(const HeapObjectTrace& o) = delete; - - std::vector &Frames() { - return frames_; - } - - int Size() const { - return size_; - } - - jbyte *Name() const { - return name_; - } - - int NameLength() const { - return name_length_; - } - - jlong ThreadId() const { - return thread_id_; - } - - void DeleteWeakReference(JNIEnv* env) { - env->DeleteWeakGlobalRef(object_); - object_ = nullptr; - } - - bool IsLive(JNIEnv *env) { - // When GC collects the object, the object represented by the weak - // reference will be considered as the same object as NULL. - return !env->IsSameObject(object_, NULL); - } - - // Make copying an explicit operation for the one case we need it (adding - // to the peak heapz storage) - HeapObjectTrace Copy() { - return HeapObjectTrace(object_, size_, frames_); - } - - private: - jweak object_; - int size_; - std::vector frames_; - jbyte *name_; - int name_length_; - jlong thread_id_; - }; - // Helper for creating a google::javaprofiler::ProfileStackTrace array // to pass to ProfileProtoBuilder. class StackTraceArrayBuilder { public: StackTraceArrayBuilder(std::size_t objects_size); - void AddTrace(HeapEventStorage::HeapObjectTrace &object); + void AddTrace(HeapObjectTrace &object); google::javaprofiler::ProfileStackTrace* GetStackTraceData() const { return stack_trace_data_.get(); @@ -216,6 +221,7 @@ class HeapEventStorage { std::mutex storage_lock_; jvmtiEnv *jvmti_; ProfileFrameCache *cache_; + const GcCallback gc_callback_; }; // Due to the JVMTI callback, everything here is static. @@ -272,23 +278,25 @@ class HeapMonitor { GetInstance()->NotifyGCWaitingThreadInternal(GcEvent::GC_FINISHED); } - static void ShutdownGCWaitingThread() { - GetInstance()->NotifyGCWaitingThreadInternal(GcEvent::SHUTDOWN); - } + static void ShutdownGCWaitingThread(); // Not copyable or movable. HeapMonitor(const HeapMonitor &) = delete; HeapMonitor &operator=(const HeapMonitor &) = delete; private: - HeapMonitor() : storage_(jvmti_.load(), GetFrameCache()) { + static const HeapEventStorage::GcCallback gc_callback_; + + HeapMonitor() : storage_(jvmti_.load(), GetFrameCache(), 200, gc_callback_) { } - // We construct the heap_monitor at the first call to GetInstance, so ensure - // Enable was called at least once before to initialize jvmti_. + static std::atomic heap_monitor_; + + // We initialize heap_monitor_ in Enable, so ensure Enable is called before + // any call to this method. static HeapMonitor *GetInstance() { - static HeapMonitor heap_monitor; - return &heap_monitor; + assert(heap_monitor_ != nullptr); + return heap_monitor_; } ProfileFrameCache *GetFrameCache() { @@ -309,6 +317,7 @@ class HeapMonitor { void GCWaitingThreadRun(JNIEnv* jni_env); GcEvent WaitForGC(); void NotifyGCWaitingThreadInternal(GcEvent event); + void WaitForShutdown(); void CompactData(JNIEnv* jni_env); @@ -322,6 +331,7 @@ class HeapMonitor { static std::atomic use_jvm_trace_; std::list gc_notify_events_; + bool gc_thread_shutdown = false; std::condition_variable gc_waiting_cv_; std::mutex gc_waiting_mutex_; HeapEventStorage storage_;