Skip to content

[Feature] Add CPU and Heap profile in BE webserver #4632

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 1 commit into from
Nov 5, 2020
Merged

[Feature] Add CPU and Heap profile in BE webserver #4632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions be/src/agent/utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,12 +227,15 @@ std::string AgentUtils::print_agent_status(AgentStatus status) {
}
}

bool AgentUtils::exec_cmd(const string& command, string* errmsg) {
bool AgentUtils::exec_cmd(const string& command, string* errmsg, bool redirect_stderr) {
// The exit status of the command.
uint32_t rc = 0;

// Redirect stderr to stdout to get error message.
string cmd = command + " 2>&1";
string cmd = command;
if (redirect_stderr) {
cmd += " 2>&1";
}

// Execute command.
FILE *fp = popen(cmd.c_str(), "r");
Expand Down
2 changes: 1 addition & 1 deletion be/src/agent/utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ class AgentUtils {
virtual std::string print_agent_status(AgentStatus status);

// Execute shell cmd
virtual bool exec_cmd(const std::string& command, std::string* errmsg);
virtual bool exec_cmd(const std::string& command, std::string* errmsg, bool redirect_stderr = true);

// Write a map to file by json format
virtual bool write_json_to_file(
Expand Down
79 changes: 58 additions & 21 deletions be/src/http/action/pprof_actions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
#include <gperftools/malloc_extension.h>
#include <gperftools/profiler.h>

#include "agent/utils.h"
#include "common/config.h"
#include "gutil/strings/substitute.h"
#include "http/ev_http_server.h"
#include "http/http_channel.h"
#include "http/http_handler.h"
Expand All @@ -36,6 +38,7 @@
#include "runtime/exec_env.h"
#include "util/bfd_parser.h"
#include "util/file_utils.h"
#include "util/pprof_utils.h"

namespace doris {

Expand Down Expand Up @@ -83,7 +86,18 @@ void HeapAction::handle(HttpRequest* req) {
std::string str = profile;
delete profile;

HttpChannel::send_reply(req, str);
const std::string& readable_str = req->param("readable");
if (!readable_str.empty()) {
std::stringstream readable_res;
Status st = PprofUtils::get_readable_profile(str, false, &readable_res);
if (!st.ok()) {
HttpChannel::send_reply(req, st.to_string());
} else {
HttpChannel::send_reply(req, readable_res.str());
}
} else {
HttpChannel::send_reply(req, str);
}
#endif
}

Expand Down Expand Up @@ -130,26 +144,50 @@ void ProfileAction::handle(HttpRequest* req) {
seconds = std::atoi(seconds_str.c_str());
}

std::ostringstream tmp_prof_file_name;
// Build a temporary file name that is hopefully unique.
tmp_prof_file_name << config::pprof_profile_dir << "/doris_profile." << getpid() << "."
<< rand();
ProfilerStart(tmp_prof_file_name.str().c_str());
sleep(seconds);
ProfilerStop();
std::ifstream prof_file(tmp_prof_file_name.str().c_str(), std::ios::in);
std::stringstream ss;
if (!prof_file.is_open()) {
ss << "Unable to open cpu profile: " << tmp_prof_file_name.str();
std::string str = ss.str();
HttpChannel::send_reply(req, str);
return;
}
ss << prof_file.rdbuf();
prof_file.close();
std::string str = ss.str();
const std::string& type_str = req->param("type");
if (type_str != "flamegraph") {
// use pprof the sample the CPU
std::ostringstream tmp_prof_file_name;
tmp_prof_file_name << config::pprof_profile_dir << "/doris_profile." << getpid() << "." << rand();
ProfilerStart(tmp_prof_file_name.str().c_str());
sleep(seconds);
ProfilerStop();

if (type_str != "text") {
// return raw content via http response directly
std::ifstream prof_file(tmp_prof_file_name.str().c_str(), std::ios::in);
std::stringstream ss;
if (!prof_file.is_open()) {
ss << "Unable to open cpu profile: " << tmp_prof_file_name.str();
std::string str = ss.str();
HttpChannel::send_reply(req, str);
return;
}
ss << prof_file.rdbuf();
prof_file.close();
std::string str = ss.str();
HttpChannel::send_reply(req, str);
}

HttpChannel::send_reply(req, str);
// text type. we will return readable content via http response
std::stringstream readable_res;
Status st = PprofUtils::get_readable_profile(tmp_prof_file_name.str(), true, &readable_res);
if (!st.ok()) {
HttpChannel::send_reply(req, st.to_string());
} else {
HttpChannel::send_reply(req, readable_res.str());
}
} else {
// generate flamegraph
std::string svg_file_content;
std::string flamegraph_install_dir = std::string(std::getenv("DORIS_HOME")) + "/tools/FlameGraph/";
Status st = PprofUtils::generate_flamegraph(30, flamegraph_install_dir, false, &svg_file_content);
if (!st.ok()) {
HttpChannel::send_reply(req, st.to_string());
} else {
HttpChannel::send_reply(req, svg_file_content);
}
}
#endif
}

Expand Down Expand Up @@ -179,7 +217,6 @@ void CmdlineAction::handle(HttpRequest* req) {
FILE* fp = fopen("/proc/self/cmdline", "r");
if (fp == nullptr) {
std::string str = "Unable to open file: /proc/self/cmdline";

HttpChannel::send_reply(req, str);
return;
}
Expand Down
148 changes: 147 additions & 1 deletion be/src/http/default_path_handlers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@

#include "http/default_path_handlers.h"

#include <gperftools/heap-profiler.h>
#include <gperftools/malloc_extension.h>

#include <boost/algorithm/string.hpp>
#include <boost/bind.hpp>
#include <sstream>

#include "agent/utils.h"
#include "common/configbase.h"
#include "gutil/strings/numbers.h"
#include "gutil/strings/substitute.h"
Expand Down Expand Up @@ -154,6 +156,148 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
(*output) << "</tbody></table>\n";
}

void heap_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) {
(*output) << "<h2>Heap Profile</h2>" << std::endl;

#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
(*output) << "<pre>" << std::endl;
(*output) << "Heap profiling is not available with address sanitizer builds." << std::endl;
(*output) << "</pre>" << std::endl;
return;

#else
(*output) << "<pre>" << std::endl;
(*output) << "Heap profiling will use pprof tool to sample and get heap profile. It will take 30 seconds" << std::endl;
(*output) << "(Only one thread can obtain profile at the same time)" << std::endl;
(*output) << std::endl;
(*output) << "If you want to get the Heap profile, you need to install gperftools-2.0 on the host machine," << std::endl;
(*output) << "and make sure there is a 'pprof' executable file in the system PATH or 'be/tools/bin/' directory." << std::endl;
(*output) << "Doris will obtain Profile in the following ways:" << std::endl;
(*output) << std::endl;
(*output) << " curl http://localhost:" << config::webserver_port << "/pprof/heap?seconds=30 > perf.data" << std::endl;
(*output) << " pprof --text be/lib/palo_be perf.data" << std::endl;
(*output) << std::endl;
(*output) << "</pre>" << std::endl;
(*output) << "<div id=\"heap\">" << std::endl;
(*output) << " <div><button type=\"button\" id=\"getHeap\">Profile It!</button></div>" << std::endl;
(*output) << " <br/>" << std::endl;
(*output) << " <div id=\"heapResult\"><pre id=\"heapContent\"></pre></div>" << std::endl;
(*output) << "</div>" << std::endl;

(*output) << "<script>" << std::endl;
(*output) << "$('#getHeap').click(function () {" << std::endl;
(*output) << " document.getElementById(\"heapContent\").innerText = \"Sampling... (30 seconds)\";" << std::endl;
(*output) << " $.ajax({" << std::endl;
(*output) << " type: \"GET\"," << std::endl;
(*output) << " dataType: \"text\"," << std::endl;
(*output) << " url: \"pprof/heap?readable=true\"," << std::endl;
(*output) << " timeout: 60000," << std::endl;
(*output) << " success: function (result) {" << std::endl;
(*output) << " $('#heapResult').removeClass('hidden');" << std::endl;
(*output) << " document.getElementById(\"heapContent\").innerText = result;" << std::endl;
(*output) << " }" << std::endl;
(*output) << " ," << std::endl;
(*output) << " error: function (result) {" << std::endl;
(*output) << " alert(result);" << std::endl;
(*output) << " }" << std::endl;
(*output) << " ," << std::endl;
(*output) << " });" << std::endl;
(*output) << "});" << std::endl;

(*output) << "</script>" << std::endl;

return;
#endif
}

void cpu_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) {
(*output) << "<h2>CPU Profile</h2>" << std::endl;

#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
(*output) << "<pre>" << std::endl;
(*output) << "CPU profiling is not available with address sanitizer builds." << std::endl;
(*output) << "</pre>" << std::endl;
return;

#else

(*output) << "<pre>" << std::endl;
(*output) << "CPU profiling will use perf tool to sample and get CPU profile. It will take 30 seconds" << std::endl;
(*output) << "(Only one thread can obtain profile at the same time)" << std::endl;
(*output) << std::endl;
(*output) << "If you want to get the CPU profile in text form, you need to install gperftools-2.0 on the host machine," << std::endl;
(*output) << "and make sure there is a 'pprof' executable file in the system PATH or 'be/tools/bin/' directory." << std::endl;
(*output) << "Doris will obtain Profile in the following ways:" << std::endl;
(*output) << std::endl;
(*output) << " curl http://localhost:" << config::webserver_port << "/pprof/profile?seconds=30 > perf.data" << std::endl;
(*output) << " pprof --text be/lib/palo_be perf.data" << std::endl;
(*output) << std::endl;
(*output) << "If you want to get the flame graph, you must first make sure that there is a 'perf' command on the host machine." << std::endl;
(*output) << "And you need to download the FlameGraph and place it under 'be/tools/FlameGraph'." << std::endl;
(*output) << "Finally, check if the following files exist" << std::endl;
(*output) << std::endl;
(*output) << " be/tools/FlameGraph/stackcollapse-perf.pl" << std::endl;
(*output) << " be/tools/FlameGraph/flamegraph.pl" << std::endl;
(*output) << std::endl;
(*output) << "Doris will obtain the flame graph in the following ways:" << std::endl;
(*output) << std::endl;
(*output) << " perf record -m 2 -g -p be_pid -o perf.data - sleep 30" << std::endl;
(*output) << " perf script -i perf.data | stackcollapse-perf.pl | flamegraph.pl > flamegraph.svg" << std::endl;
(*output) << std::endl;
(*output) << "</pre>" << std::endl;
(*output) << "<div id=\"cpu\">" << std::endl;
(*output) << " <div><button type=\"button\" id=\"getCpu\">Profile It! (Text)</button><button type=\"button\" id=\"getCpuGraph\">Profile It! (FlameGraph)</button></div>" << std::endl;
(*output) << " <br/>" << std::endl;
(*output) << " <div id=\"cpuResult\"><pre id=\"cpuContent\"></pre></div>" << std::endl;
(*output) << " <br/>" << std::endl;
(*output) << " <div id=\"cpuResultGraph\"><pre id=\"cpuContentGraph\"></pre></div>" << std::endl;
(*output) << "</div>" << std::endl;

// for text profile
(*output) << "<script>" << std::endl;
(*output) << "$('#getCpu').click(function () {" << std::endl;
(*output) << " document.getElementById(\"cpuContent\").innerText = \"Sampling... (30 seconds)\";" << std::endl;
(*output) << " $.ajax({" << std::endl;
(*output) << " type: \"GET\"," << std::endl;
(*output) << " dataType: \"text\"," << std::endl;
(*output) << " url: \"pprof/profile?type=text\"," << std::endl;
(*output) << " timeout: 60000," << std::endl;
(*output) << " success: function (result) {" << std::endl;
(*output) << " document.getElementById(\"cpuContent\").innerText = result;" << std::endl;
(*output) << " }" << std::endl;
(*output) << " ," << std::endl;
(*output) << " error: function (result) {" << std::endl;
(*output) << " alert(result);" << std::endl;
(*output) << " }" << std::endl;
(*output) << " ," << std::endl;
(*output) << " });" << std::endl;
(*output) << "});" << std::endl;

// for graph profile
(*output) << "$('#getCpuGraph').click(function () {" << std::endl;
(*output) << " document.getElementById(\"cpuContentGraph\").innerText = \"Sampling... (30 seconds)\";" << std::endl;
(*output) << " $.ajax({" << std::endl;
(*output) << " type: \"GET\"," << std::endl;
(*output) << " dataType: \"text\"," << std::endl;
(*output) << " url: \"pprof/profile?type=flamegraph\"," << std::endl;
(*output) << " timeout: 60000," << std::endl;
(*output) << " success: function (result) {" << std::endl;
(*output) << " document.getElementById(\"cpuResultGraph\").innerHTML = result;" << std::endl;
(*output) << " }" << std::endl;
(*output) << " ," << std::endl;
(*output) << " error: function (result) {" << std::endl;
(*output) << " alert(result);" << std::endl;
(*output) << " }" << std::endl;
(*output) << " ," << std::endl;
(*output) << " });" << std::endl;
(*output) << "});" << std::endl;

(*output) << "</script>" << std::endl;

return;
#endif
}

void add_default_path_handlers(WebPageHandler* web_page_handler,
const std::shared_ptr<MemTracker>& process_mem_tracker) {
// TODO(yingchun): logs_handler is not implemented yet, so not show it on navigate bar
Expand All @@ -162,7 +306,9 @@ void add_default_path_handlers(WebPageHandler* web_page_handler,
web_page_handler->register_page(
"/memz", "Memory", boost::bind<void>(&mem_usage_handler, process_mem_tracker, _1, _2),
true /* is_on_nav_bar */);
web_page_handler->register_page("/mem_tracker", "MemTracker", mem_tracker_handler,true /* is_on_nav_bar */);
web_page_handler->register_page("/mem_tracker", "MemTracker", mem_tracker_handler, true /* is_on_nav_bar */);
web_page_handler->register_page("/heap", "Heap Profile", heap_handler, true /* is_on_nav_bar */);
web_page_handler->register_page("/cpu", "CPU Profile", cpu_handler, true /* is_on_nav_bar */);
register_thread_display_page(web_page_handler);
web_page_handler->register_template_page("/tablets_page", "Tablets", boost::bind<void>(&display_tablets_callback, _1, _2), true /* is_on_nav_bar */);
}
Expand Down
2 changes: 1 addition & 1 deletion be/src/http/web_page_handler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ static const std::string kMainTemplate = R"(
<meta charset='utf-8'/>
<link href='/Bootstrap-3.3.7/css/bootstrap.min.css' rel='stylesheet' media='screen' />
<link href='/Bootstrap-3.3.7/css/bootstrap-table.min.css' rel='stylesheet' media='screen' />
<script src='/jQuery-3.3.1/jquery-3.3.1.min.js' defer></script>
<script src='/jQuery-3.3.1/jquery-3.3.1.min.js'></script>
<script src='/Bootstrap-3.3.7/js/bootstrap.min.js' defer></script>
<script src='/Bootstrap-3.3.7/js/bootstrap-table.min.js' defer></script>
<script src='/doris.js' defer></script>
Expand Down
1 change: 1 addition & 0 deletions be/src/util/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ set(UTIL_FILES
mustache/mustache.cc
brpc_stub_cache.cpp
zlib.cpp
pprof_utils.cpp
)

if (WITH_MYSQL)
Expand Down
Loading