From 9e8f7b66a63a76e064258f5fb50a4234bdd37d86 Mon Sep 17 00:00:00 2001 From: Damian Meden Date: Fri, 17 Nov 2023 15:33:11 +0100 Subject: [PATCH] trafic_crashlog: Moving the removed ATS backtrace logs from traffic_manager to traffic_crashlog. --- cmake/Findunwind.cmake | 7 +- src/traffic_crashlog/CMakeLists.txt | 6 +- src/traffic_crashlog/backtrace.cc | 208 +++++++++++++++++++++++ src/traffic_crashlog/traffic_crashlog.cc | 60 ++----- 4 files changed, 235 insertions(+), 46 deletions(-) create mode 100644 src/traffic_crashlog/backtrace.cc diff --git a/cmake/Findunwind.cmake b/cmake/Findunwind.cmake index fd77524c6fd..b192ccde8f8 100644 --- a/cmake/Findunwind.cmake +++ b/cmake/Findunwind.cmake @@ -28,8 +28,9 @@ # unwind::unwind # -find_library(unwind_LIBRARY NAMES unwind) -find_path(unwind_INCLUDE_DIR NAMES libunwind.h libunwind/libunwind.h) +find_library(unwind_ptrace_LIBRARY NAMES unwind-ptrace) +find_library(unwind_generic_LIBRARY NAMES unwind-generic) +find_path(unwind_INCLUDE_DIR NAMES libunwind.h) mark_as_advanced(unwind_FOUND unwind_LIBRARY unwind_INCLUDE_DIR) @@ -43,5 +44,5 @@ endif() if(unwind_FOUND AND NOT TARGET unwind::unwind) add_library(unwind::unwind INTERFACE IMPORTED) target_include_directories(unwind::unwind INTERFACE ${unwind_INCLUDE_DIRS}) - target_link_libraries(unwind::unwind INTERFACE "${unwind_LIBRARY}") + target_link_libraries(unwind::unwind INTERFACE "${unwind_ptrace_LIBRARY}" "${unwind_generic_LIBRARY}") endif() diff --git a/src/traffic_crashlog/CMakeLists.txt b/src/traffic_crashlog/CMakeLists.txt index e78cb924dff..5d98d633cc5 100644 --- a/src/traffic_crashlog/CMakeLists.txt +++ b/src/traffic_crashlog/CMakeLists.txt @@ -15,8 +15,12 @@ # ####################### -add_executable(traffic_crashlog procinfo.cc traffic_crashlog.cc) +add_executable(traffic_crashlog procinfo.cc backtrace.cc traffic_crashlog.cc) target_link_libraries(traffic_crashlog PRIVATE ts::inkevent ts::records ts::tscore ts::tsapicore) +if(TS_USE_REMOTE_UNWINDING) + target_link_libraries(traffic_crashlog PRIVATE unwind::unwind) +endif() + install(TARGETS traffic_crashlog) diff --git a/src/traffic_crashlog/backtrace.cc b/src/traffic_crashlog/backtrace.cc new file mode 100644 index 00000000000..ed9fbc503d2 --- /dev/null +++ b/src/traffic_crashlog/backtrace.cc @@ -0,0 +1,208 @@ +/** @file + + backtrace.cc + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +/**************************************************************************** + + backtrace.cc + + Functions to generate backtrace from a TS process. + + ****************************************************************************/ +#include "tscore/ink_config.h" + +#if TS_USE_REMOTE_UNWINDING +#include "tscore/Diags.h" + +#include "tscore/TextBuffer.h" +#include +#include +#include +#include +#include +#include + +namespace +{ +using threadlist = std::vector; + +static threadlist +threads_for_process(pid_t proc) +{ + DIR *dir = nullptr; + struct dirent *entry = nullptr; + + char path[64]; + threadlist threads; + + if (snprintf(path, sizeof(path), "/proc/%ld/task", static_cast(proc)) >= static_cast(sizeof(path))) { + goto done; + } + + dir = opendir(path); + if (dir == nullptr) { + goto done; + } + + while ((entry = readdir(dir))) { + pid_t threadid; + + if (isdot(entry->d_name) || isdotdot(entry->d_name)) { + continue; + } + + threadid = strtol(entry->d_name, nullptr, 10); + if (threadid > 0) { + threads.push_back(threadid); + Debug("backtrace", "found thread %ld\n", (long)threadid); + } + } + +done: + if (dir) { + closedir(dir); + } + + return threads; +} + +static void +backtrace_for_thread(pid_t threadid, TextBuffer &text) +{ + int status; + unw_addr_space_t addr_space = nullptr; + unw_cursor_t cursor; + void *ap = nullptr; + pid_t target = -1; + unsigned level = 0; + + // First, attach to the child, causing it to stop. + status = ptrace(PTRACE_ATTACH, threadid, 0, 0); + if (status < 0) { + Debug("backtrace", "ptrace(ATTACH, %ld) -> %s (%d)\n", (long)threadid, strerror(errno), errno); + return; + } + + // Wait for it to stop (XXX should be a timed wait ...) + target = waitpid(threadid, &status, __WALL | WUNTRACED); + Debug("backtrace", "waited for target %ld, found PID %ld, %s\n", (long)threadid, (long)target, + WIFSTOPPED(status) ? "STOPPED" : "???"); + if (target < 0) { + goto done; + } + + ap = _UPT_create(threadid); + Debug("backtrace", "created UPT %p", ap); + if (ap == nullptr) { + goto done; + } + + addr_space = unw_create_addr_space(&_UPT_accessors, 0 /* byteorder */); + Debug("backtrace", "created address space %p\n", addr_space); + if (addr_space == nullptr) { + goto done; + } + + status = unw_init_remote(&cursor, addr_space, ap); + Debug("backtrace", "unw_init_remote(...) -> %d\n", status); + if (status != 0) { + goto done; + } + + while (unw_step(&cursor) > 0) { + unw_word_t ip; + unw_word_t offset; + char buf[256]; + + unw_get_reg(&cursor, UNW_REG_IP, &ip); + + if (unw_get_proc_name(&cursor, buf, sizeof(buf), &offset) == 0) { + int status; + char *name = abi::__cxa_demangle(buf, nullptr, nullptr, &status); + text.format("%-4u 0x%016llx %s + %p\n", level, static_cast(ip), name ? name : buf, (void *)offset); + free(name); + } else { + text.format("%-4u 0x%016llx 0x0 + %p\n", level, static_cast(ip), (void *)offset); + } + + ++level; + } + +done: + if (addr_space) { + unw_destroy_addr_space(addr_space); + } + + if (ap) { + _UPT_destroy(ap); + } + + status = ptrace(PTRACE_DETACH, target, NULL, NULL); + Debug("backtrace", "ptrace(DETACH, %ld) -> %d (errno %d)\n", (long)target, status, errno); +} +} // namespace +int +ServerBacktrace(unsigned /* options */, int pid, char **trace) +{ + *trace = nullptr; + + threadlist threads(threads_for_process(pid)); + TextBuffer text(0); + + Debug("backtrace", "tracing %zd threads for traffic_server PID %ld\n", threads.size(), (long)pid); + + for (auto threadid : threads) { + Debug("backtrace", "tracing thread %ld\n", (long)threadid); + // Get the thread name using /proc/PID/comm + ats_scoped_fd fd; + char threadname[128]; + + snprintf(threadname, sizeof(threadname), "/proc/%ld/comm", static_cast(threadid)); + fd = open(threadname, O_RDONLY); + if (fd >= 0) { + text.format("Thread %ld, ", static_cast(threadid)); + text.readFromFD(fd); + text.chomp(); + } else { + text.format("Thread %ld", static_cast(threadid)); + } + + text.format(":\n"); + + backtrace_for_thread(threadid, text); + text.format("\n"); + } + + *trace = text.release(); + return 0; +} + +#else /* TS_USE_REMOTE_UNWINDING */ + +int +ServerBacktrace(unsigned /* options */, int pid, char **trace) +{ + *trace = nullptr; + return -1; +} + +#endif diff --git a/src/traffic_crashlog/traffic_crashlog.cc b/src/traffic_crashlog/traffic_crashlog.cc index 3b922a03864..9b90d4f8b78 100644 --- a/src/traffic_crashlog/traffic_crashlog.cc +++ b/src/traffic_crashlog/traffic_crashlog.cc @@ -88,46 +88,27 @@ crashlog_open(const char *path) return (fd == -1) ? nullptr : fdopen(fd, "w"); } -static unsigned -max_passwd_size() -{ -#if defined(_SC_GETPW_R_SIZE_MAX) - long val = sysconf(_SC_GETPW_R_SIZE_MAX); - if (val > 0) { - return static_cast(val); - } -#endif - - return 4096; -} +extern int ServerBacktrace(unsigned /* options */, int pid, char **trace); -static void -change_privileges() +bool +crashlog_write_backtrace(FILE *fp, pid_t pid, const crashlog_target &) { - struct passwd *pwd; - struct passwd pbuf; - char buf[max_passwd_size()]; + char *trace = nullptr; + int mgmterr; - if (getpwnam_r(user, &pbuf, buf, sizeof(buf), &pwd) != 0) { - Error("missing password database entry for username '%s': %s", user, strerror(errno)); - return; - } - - if (pwd == nullptr) { - // Password entry not found ... - Error("missing password database entry for '%s'", user); - return; - } + // NOTE: sometimes we can't get a backtrace because the ptrace attach will fail with + // EPERM. I've seen this happen when a debugger is attached, which makes sense, but it + // can also happen without a debugger. Possibly in that case, there is a race with the + // kernel locking the process information? - if (setegid(pwd->pw_gid) != 0) { - Error("setegid(%d) failed: %s", pwd->pw_gid, strerror(errno)); - return; + if ((mgmterr = ServerBacktrace(0, static_cast(pid), &trace)) != 0) { + fprintf(fp, "Unable to retrieve backtrace: %d\n", mgmterr); + return false; } - if (setreuid(pwd->pw_uid, 0) != 0) { - Error("setreuid(%d, %d) failed: %s", pwd->pw_uid, 0, strerror(errno)); - return; - } + fprintf(fp, "%s", trace); + free(trace); + return true; } int @@ -145,13 +126,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) // Process command line arguments and dump into variables process_args(&version, argument_descriptions, countof(argument_descriptions), argv); - // XXX This is a hack. traffic_manager starts traffic_server with the euid of the admin user. We are still - // privileged, but won't be able to open files in /proc or ptrace the target. This really should be fixed - // in traffic_manager. - if (getuid() == 0) { - change_privileges(); - } - if (wait_mode) { EnableDeathSignal(SIGKILL); kill(getpid(), SIGSTOP); @@ -240,8 +214,10 @@ main(int /* argc ATS_UNUSED */, const char **argv) crashlog_write_registers(fp, target); fprintf(fp, "\n"); - crashlog_write_procstatus(fp, target); + crashlog_write_backtrace(fp, parent, target); + fprintf(fp, "\n"); + crashlog_write_procstatus(fp, target); fprintf(fp, "\n"); crashlog_write_proclimits(fp, target);