trafic_crashlog: Moving the removed ATS backtrace logs.

cmake/Findunwind.cmake

@@ -28,8 +28,9 @@
 #     unwind::unwind
 #
 
-find_library(unwind_LIBRARY NAMES unwind)
-find_path(unwind_INCLUDE_DIR NAMES libunwind.h libunwind/libunwind.h)
+find_library(unwind_ptrace_LIBRARY NAMES unwind-ptrace)
+find_library(unwind_generic_LIBRARY NAMES unwind-generic)
+find_path(unwind_INCLUDE_DIR NAMES libunwind.h)
 
 mark_as_advanced(unwind_FOUND unwind_LIBRARY unwind_INCLUDE_DIR)
 
@@ -43,5 +44,5 @@ endif()
 if(unwind_FOUND AND NOT TARGET unwind::unwind)
   add_library(unwind::unwind INTERFACE IMPORTED)
   target_include_directories(unwind::unwind INTERFACE ${unwind_INCLUDE_DIRS})
-  target_link_libraries(unwind::unwind INTERFACE "${unwind_LIBRARY}")
+  target_link_libraries(unwind::unwind INTERFACE "${unwind_ptrace_LIBRARY}" "${unwind_generic_LIBRARY}")
 endif()

src/traffic_crashlog/CMakeLists.txt

@@ -15,8 +15,12 @@
 #
 #######################
 
-add_executable(traffic_crashlog procinfo.cc traffic_crashlog.cc)
+add_executable(traffic_crashlog procinfo.cc backtrace.cc traffic_crashlog.cc)
 
 target_link_libraries(traffic_crashlog PRIVATE ts::inkevent ts::records ts::tscore ts::tsapicore)
 
+if(TS_USE_REMOTE_UNWINDING)
+  target_link_libraries(traffic_crashlog PRIVATE unwind::unwind)
+endif()
+
 install(TARGETS traffic_crashlog)

src/traffic_crashlog/backtrace.cc

@@ -0,0 +1,208 @@
+/** @file
+
+  backtrace.cc
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+/****************************************************************************
+
+  backtrace.cc
+
+  Functions to generate backtrace from a TS process.
+
+ ****************************************************************************/
+#include "tscore/ink_config.h"
+
+#if TS_USE_REMOTE_UNWINDING
+#include "tscore/Diags.h"
+
+#include "tscore/TextBuffer.h"
+#include <string.h>
+#include <libunwind.h>
+#include <libunwind-ptrace.h>
+#include <sys/ptrace.h>
+#include <cxxabi.h>
+#include <vector>
+
+namespace
+{
+using threadlist = std::vector<pid_t>;
+
+static threadlist
+threads_for_process(pid_t proc)
+{
+  DIR *dir             = nullptr;
+  struct dirent *entry = nullptr;
+
+  char path[64];
+  threadlist threads;
+
+  if (snprintf(path, sizeof(path), "/proc/%ld/task", static_cast<long>(proc)) >= static_cast<int>(sizeof(path))) {
+    goto done;
+  }
+
+  dir = opendir(path);
+  if (dir == nullptr) {
+    goto done;
+  }
+
+  while ((entry = readdir(dir))) {
+    pid_t threadid;
+
+    if (isdot(entry->d_name) || isdotdot(entry->d_name)) {
+      continue;
+    }
+
+    threadid = strtol(entry->d_name, nullptr, 10);
+    if (threadid > 0) {
+      threads.push_back(threadid);
+      Debug("backtrace", "found thread %ld\n", (long)threadid);
+    }
+  }
+
+done:
+  if (dir) {
+    closedir(dir);
+  }
+
+  return threads;
+}
+
+static void
+backtrace_for_thread(pid_t threadid, TextBuffer &text)
+{
+  int status;
+  unw_addr_space_t addr_space = nullptr;
+  unw_cursor_t cursor;
+  void *ap       = nullptr;
+  pid_t target   = -1;
+  unsigned level = 0;
+
+  // First, attach to the child, causing it to stop.
+  status = ptrace(PTRACE_ATTACH, threadid, 0, 0);
+  if (status < 0) {
+    Debug("backtrace", "ptrace(ATTACH, %ld) -> %s (%d)\n", (long)threadid, strerror(errno), errno);
+    return;
+  }
+
+  // Wait for it to stop (XXX should be a timed wait ...)
+  target = waitpid(threadid, &status, __WALL | WUNTRACED);
+  Debug("backtrace", "waited for target %ld, found PID %ld, %s\n", (long)threadid, (long)target,
+        WIFSTOPPED(status) ? "STOPPED" : "???");
+  if (target < 0) {
+    goto done;
+  }
+
+  ap = _UPT_create(threadid);
+  Debug("backtrace", "created UPT %p", ap);
+  if (ap == nullptr) {
+    goto done;
+  }
+
+  addr_space = unw_create_addr_space(&_UPT_accessors, 0 /* byteorder */);
+  Debug("backtrace", "created address space %p\n", addr_space);
+  if (addr_space == nullptr) {
+    goto done;
+  }
+
+  status = unw_init_remote(&cursor, addr_space, ap);
+  Debug("backtrace", "unw_init_remote(...) -> %d\n", status);
+  if (status != 0) {
+    goto done;
+  }
+
+  while (unw_step(&cursor) > 0) {
+    unw_word_t ip;
+    unw_word_t offset;
+    char buf[256];
+
+    unw_get_reg(&cursor, UNW_REG_IP, &ip);
+
+    if (unw_get_proc_name(&cursor, buf, sizeof(buf), &offset) == 0) {
+      int status;
+      char *name = abi::__cxa_demangle(buf, nullptr, nullptr, &status);
+      text.format("%-4u 0x%016llx %s + %p\n", level, static_cast<unsigned long long>(ip), name ? name : buf, (void *)offset);
+      free(name);
+    } else {
+      text.format("%-4u 0x%016llx 0x0 + %p\n", level, static_cast<unsigned long long>(ip), (void *)offset);
+    }
+
+    ++level;
+  }
+
+done:
+  if (addr_space) {
+    unw_destroy_addr_space(addr_space);
+  }
+
+  if (ap) {
+    _UPT_destroy(ap);
+  }
+
+  status = ptrace(PTRACE_DETACH, target, NULL, NULL);
+  Debug("backtrace", "ptrace(DETACH, %ld) -> %d (errno %d)\n", (long)target, status, errno);
+}
+} // namespace
+int
+ServerBacktrace(unsigned /* options */, int pid, char **trace)
+{
+  *trace = nullptr;
+
+  threadlist threads(threads_for_process(pid));
+  TextBuffer text(0);
+
+  Debug("backtrace", "tracing %zd threads for traffic_server PID %ld\n", threads.size(), (long)pid);
+
+  for (auto threadid : threads) {
+    Debug("backtrace", "tracing thread %ld\n", (long)threadid);
+    // Get the thread name using /proc/PID/comm
+    ats_scoped_fd fd;
+    char threadname[128];
+
+    snprintf(threadname, sizeof(threadname), "/proc/%ld/comm", static_cast<long>(threadid));
+    fd = open(threadname, O_RDONLY);
+    if (fd >= 0) {
+      text.format("Thread %ld, ", static_cast<long>(threadid));
+      text.readFromFD(fd);
+      text.chomp();
+    } else {
+      text.format("Thread %ld", static_cast<long>(threadid));
+    }
+
+    text.format(":\n");
+
+    backtrace_for_thread(threadid, text);
+    text.format("\n");
+  }
+
+  *trace = text.release();
+  return 0;
+}
+
+#else /* TS_USE_REMOTE_UNWINDING */
+
+int
+ServerBacktrace(unsigned /* options */, int pid, char **trace)
+{
+  *trace = nullptr;
+  return -1;
+}
+
+#endif

src/traffic_crashlog/traffic_crashlog.cc

@@ -88,46 +88,27 @@ crashlog_open(const char *path)
   return (fd == -1) ? nullptr : fdopen(fd, "w");
 }
 
-static unsigned
-max_passwd_size()
-{
-#if defined(_SC_GETPW_R_SIZE_MAX)
-  long val = sysconf(_SC_GETPW_R_SIZE_MAX);
-  if (val > 0) {
-    return static_cast<unsigned>(val);
-  }
-#endif
-
-  return 4096;
-}
+extern int ServerBacktrace(unsigned /* options */, int pid, char **trace);
 
-static void
-change_privileges()
+bool
+crashlog_write_backtrace(FILE *fp, pid_t pid, const crashlog_target &)
 {
-  struct passwd *pwd;
-  struct passwd pbuf;
-  char buf[max_passwd_size()];
+  char *trace = nullptr;
+  int mgmterr;
 
-  if (getpwnam_r(user, &pbuf, buf, sizeof(buf), &pwd) != 0) {
-    Error("missing password database entry for username '%s': %s", user, strerror(errno));
-    return;
-  }
-
-  if (pwd == nullptr) {
-    // Password entry not found ...
-    Error("missing password database entry for '%s'", user);
-    return;
-  }
+  // NOTE: sometimes we can't get a backtrace because the ptrace attach will fail with
+  // EPERM. I've seen this happen when a debugger is attached, which makes sense, but it
+  // can also happen without a debugger. Possibly in that case, there is a race with the
+  // kernel locking the process information?
 
-  if (setegid(pwd->pw_gid) != 0) {
-    Error("setegid(%d) failed: %s", pwd->pw_gid, strerror(errno));
-    return;
+  if ((mgmterr = ServerBacktrace(0, static_cast<int>(pid), &trace)) != 0) {
+    fprintf(fp, "Unable to retrieve backtrace: %d\n", mgmterr);
+    return false;
   }
 
-  if (setreuid(pwd->pw_uid, 0) != 0) {
-    Error("setreuid(%d, %d) failed: %s", pwd->pw_uid, 0, strerror(errno));
-    return;
-  }
+  fprintf(fp, "%s", trace);
+  free(trace);
+  return true;
 }
 
 int
@@ -145,13 +126,6 @@ main(int /* argc ATS_UNUSED */, const char **argv)
   // Process command line arguments and dump into variables
   process_args(&version, argument_descriptions, countof(argument_descriptions), argv);
 
-  // XXX This is a hack. traffic_manager starts traffic_server with the euid of the admin user. We are still
-  // privileged, but won't be able to open files in /proc or ptrace the target. This really should be fixed
-  // in traffic_manager.
-  if (getuid() == 0) {
-    change_privileges();
-  }
-
   if (wait_mode) {
     EnableDeathSignal(SIGKILL);
     kill(getpid(), SIGSTOP);
@@ -240,8 +214,10 @@ main(int /* argc ATS_UNUSED */, const char **argv)
   crashlog_write_registers(fp, target);
 
   fprintf(fp, "\n");
-  crashlog_write_procstatus(fp, target);
+  crashlog_write_backtrace(fp, parent, target);
 
+  fprintf(fp, "\n");
+  crashlog_write_procstatus(fp, target);
   fprintf(fp, "\n");
   crashlog_write_proclimits(fp, target);