diff --git a/bazel/BUILD b/bazel/BUILD index 3a88d84ac3373..dcaf8989ebfd7 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -26,3 +26,8 @@ config_setting( name = "disable_tcmalloc", values = {"define": "tcmalloc=disabled"}, ) + +config_setting( + name = "disable_signal_trace", + values = {"define": "signal_trace=disabled"}, +) diff --git a/bazel/README.md b/bazel/README.md index ad046ef4f5f7f..43d634274f245 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -93,11 +93,11 @@ bazel test //test/common/http:async_client_impl_test --strategy=TestRunner=stand ``` # Stack trace symbol resolution -Envoy can produce backtraces on demand or from assertions and other activity. -The stack traces written in the log or to stderr contain addresses rather than -resolved symbols. The `tools/stack_decode.py` script exists to process the output -and do symbol resolution to make the stack traces useful. Any log lines not -relevant to the backtrace capability are passed through the script unchanged +Envoy can produce backtraces on demand and from assertions and other fatal +actions like segfaults. The stack traces written in the log or to stderr contain +addresses rather than resolved symbols. The `tools/stack_decode.py` script exists +to process the output and do symbol resolution to make the stack traces useful. Any +log lines not relevant to the backtrace capability are passed through the script unchanged (it acts like a filter). The script runs in one of two modes. If passed no arguments it anticipates @@ -116,6 +116,13 @@ bazel test -c dbg //test/server:backtrace_test You will need to use either a `dbg` build type or the `--define debug_symbols=yes` option to get symbol information in the binaries. +By default main.cc will install signal handlers to print backtraces at the +location where a fatal signal occurred. The signal handler will re-raise the +fatal signal with the default handler so a core file will still be dumped after +the stack trace is logged. To inhibit this behavior use +`--define=signal_trace=disabled` on the Bazel command line. No signal handlers will +be installed. + # Running a single Bazel test under GDB ``` diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl index 15c0331501259..b1d7bcbf2f167 100644 --- a/bazel/envoy_build_system.bzl +++ b/bazel/envoy_build_system.bzl @@ -21,6 +21,9 @@ def envoy_copts(repository, test = False): }) + select({ repository + "//bazel:disable_tcmalloc": [], "//conditions:default": ["-DTCMALLOC"], + }) + select({ + repository + "//bazel:disable_signal_trace": [], + "//conditions:default": ["-DENVOY_HANDLE_SIGNALS"], }) # References to Envoy external dependencies should be wrapped with this function. diff --git a/source/exe/BUILD b/source/exe/BUILD index 41b8f5f0334c4..a1a5cb8456e81 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -54,7 +54,10 @@ envoy_cc_library( deps = [ ":envoy_common_lib", ":hot_restart_lib", - ], + ] + select({ + "//bazel:disable_signal_trace": [], + "//conditions:default": [":sigaction_lib"], + }), ) envoy_cc_library( @@ -72,3 +75,14 @@ envoy_cc_library( "//source/common/stats:stats_lib", ], ) + +envoy_cc_library( + name = "sigaction_lib", + srcs = ["signal_action.cc"], + hdrs = ["signal_action.h"], + deps = [ + "//source/common/common:assert_lib", + "//source/common/common:non_copyable", + "//source/server:backtrace_lib", + ], +) diff --git a/source/exe/main.cc b/source/exe/main.cc index 1076c3255efad..5ddf2c0cc2ea4 100644 --- a/source/exe/main.cc +++ b/source/exe/main.cc @@ -8,6 +8,10 @@ #include "exe/hot_restart.h" +#ifdef ENVOY_HANDLE_SIGNALS +#include "exe/signal_action.h" +#endif + #include "server/drain_manager_impl.h" #include "server/options_impl.h" #include "server/server.h" @@ -34,6 +38,10 @@ class ProdComponentFactory : public ComponentFactory { } // Server int main(int argc, char** argv) { +#ifdef ENVOY_HANDLE_SIGNALS + // Enabled by default. Control with "bazel --define=signal_trace=disabled" + SignalAction handle_sigs; +#endif ares_library_init(ARES_LIB_INIT_ALL); Event::Libevent::Global::initialize(); OptionsImpl options(argc, argv, Server::SharedMemory::version(), spdlog::level::warn); diff --git a/source/exe/signal_action.cc b/source/exe/signal_action.cc new file mode 100644 index 0000000000000..d0588611c48b6 --- /dev/null +++ b/source/exe/signal_action.cc @@ -0,0 +1,95 @@ +#include "exe/signal_action.h" + +#include +#include + +#include "common/common/assert.h" + +constexpr int SignalAction::FATAL_SIGS[]; + +void SignalAction::sigHandler(int sig, siginfo_t* info, void* context) { + void* error_pc = 0; + + const ucontext_t* ucontext = reinterpret_cast(context); + if (ucontext != nullptr) { +#ifdef REG_RIP + // x86_64 + error_pc = reinterpret_cast(ucontext->uc_mcontext.gregs[REG_RIP]); +#else +#warning "Please enable and test PC retrieval code for your arch in signal_action.cc" +// x86 Classic: reinterpret_cast(ucontext->uc_mcontext.gregs[REG_EIP]); +// ARM: reinterpret_cast(ucontext->uc_mcontext.arm_pc); +// PPC: reinterpret_cast(ucontext->uc_mcontext.regs->nip); +#endif + } + + BackwardsTrace tracer; + tracer.logFault(strsignal(sig), info->si_addr); + if (error_pc != 0) { + tracer.captureFrom(error_pc); + } else { + tracer.capture(); + } + tracer.logTrace(); + + signal(sig, SIG_DFL); + raise(sig); +} + +void SignalAction::installSigHandlers() { + stack_t stack; + stack.ss_sp = altstack_ + guard_size_; // Guard page at one end ... + stack.ss_size = altstack_size_; // ... guard page at the other + stack.ss_flags = 0; + + RELEASE_ASSERT(sigaltstack(&stack, nullptr) == 0); + + for (const auto& sig : FATAL_SIGS) { + struct sigaction saction; + std::memset(&saction, 0, sizeof(saction)); + sigemptyset(&saction.sa_mask); + saction.sa_flags = (SA_SIGINFO | SA_ONSTACK | SA_RESETHAND | SA_NODEFER); + saction.sa_sigaction = sigHandler; + RELEASE_ASSERT(sigaction(sig, &saction, nullptr) == 0); + } +} + +void SignalAction::removeSigHandlers() { + for (const auto& sig : FATAL_SIGS) { + signal(sig, SIG_DFL); + } +} + +void SignalAction::mapAndProtectStackMemory() { + // Per docs MAP_STACK doesn't actually do anything today but provides a + // library hint that might be used in the future. + altstack_ = static_cast(mmap(nullptr, mapSizeWithGuards(), PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0)); + RELEASE_ASSERT(altstack_); + RELEASE_ASSERT(mprotect(altstack_, guard_size_, PROT_NONE) == 0); + RELEASE_ASSERT(mprotect(altstack_ + guard_size_ + altstack_size_, guard_size_, PROT_NONE) == 0); +} + +void SignalAction::unmapStackMemory() { munmap(altstack_, mapSizeWithGuards()); } + +void SignalAction::doGoodAccessForTest() { + volatile char* altaltstack = altstack_; + for (size_t i = 0; i < altstack_size_; ++i) { + *(altaltstack + guard_size_ + i) = 42; + } + for (size_t i = 0; i < altstack_size_; ++i) { + ASSERT(*(altaltstack + guard_size_ + i) == 42); + } +} + +void SignalAction::tryEvilAccessForTest(bool end) { + volatile char* altaltstack = altstack_; + if (end) { + // One byte past the valid region + // http://oeis.org/A001969 + *(altaltstack + guard_size_ + altstack_size_) = 43; + } else { + // One byte before the valid region + *(altaltstack + guard_size_ - 1) = 43; + } +} diff --git a/source/exe/signal_action.h b/source/exe/signal_action.h new file mode 100644 index 0000000000000..3936d5c118d35 --- /dev/null +++ b/source/exe/signal_action.h @@ -0,0 +1,117 @@ +#pragma once + +#include +#include + +#include "common/common/non_copyable.h" + +#include "server/backtrace.h" + +/** + * This class installs signal handlers for fatal signal types. + * + * These signals are handled: + * SIGABRT + * SIGBUS + * SIGFPE + * SIGILL + * SIGSEGV + * + * Upon intercepting the signal the following actions are taken: + * + * A Backtrace is printed from the address the signal was encountered at, if + * it is possible to retrieve. + * + * The signal handler is reset to the default handler (which is expected to + * terminate the process). + * + * The signal is raised again (which ultimately kills the process) + * + * The signal handler must run on an alternative stack so that we can do the + * stack unwind on the original stack. Memory is allocated for this purpose when + * this object is constructed. When this object goes out of scope the memory + * used for the alternate signal stack is destroyed and the default signal handler + * is restored. + * + * NOTE: Existing non-default signal handlers are overridden and will not be + * restored. If this behavior is ever required it can be implemented. + * + * It is recommended that this object be instantiated at the highest possible + * scope, eg, in main(). This enables fatal signal handling for almost all code + * executed. + */ +class SignalAction : NonCopyable { +public: + SignalAction() + : guard_size_(sysconf(_SC_PAGE_SIZE)), altstack_size_(guard_size_ * 4), altstack_(nullptr) { + mapAndProtectStackMemory(); + installSigHandlers(); + } + ~SignalAction() { + removeSigHandlers(); + unmapStackMemory(); + } + /** + * Helpers for testing guarded stack memory + */ + void doGoodAccessForTest(); + void tryEvilAccessForTest(bool end); + +private: + /** + * Allocate this many bytes on each side of the area used for alt stack. + * + * Set to system page size. + * + * The memory will be protected from read and write. + */ + const size_t guard_size_; + /** + * Use this many bytes for the alternate signal handling stack. + * + * Initialized as a multiple of page size (although sigaltstack will + * do alignment as needed). + * + * Additionally, two guard pages will be allocated to bookend the usable area. + */ + const size_t altstack_size_; + /** + * This constant array defines the signals we will insert handlers for. + * + * Essentially this is the list of signals that would cause a core dump. + */ + static constexpr int FATAL_SIGS[] = {SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGSEGV}; + /** + * Return the memory size we actually map including two guard pages. + */ + size_t mapSizeWithGuards() const { return altstack_size_ + guard_size_ * 2; } + /** + * The actual signal handler function with prototype matching signal.h + */ + static void sigHandler(int sig, siginfo_t* info, void* context); + /** + * Install all signal handlers and setup signal handling stack. + */ + void installSigHandlers(); + /** + * Remove all signal handlers. + * + * Must be executed before the alt stack memory goes away. + * + * Signal handlers will be reset to the default, NOT back to any signal + * handler existing before InstallSigHandlers(). + */ + void removeSigHandlers(); + /** + * Use mmap to map anonymous memory for the alternative stack. + * + * GUARD_SIZE on either end of the memory will be marked PROT_NONE, protected + * from all access. + */ + void mapAndProtectStackMemory(); + /** + * Unmap alternative stack memory. + */ + void unmapStackMemory(); + char* altstack_; +}; diff --git a/source/server/backtrace.h b/source/server/backtrace.h index e2fe2e1f5c024..4711c7bdb3961 100644 --- a/source/server/backtrace.h +++ b/source/server/backtrace.h @@ -7,15 +7,8 @@ #define BACKTRACE_LOG() \ do { \ BackwardsTrace t; \ - t.Capture(); \ - t.Log(); \ - } while (0) - -#define BACKTRACE_PROD_LOG() \ - do { \ - BackwardsTrace t(true); \ - t.Capture(); \ - t.Log(); \ + t.capture(); \ + t.logTrace(); \ } while (0) /** @@ -23,16 +16,15 @@ * stack traces on demand. To use this just do: * * BackwardsTrace tracer; - * tracer.Capture(); // Trace is captured as of here. - * tracer.Log(); // Output the captured trace to the log. + * tracer.capture(); // Trace is captured as of here. + * tracer.logTrace(); // Output the captured trace to the log. * * The capture and log steps are separated to enable debugging in the case where * you want to capture a stack trace from inside some logic but don't know whether * you want to bother logging it until later. * * For convenience a macro is provided BACKTRACE_LOG() which performs the - * construction, capture, and log in one shot. Use BACKTRACE_PROD_LOG() if you - * want the logs to appear in production (critical level, with NDEBUG) + * construction, capture, and log in one shot. * * To resolve the addresses in the backtrace output and de-interleave * multithreaded output use the tools/stack_decode.py command and pass the @@ -51,81 +43,63 @@ */ class BackwardsTrace : Logger::Loggable { public: + BackwardsTrace() {} + /** - * Construct a BackwardsTrace with optional production enablement. - * - * If the optional argument log_in_prod is true then this BackwardsTrace will - * capture even when NDEBUG is defined and will log at critical level. This - * allows defining backtrace captures associated with PANIC() and other - * serious errors encountered in real production while also defining backtrace - * captures used only in debug builds. - * - * By default log_in_prod is false and if NDEBUG is defined these methods are - * all no-ops. Backtraces will be logged at debug level when NDEBUG is not defined. + * Capture a stack trace. * - * @param log_in_prod Log at a critical level so we see output even in - * optimized builds. + * The trace will begin with the call to capture(). */ - BackwardsTrace(const bool log_in_prod = false) : log_in_prod_(log_in_prod) {} + void capture() { stack_trace_.load_here(MAX_STACK_DEPTH); } /** - * Capture a stack trace. + * Capture a stack trace from a particular address. * - * The trace will begin with the call to Capture(). + * This can be used to capture a useful stack trace from a fatal signal + * handler. + * + * @param address The stack trace will begin from this address. */ - void Capture() { -#ifdef NDEBUG - if (!log_in_prod_) { - return; - } -#endif - stack_trace_.load_here(MAX_STACK_DEPTH); - } + void captureFrom(void* address) { stack_trace_.load_from(address, MAX_STACK_DEPTH); } /** * Log the stack trace. */ - void Log() { -#ifdef NDEBUG - if (!log_in_prod_) { - return; - } -#endif + void logTrace() { backward::TraceResolver resolver; resolver.load_stacktrace(stack_trace_); // If there's nothing in the captured trace we cannot do anything. - if (stack_trace_.size() < 1) { - LogAtLevel("Back trace attempt failed"); + // The size must be at least two for useful info - there is a sentinel frame + // at the end that we ignore. + if (stack_trace_.size() < 2) { + log().critical("Back trace attempt failed"); return; } const auto thread_id = stack_trace_.thread_id(); - // Trick to figure out our own object file name for script to use: - backward::ResolvedTrace trace = resolver.resolve(stack_trace_[0]); - auto obj_name = trace.object_filename; - if (obj_name.empty()) { - obj_name = "path/to/envoy-executable"; - } + backward::ResolvedTrace first_frame_trace = resolver.resolve(stack_trace_[0]); + auto obj_name = first_frame_trace.object_filename; - LogAtLevel("Backtrace obj<{}> thr<{}> (use tools/stack_decode.py):", obj_name, thread_id); + log().critical("Backtrace obj<{}> thr<{}> (use tools/stack_decode.py):", obj_name, thread_id); - // Why start at 2? To hide the function call to backward that began the - // trace. - for (unsigned int i = 0; i < stack_trace_.size(); ++i) { - LogAtLevel("thr<{}> #{} {}", thread_id, stack_trace_[i].idx, stack_trace_[i].addr); + // Backtrace gets tagged by ASAN when we try the object name resolution for the last + // frame on stack, so skip the last one. It has no useful info anyway. + for (unsigned int i = 0; i < stack_trace_.size() - 1; ++i) { + backward::ResolvedTrace trace = resolver.resolve(stack_trace_[i]); + if (trace.object_filename != obj_name) { + obj_name = trace.object_filename; + log().critical("thr<{}> obj<{}>", thread_id, obj_name); + } + log().critical("thr<{}> #{} {}", thread_id, stack_trace_[i].idx, stack_trace_[i].addr); } - LogAtLevel("end backtrace thread {}", stack_trace_.thread_id()); + log().critical("end backtrace thread {}", stack_trace_.thread_id()); } -private: - template void LogAtLevel(T format_str, Args... args) { - if (log_in_prod_) { - log().critical(format_str, args...); - } else { - log().debug(format_str, args...); - } + void logFault(const char* signame, const void* addr) { + log().critical("Caught {}, suspect faulting address {}", signame, addr); } + +private: static const int MAX_STACK_DEPTH = 64; backward::StackTrace stack_trace_; - const bool log_in_prod_; }; diff --git a/test/exe/BUILD b/test/exe/BUILD index afd99286af1c4..55dd02fd0b474 100644 --- a/test/exe/BUILD +++ b/test/exe/BUILD @@ -2,6 +2,7 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", + "envoy_cc_test", "envoy_package", ) @@ -15,3 +16,9 @@ sh_test( # test when doing ASAN. tags = ["no_asan"], ) + +envoy_cc_test( + name = "signals_test", + srcs = ["signals_test.cc"], + deps = ["//source/exe:sigaction_lib"], +) diff --git a/test/exe/signals_test.cc b/test/exe/signals_test.cc new file mode 100644 index 0000000000000..d1abc187e6af3 --- /dev/null +++ b/test/exe/signals_test.cc @@ -0,0 +1,96 @@ +#include +#include + +#include "exe/signal_action.h" + +#include "gtest/gtest.h" + +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#define ASANITIZED /* Sanitized by Clang */ +#endif +#endif + +#if defined(__SANITIZE_ADDRESS__) +#define ASANITIZED /* Sanitized by GCC */ +#endif + +// Memory violation signal tests are disabled under address sanitizer. The +// sanitizer does its own special signal handling and prints messages that are +// not ours instead of what this test expects. The signals special handled by ASAN +// include SIGSEGV, SIGBUS, and SIGFPE. +#ifndef ASANITIZED +TEST(Signals, InvalidAddressDeathTest) { + SignalAction actions; + EXPECT_DEATH([]() -> void { + // Oooooops! + volatile int* nasty_ptr = reinterpret_cast(0x0); + *(nasty_ptr) = 0; + }(), "Segmentation fault"); +} + +TEST(Signals, BusDeathTest) { + SignalAction actions; + EXPECT_DEATH([]() -> void { + // Bus error is tricky. There's one way that can work on POSIX systems + // described below but it depends on mmaping a file. Just make it easy and + // raise a bus. + // + // FILE *f = tmpfile(); + // int *p = mmap(0, 4, PROT_WRITE, MAP_PRIVATE, fileno(f), 0); + // *p = 0; + raise(SIGBUS); + }(), "Bus"); +} + +TEST(Signals, BadMathDeathTest) { + SignalAction actions; + EXPECT_DEATH([]() -> void { + // It turns out to be really hard to not have the optimizer get rid of a + // division by zero. Just raise the signal for this test. + raise(SIGFPE); + }(), "Floating point"); +} +#endif + +#if defined(__x86_64__) || defined(__i386__) +// Unfortunately we don't have a reliable way to do this on other platforms +TEST(Signals, IllegalInstructionDeathTest) { + SignalAction actions; + EXPECT_DEATH([]() -> void { + // Intel defines the "ud2" opcode to be an invalid instruction: + __asm__("ud2"); + }(), "Illegal"); +} +#endif + +TEST(Signals, AbortDeathTest) { + SignalAction actions; + EXPECT_DEATH([]() -> void { abort(); }(), "Aborted"); +} + +TEST(Signals, IllegalStackAccessDeathTest) { + SignalAction actions; + EXPECT_DEATH(actions.tryEvilAccessForTest(false), ""); + EXPECT_DEATH(actions.tryEvilAccessForTest(true), ""); +} + +TEST(Signals, LegalTest) { + // Don't do anything wrong. + { SignalAction actions; } + // Nothing should happen... +} + +TEST(Signals, RaiseNonFatalTest) { + { + SignalAction actions; + // I urgently request that you do nothing please! + raise(SIGURG); + } + // Nothing should happen... +} + +TEST(Signals, LegalStackAccessTest) { + SignalAction actions; + actions.doGoodAccessForTest(); +} diff --git a/test/server/backtrace_test.cc b/test/server/backtrace_test.cc index 6c6da20deb133..08df4c3c3ccf5 100644 --- a/test/server/backtrace_test.cc +++ b/test/server/backtrace_test.cc @@ -6,13 +6,13 @@ TEST(Backward, Basic) { // There isn't much to test here and this feature is really just useful for // debugging. This test simply verifies that we do not cause a crash when // logging a backtrace, and covers the added lines. - BackwardsTrace tracer(true); // Log at a level that means we cover lines even in opt - tracer.Capture(); - tracer.Log(); + BackwardsTrace tracer; + tracer.capture(); + tracer.logTrace(); } TEST(Backward, InvalidUsageTest) { // Ensure we do not crash if logging is attempted when there was no trace captured - BackwardsTrace tracer(true); - tracer.Log(); + BackwardsTrace tracer; + tracer.logTrace(); } diff --git a/tools/bazel.rc b/tools/bazel.rc index 0983941902e29..81edaa7abbf53 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -8,3 +8,4 @@ build:asan --linkopt -ldl build:asan --define tcmalloc=disabled build:asan --build_tag_filters=-no_asan build:asan --test_tag_filters=-no_asan +build:asan --define signal_trace=disabled diff --git a/tools/stack_decode.py b/tools/stack_decode.py index b7976e1373efd..e95af62c14192 100755 --- a/tools/stack_decode.py +++ b/tools/stack_decode.py @@ -17,8 +17,8 @@ import subprocess import sys -Backtrace = collections.namedtuple("Backtrace", - "log_prefix obj_file address_list") +Backtrace = collections.namedtuple("Backtrace", "log_prefix obj_list") +AddressList = collections.namedtuple("AddressList", "obj_file addresses") # Process the log output looking for stacktrace snippets, print them out once @@ -28,6 +28,7 @@ def decode_stacktrace_log(input_source): trace_begin_re = re.compile( "^(.+)\[backtrace\] Backtrace obj<(.+)> thr<(\d+)") stackaddr_re = re.compile("\[backtrace\] thr<(\d+)> #\d+ (0x[0-9a-fA-F]+)$") + new_object_re = re.compile("\[backtrace\] thr<(\d+)> obj<(.+)>$") trace_end_re = re.compile("\[backtrace\] end backtrace thread (\d+)") # build a dictionary indexed by thread_id, value is a Backtrace namedtuple @@ -39,13 +40,20 @@ def decode_stacktrace_log(input_source): begin_trace_match = trace_begin_re.search(line) if begin_trace_match: log_prefix, objfile, thread_id = begin_trace_match.groups() - traces[thread_id] = Backtrace( - log_prefix=log_prefix, obj_file=objfile, address_list=[]) + traces[thread_id] = Backtrace(log_prefix=log_prefix, obj_list=[]) + traces[thread_id].obj_list.append( + AddressList(obj_file=objfile, addresses=[])) continue stackaddr_match = stackaddr_re.search(line) if stackaddr_match: thread_id, address = stackaddr_match.groups() - traces[thread_id].address_list.append(address) + traces[thread_id].obj_list[-1].addresses.append(address) + continue + new_object_match = new_object_re.search(line) + if new_object_match: + thread_id, newobj = new_object_match.groups() + traces[thread_id].obj_list.append( + AddressList(obj_file=newobj, addresses=[])) continue trace_end_match = trace_end_re.search(line) if trace_end_match: @@ -58,18 +66,29 @@ def decode_stacktrace_log(input_source): return -# Output one stacktrace after passing it through addr2line with appropriate -# options -def output_stacktrace(thread_id, traceinfo): - piped_input = "" - for stack_addr in traceinfo.address_list: - piped_input += (stack_addr + "\n") +# Execute addr2line with a particular object file and input string of addresses +# to resolve, one per line. +# +# Returns list of result lines +def run_addr2line(obj_file, piped_input): addr2line = subprocess.Popen( - ["addr2line", "-Cpisfe", traceinfo.obj_file], + ["addr2line", "-Cpisfe", obj_file], stdin=subprocess.PIPE, stdout=subprocess.PIPE) output_stdout, _ = addr2line.communicate(piped_input) - output_lines = output_stdout.split("\n") + return output_stdout.split("\n") + + +# Output one stacktrace after passing it through addr2line with appropriate +# options +def output_stacktrace(thread_id, traceinfo): + output_lines = [] + for address_list in traceinfo.obj_list: + piped_input = "" + obj_name = address_list.obj_file + for stack_addr in address_list.addresses: + piped_input += (stack_addr + "\n") + output_lines += run_addr2line(obj_name, piped_input) resolved_stack_frames = enumerate(output_lines, start=1) sys.stdout.write("%s Backtrace (most recent call first) from thread %s:\n" % @@ -83,6 +102,8 @@ def output_stacktrace(thread_id, traceinfo): rununder = subprocess.Popen( sys.argv[1:], stdout=subprocess.PIPE, stderr=subprocess.PIPE) decode_stacktrace_log(rununder.stderr) + rununder.wait() + sys.exit(rununder.returncode) # Pass back test pass/fail result else: decode_stacktrace_log(sys.stdin) sys.exit(0)