From c2c01825c1129eb4284288b207b95bfe498e4561 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Sun, 6 Aug 2023 15:53:53 +0800 Subject: [PATCH] [opt](stacktrace) Optimize stacktrace output #22467 --- be/src/common/config.cpp | 1 + be/src/common/config.h | 6 +++ be/src/common/stack_trace.cpp | 75 ++++++++++++++++++++++------------ be/src/util/stack_util.cpp | 6 +-- thirdparty/build-thirdparty.sh | 6 +-- 5 files changed, 61 insertions(+), 33 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 2f8445e73f..12eb3caaa2 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1028,6 +1028,7 @@ DEFINE_Bool(allow_invalid_decimalv2_literal, "false"); DEFINE_mInt64(kerberos_expiration_time_seconds, "43200"); DEFINE_mString(get_stack_trace_tool, "libunwind"); +DEFINE_mString(dwarf_location_info_mode, "FAST"); // the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer DEFINE_mInt64(auto_inc_prefetch_size_ratio, "10"); diff --git a/be/src/common/config.h b/be/src/common/config.h index a1e2afdfe5..752d008ed4 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1072,6 +1072,12 @@ DECLARE_mInt64(kerberos_expiration_time_seconds); // Values include `none`, `glog`, `boost`, `glibc`, `libunwind` DECLARE_mString(get_stack_trace_tool); +// DISABLED: Don't resolve location info. +// FAST: Perform CU lookup using .debug_aranges (might be incomplete). +// FULL: Scan all CU in .debug_info (slow!) on .debug_aranges lookup failure. +// FULL_WITH_INLINE: Scan .debug_info (super slower, use with caution) for inline functions in addition to FULL. +DECLARE_mString(dwarf_location_info_mode); + // the ratio of _prefetch_size/_batch_size in AutoIncIDBuffer DECLARE_mInt64(auto_inc_prefetch_size_ratio); diff --git a/be/src/common/stack_trace.cpp b/be/src/common/stack_trace.cpp index 9e951b6532..f5fb2a1901 100644 --- a/be/src/common/stack_trace.cpp +++ b/be/src/common/stack_trace.cpp @@ -34,11 +34,14 @@ #include #include "config.h" +#include "util/string_util.h" #include "vec/common/demangle.h" #include "vec/common/hex.h" #if USE_UNWIND #include +#else +#include #endif namespace { @@ -294,12 +297,14 @@ StackTrace::StackTrace(const ucontext_t& signal_context) { } void StackTrace::tryCapture() { + // When unw_backtrace is not available, fall back on the standard + // `backtrace` function from execinfo.h. #if USE_UNWIND size = unw_backtrace(frame_pointers.data(), capacity); - __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); #else - size = 0; + size = backtrace(frame_pointers.data(), capacity); #endif + __msan_unpoison(frame_pointers.data(), size * sizeof(frame_pointers[0])); } /// ClickHouse uses bundled libc++ so type names will be the same on every system thus it's safe to hardcode them @@ -340,7 +345,7 @@ constexpr bool operator<(const MaybeRef auto& left, const MaybeRef auto& right) std::tuple {right.pointers, right.size, right.offset}; } -static void toStringEveryLineImpl([[maybe_unused]] bool fatal, +static void toStringEveryLineImpl([[maybe_unused]] const std::string dwarf_location_info_mode, const StackTraceRefTriple& stack_trace, std::function callback) { if (stack_trace.size == 0) { @@ -349,7 +354,20 @@ static void toStringEveryLineImpl([[maybe_unused]] bool fatal, #if defined(__ELF__) && !defined(__FreeBSD__) using enum doris::Dwarf::LocationInfoMode; - const auto mode = fatal ? FULL_WITH_INLINE : FAST; + doris::Dwarf::LocationInfoMode mode; + auto dwarf_location_info_mode_lower = doris::to_lower(dwarf_location_info_mode); + if (dwarf_location_info_mode_lower == "disabled") { + mode = DISABLED; + } else if (dwarf_location_info_mode_lower == "fast") { + mode = FAST; + } else if (dwarf_location_info_mode_lower == "full") { + mode = FULL; + } else if (dwarf_location_info_mode_lower == "full_with_inline") { + mode = FULL_WITH_INLINE; + } else { + LOG(INFO) << "invalid LocationInfoMode: " << dwarf_location_info_mode; + mode = DISABLED; + } auto symbol_index_ptr = doris::SymbolIndex::instance(); const doris::SymbolIndex& symbol_index = *symbol_index_ptr; std::unordered_map dwarfs; @@ -362,7 +380,21 @@ static void toStringEveryLineImpl([[maybe_unused]] bool fatal, reinterpret_cast(uintptr_t(virtual_addr) - virtual_offset); std::stringstream out; - out << i << ". "; + out << "\t" << i << ". "; + if (i < 10) { // for alignment + out << " "; + } + + if (shouldShowAddress(physical_addr)) { + out << "@ "; + writePointerHex(physical_addr, out); + } + + if (const auto* const symbol = symbol_index.findSymbol(virtual_addr)) { + out << " " << collapseNames(demangle(symbol->name)); + } else { + out << " ?"; + } if (std::error_code ec; object && std::filesystem::exists(object->name, ec) && !ec) { auto dwarf_it = dwarfs.try_emplace(object->name, object->elf).first; @@ -371,31 +403,20 @@ static void toStringEveryLineImpl([[maybe_unused]] bool fatal, if (dwarf_it->second.findAddress(uintptr_t(physical_addr), location, mode, inline_frames)) { - out << location.file.toString() << ":" << location.line << ": "; + out << " " << location.file.toString() << ":" << location.line; } } - if (const auto* const symbol = symbol_index.findSymbol(virtual_addr)) { - out << collapseNames(demangle(symbol->name)); - } else { - out << "?"; - } + out << " in " << (object ? object->name : "?"); - if (shouldShowAddress(physical_addr)) { - out << " @ "; - writePointerHex(physical_addr, out); - } - - out << " in " << (object ? object->name : "?"); + callback(out.str()); for (size_t j = 0; j < inline_frames.size(); ++j) { const auto& frame = inline_frames[j]; - callback(fmt::format("{}.{}. inlined from {}:{}: {}", i, j + 1, - frame.location.file.toString(), frame.location.line, - collapseNames(demangle(frame.name)))); + callback(fmt::format("\t{}.{}. inlined from {}: {}:{}", i, j + 1, + collapseNames(demangle(frame.name)), + frame.location.file.toString(), frame.location.line)); } - - callback(out.str()); } #else for (size_t i = stack_trace.offset; i < stack_trace.size; ++i) @@ -405,7 +426,7 @@ static void toStringEveryLineImpl([[maybe_unused]] bool fatal, } void StackTrace::toStringEveryLine(std::function callback) const { - toStringEveryLineImpl(true, {frame_pointers, offset, size}, std::move(callback)); + toStringEveryLineImpl("FULL_WITH_INLINE", {frame_pointers, offset, size}, std::move(callback)); } using StackTraceCache = std::map>; @@ -430,14 +451,18 @@ std::string toStringCached(const StackTrace::FramePointers& pointers, size_t off return it->second; } else { std::stringstream out; - toStringEveryLineImpl(false, key, [&](std::string_view str) { out << str << '\n'; }); + toStringEveryLineImpl(doris::config::dwarf_location_info_mode, key, + [&](std::string_view str) { out << str << '\n'; }); return cache.emplace(StackTraceTriple {pointers, offset, size}, out.str()).first->second; } } std::string StackTrace::toString() const { - return toStringCached(frame_pointers, offset, size); + // Delete the first three frame pointers, which are inside the stacktrace. + StackTrace::FramePointers frame_pointers_raw {}; + std::copy(frame_pointers.begin() + 3, frame_pointers.end(), frame_pointers_raw.begin()); + return toStringCached(frame_pointers_raw, offset, size - 3); } std::string StackTrace::toString(void** frame_pointers_raw, size_t offset, size_t size) { diff --git a/be/src/util/stack_util.cpp b/be/src/util/stack_util.cpp index dd143ead3f..01ced12708 100644 --- a/be/src/util/stack_util.cpp +++ b/be/src/util/stack_util.cpp @@ -45,11 +45,7 @@ std::string get_stack_trace() { } else if (tool == "glibc") { return get_stack_trace_by_glibc(); } else if (tool == "libunwind") { -#if USE_UNWIND return get_stack_trace_by_libunwind(); -#else - return get_stack_trace_by_glog(); -#endif } else { return "no stack"; } @@ -82,7 +78,7 @@ std::string get_stack_trace_by_glibc() { } std::string get_stack_trace_by_libunwind() { - return StackTrace().toString(); + return "\n" + StackTrace().toString(); } } // namespace doris diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 1f376d0d68..a9548c6b0f 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -1434,9 +1434,9 @@ build_jemalloc() { # libunwind build_libunwind() { - # https://github.com/libunwind/libunwind - # https://github.com/libunwind/libunwind/issues/189 - # https://stackoverflow.com/questions/27842377/building-libunwind-for-mac + # There are two major variants of libunwind. libunwind on Linux + # (https://www.nongnu.org/libunwind/) provides unw_backtrace, and + # Apache/LLVM libunwind (notably used on Apple platforms) doesn't if [[ "${KERNEL}" != 'Darwin' ]]; then check_if_source_exist "${LIBUNWIND_SOURCE}" cd "${TP_SOURCE_DIR}/${LIBUNWIND_SOURCE}"