From fdf66b81028e9e91e9b53227a46a066d49ca5211 Mon Sep 17 00:00:00 2001 From: HuangWei Date: Mon, 1 Jun 2020 23:16:25 +0800 Subject: [PATCH] [MemTracker] add log depth & auto unregister (#3701) --- be/src/runtime/mem_tracker.cpp | 62 +++++++++++++----------- be/src/runtime/mem_tracker.h | 86 ++++++++++++++++------------------ 2 files changed, 75 insertions(+), 73 deletions(-) diff --git a/be/src/runtime/mem_tracker.cpp b/be/src/runtime/mem_tracker.cpp index 0b6f17c71d..0f0a0c0667 100644 --- a/be/src/runtime/mem_tracker.cpp +++ b/be/src/runtime/mem_tracker.cpp @@ -28,12 +28,12 @@ #include "gutil/strings/substitute.h" #include "runtime/exec_env.h" #include "runtime/runtime_state.h" -#include "util/doris_metrics.h" #include "util/debug_util.h" +#include "util/doris_metrics.h" #include "util/mem_info.h" #include "util/pretty_printer.h" -#include "util/uid_util.h" #include "util/stack_util.h" +#include "util/uid_util.h" //using std::shared_ptr; //using std::weak_ptr; @@ -50,7 +50,7 @@ const std::string MemTracker::COUNTER_NAME = "PeakMemoryUsage"; const std::string REQUEST_POOL_MEM_TRACKER_LABEL_FORMAT = "RequestPool=$0"; MemTracker::MemTracker( - int64_t byte_limit, const std::string& label, MemTracker* parent, bool log_usage_if_zero) + int64_t byte_limit, const std::string& label, MemTracker* parent, bool auto_unregister, bool log_usage_if_zero) : _limit(byte_limit), _label(label), _parent(parent), @@ -61,7 +61,8 @@ MemTracker::MemTracker( _num_gcs_metric(NULL), _bytes_freed_by_last_gc_metric(NULL), _bytes_over_limit_metric(NULL), - _limit_metric(NULL) { + _limit_metric(NULL), + _auto_unregister(auto_unregister) { if (parent != NULL) _parent->add_child_tracker(this); Init(); } @@ -113,8 +114,7 @@ void MemTracker::Init() { } // TODO chenhao , set MemTracker close state -void MemTracker::close() { -} +void MemTracker::close() {} void MemTracker::enable_reservation_reporting(const ReservationTrackerCounters& counters) { ReservationTrackerCounters* new_counters = new ReservationTrackerCounters(counters); @@ -124,7 +124,7 @@ void MemTracker::enable_reservation_reporting(const ReservationTrackerCounters& int64_t MemTracker::GetPoolMemReserved() const { // Pool trackers should have a _pool_name and no limit. DCHECK(!_pool_name.empty()); - DCHECK_EQ(_limit, -1) << LogUsage(""); + DCHECK_EQ(_limit, -1) << LogUsage(UNLIMITED_DEPTH); int64_t mem_reserved = 0L; std::lock_guard l(_child_trackers_lock); @@ -134,7 +134,7 @@ int64_t MemTracker::GetPoolMemReserved() const { // Make sure we don't overflow if the query limits are set to ridiculous values. mem_reserved += std::min(child_limit, MemInfo::physical_mem()); } else { - DCHECK_EQ(child_limit, -1) << child->LogUsage(""); + DCHECK_EQ(child_limit, -1) << child->LogUsage(UNLIMITED_DEPTH); mem_reserved += child->consumption(); } } @@ -195,6 +195,10 @@ MemTracker::~MemTracker() { } } delete _reservation_counters.load(); + + if (_auto_unregister && parent()) { + unregister_from_parent(); + } } // Calling this on the query tracker results in output like: @@ -218,7 +222,8 @@ MemTracker::~MemTracker() { // TrackerName: Limit=5.00 MB BufferPoolUsed/Reservation=0/5.00 MB OtherMemory=1.04 MB // Total=6.04 MB Peak=6.45 MB // -std::string MemTracker::LogUsage(const std::string& prefix, int64_t* logged_consumption) const { +std::string MemTracker::LogUsage(int max_recursive_depth, const std::string& prefix, + int64_t* logged_consumption) const { int64_t curr_consumption = consumption(); int64_t peak_consumption = _consumption->value(); if (logged_consumption != nullptr) *logged_consumption = curr_consumption; @@ -234,32 +239,33 @@ std::string MemTracker::LogUsage(const std::string& prefix, int64_t* logged_cons ReservationTrackerCounters* reservation_counters = _reservation_counters.load(); if (reservation_counters != nullptr) { int64_t reservation = reservation_counters->peak_reservation->current_value(); - int64_t used_reservation = - reservation_counters->peak_used_reservation->current_value(); + int64_t used_reservation = reservation_counters->peak_used_reservation->current_value(); int64_t reservation_limit = 0; - //TODO chenhao, reservation_limit is null when ReservationTracker + //TODO chenhao, reservation_limit is null when ReservationTracker // does't have reservation limit if (reservation_counters->reservation_limit != nullptr) { - reservation_limit = reservation_counters->reservation_limit->value(); + reservation_limit = reservation_counters->reservation_limit->value(); } - ss << " BufferPoolUsed/Reservation=" - << PrettyPrinter::print(used_reservation, TUnit::BYTES) << "/" - << PrettyPrinter::print(reservation, TUnit::BYTES); + ss << " BufferPoolUsed/Reservation=" << PrettyPrinter::print(used_reservation, TUnit::BYTES) + << "/" << PrettyPrinter::print(reservation, TUnit::BYTES); if (reservation_limit != std::numeric_limits::max()) { ss << " BufferPoolLimit=" << PrettyPrinter::print(reservation_limit, TUnit::BYTES); } - ss << " OtherMemory=" - << PrettyPrinter::print(curr_consumption - reservation, TUnit::BYTES); + ss << " OtherMemory=" << PrettyPrinter::print(curr_consumption - reservation, TUnit::BYTES); } ss << " Total=" << PrettyPrinter::print(curr_consumption, TUnit::BYTES) - << " Peak=" << PrettyPrinter::print(peak_consumption, TUnit::BYTES); + << " Peak=" << PrettyPrinter::print(peak_consumption, TUnit::BYTES); + + // This call does not need the children, so return early. + if (max_recursive_depth == 0) return ss.str(); std::string new_prefix = strings::Substitute(" $0", prefix); int64_t child_consumption; std::string child_trackers_usage; { std::lock_guard l(_child_trackers_lock); - child_trackers_usage = LogUsage(new_prefix, _child_trackers, &child_consumption); + child_trackers_usage = + LogUsage(max_recursive_depth - 1, new_prefix, _child_trackers, &child_consumption); } if (!child_trackers_usage.empty()) ss << "\n" << child_trackers_usage; @@ -269,23 +275,24 @@ std::string MemTracker::LogUsage(const std::string& prefix, int64_t* logged_cons // we did not necessarily get a consistent snapshot of the consumption values for all // children at a single moment in time, but is good enough for our purposes. int64_t untracked_bytes = curr_consumption - child_consumption; + ss << "\n" << new_prefix << "Untracked Memory: Total="; ss << "\n" - << new_prefix << "Untracked Memory: Total="; - ss << "\n" - << new_prefix << "Untracked Memory: Total=" - << PrettyPrinter::print(untracked_bytes, TUnit::BYTES); + << new_prefix + << "Untracked Memory: Total=" << PrettyPrinter::print(untracked_bytes, TUnit::BYTES); } return ss.str(); } -std::string MemTracker::LogUsage(const std::string& prefix, const std::list& trackers, - int64_t* logged_consumption) { +std::string MemTracker::LogUsage(int max_recursive_depth, const std::string& prefix, + const std::list& trackers, + int64_t* logged_consumption) { *logged_consumption = 0; std::vector usage_strings; for (MemTracker* tracker : trackers) { int64_t tracker_consumption; - std::string usage_string = tracker->LogUsage(prefix, &tracker_consumption); + std::string usage_string = + tracker->LogUsage(max_recursive_depth, prefix, &tracker_consumption); if (!usage_string.empty()) usage_strings.push_back(usage_string); *logged_consumption += tracker_consumption; } @@ -363,4 +370,3 @@ bool MemTracker::GcMemory(int64_t max_consumption) { } } // end namespace doris - diff --git a/be/src/runtime/mem_tracker.h b/be/src/runtime/mem_tracker.h index 416332807a..cbcc500ef7 100644 --- a/be/src/runtime/mem_tracker.h +++ b/be/src/runtime/mem_tracker.h @@ -24,11 +24,11 @@ #include #include +#include "common/status.h" #include "gen_cpp/Types_types.h" #include "util/metrics.h" #include "util/runtime_profile.h" #include "util/spinlock.h" -#include "common/status.h" namespace doris { @@ -68,10 +68,11 @@ class MemTracker { public: /// 'byte_limit' < 0 means no limit /// 'label' is the label used in the usage string (LogUsage()) + /// If 'auto_unregister' is true, never call unregister_from_parent(). /// If 'log_usage_if_zero' is false, this tracker (and its children) will not be included /// in LogUsage() output if consumption is 0. MemTracker(int64_t byte_limit = -1, const std::string& label = std::string(), - MemTracker* parent = NULL, bool log_usage_if_zero = true); + MemTracker* parent = NULL, bool auto_unregister = false, bool log_usage_if_zero = true); /// C'tor for tracker for which consumption counter is created as part of a profile. /// The counter is created with name COUNTER_NAME. @@ -227,7 +228,8 @@ public: /// consistent.) if ((*tracker)->_consumption_metric == NULL) { DCHECK_GE((*tracker)->_consumption->current_value(), 0) - << std::endl << (*tracker)->LogUsage(); + << std::endl + << (*tracker)->LogUsage(UNLIMITED_DEPTH); } } @@ -277,22 +279,13 @@ public: _consumption->set(_consumption_metric->value()); } + bool limit_exceeded() const { return _limit >= 0 && _limit < consumption(); } - bool limit_exceeded() const{ - return _limit >= 0 && _limit < consumption(); - } + int64_t limit() const { return _limit; } - int64_t limit() const { - return _limit; - } + bool has_limit() const { return _limit >= 0; } - bool has_limit() const { - return _limit >= 0; - } - - const std::string& label() const { - return _label; - } + const std::string& label() const { return _label; } /// Returns the lowest limit for this tracker and its ancestors. Returns /// -1 if there is no limit. @@ -312,19 +305,14 @@ public: /// admission control). Otherwise the current consumption is used. int64_t GetPoolMemReserved() const; - int64_t consumption() const { - return _consumption->current_value();; - } - + int64_t consumption() const { return _consumption->current_value(); } /// Note that if _consumption is based on _consumption_metric, this will the max value /// we've recorded in consumption(), not necessarily the highest value /// _consumption_metric has ever reached. int64_t peak_consumption() const { return _consumption->value(); } - MemTracker* parent() const { - return _parent; - } + MemTracker* parent() const { return _parent; } /// Signature for function that can be called to free some memory after limit is /// reached. The function should try to free at least 'bytes_to_free' bytes of @@ -342,12 +330,24 @@ public: /// ".". void RegisterMetrics(MetricRegistry* metrics, const std::string& prefix); - /// Logs the usage of this tracker and all of its children (recursively). + /// Logs the usage of this tracker and optionally its children (recursively). /// If 'logged_consumption' is non-NULL, sets the consumption value logged. + /// 'max_recursive_depth' specifies the maximum number of levels of children + /// to include in the dump. If it is zero, then no children are dumped. + /// Limiting the recursive depth reduces the cost of dumping, particularly + /// for the process MemTracker. /// TODO: once all memory is accounted in ReservationTracker hierarchy, move /// reporting there. - std::string LogUsage( - const std::string& prefix = "", int64_t* logged_consumption = nullptr) const; + std::string LogUsage(int max_recursive_depth, const std::string& prefix = "", + int64_t* logged_consumption = nullptr) const; + /// Dumping the process MemTracker is expensive. Limiting the recursive depth + /// to two levels limits the level of detail to a one-line summary for each query + /// MemTracker, avoiding all MemTrackers below that level. This provides a summary + /// of process usage with substantially lower cost than the full dump. + static const int PROCESS_MEMTRACKER_LIMITED_DEPTH = 2; + /// Unlimited dumping is useful for query memtrackers or error conditions that + /// are not performance sensitive + static const int UNLIMITED_DEPTH = INT_MAX; /// Log the memory usage when memory limit is exceeded and return a status object with /// details of the allocation which caused the limit to be exceeded. @@ -356,8 +356,6 @@ public: Status MemLimitExceeded(RuntimeState* state, const std::string& details, int64_t failed_allocation = 0); - static const int UNLIMITED_DEPTH = INT_MAX; - static const std::string COUNTER_NAME; static void update_limits(int64_t bytes, std::vector* limits) { @@ -367,8 +365,7 @@ public: } static bool limit_exceeded(const std::vector& limits) { - for (std::vector::const_iterator i = limits.begin(); i != limits.end(); - ++i) { + for (std::vector::const_iterator i = limits.begin(); i != limits.end(); ++i) { if ((*i)->limit_exceeded()) { // TODO: remove logging LOG(WARNING) << "exceeded limit: limit=" << (*i)->limit() << " consumption=" @@ -391,10 +388,8 @@ public: return msg.str(); } - bool is_consumption_metric_null() { - return _consumption_metric == nullptr; - } - + bool is_consumption_metric_null() { return _consumption_metric == nullptr; } + private: friend class PoolMemTrackerRegistry; @@ -413,9 +408,11 @@ private: } /// Log consumption of all the trackers provided. Returns the sum of consumption in - /// 'logged_consumption'. - static std::string LogUsage(const std::string& prefix, - const std::list& trackers, int64_t* logged_consumption); + /// 'logged_consumption'. 'max_recursive_depth' specifies the maximum number of levels + /// of children to include in the dump. If it is zero, then no children are dumped. + static std::string LogUsage(int max_recursive_depth, const std::string& prefix, + const std::list& trackers, + int64_t* logged_consumption); /// Lock to protect GcMemory(). This prevents many GCs from occurring at once. std::mutex _gc_lock; @@ -427,7 +424,7 @@ private: // contains only weak ptrs. MemTrackers that are handed out via get_query_mem_tracker() // are shared ptrs. When all the shared ptrs are no longer referenced, the MemTracker // d'tor will be called at which point the weak ptr will be removed from the map. - typedef std::unordered_map > RequestTrackersMap; + typedef std::unordered_map> RequestTrackersMap; static RequestTrackersMap _s_request_to_mem_trackers; // Only valid for MemTrackers returned from get_query_mem_tracker() @@ -437,7 +434,7 @@ private: /// Only valid for MemTrackers returned from GetRequestPoolMemTracker() std::string _pool_name; - int64_t _limit; // in bytes + int64_t _limit; // in bytes //int64_t _consumption; // in bytes std::string _label; @@ -459,8 +456,8 @@ private: /// are owned by the fragment's RuntimeProfile. AtomicPtr _reservation_counters; - std::vector _all_trackers; // this tracker plus all of its ancestors - std::vector _limit_trackers; // _all_trackers with valid limits + std::vector _all_trackers; // this tracker plus all of its ancestors + std::vector _limit_trackers; // _all_trackers with valid limits // All the child trackers of this tracker. Used for error reporting only. // i.e., Updating a parent tracker does not update the children. @@ -497,7 +494,7 @@ private: // process tracker never gets deleted so it is safe to reference it in the dtor. // The query tracker has lifetime shared by multiple plan fragments so it's hard // to do cleanup another way. - bool _auto_unregister; + bool _auto_unregister = false; }; /// Global registry for query and pool MemTrackers. Owned by ExecEnv. @@ -510,8 +507,7 @@ class PoolMemTrackerRegistry { /// with the process tracker as its parent. There is no explicit per-pool byte_limit /// set at any particular impalad, so newly created trackers will always have a limit /// of -1. - MemTracker* GetRequestPoolMemTracker( - const std::string& pool_name, bool create_if_not_present); + MemTracker* GetRequestPoolMemTracker(const std::string& pool_name, bool create_if_not_present); private: /// All per-request pool MemTracker objects. It is assumed that request pools will live @@ -524,6 +520,6 @@ class PoolMemTrackerRegistry { SpinLock _pool_to_mem_trackers_lock; }; -} +} // namespace doris #endif