database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[MemTracker] add log depth & auto unregister (#3701)

Browse Source
This commit is contained in:
HuangWei
2020-06-01 23:16:25 +08:00
committed by GitHub
parent ee260d5721
commit fdf66b8102
2 changed files with 75 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
be/src/runtime/mem_tracker.cpp
View File

@ -28,12 +28,12 @@
#include "gutil/strings/substitute.h"
#include "runtime/exec_env.h"
#include "runtime/runtime_state.h"
#include "util/doris_metrics.h"
#include "util/debug_util.h"
#include "util/doris_metrics.h"
#include "util/mem_info.h"
#include "util/pretty_printer.h"
#include "util/uid_util.h"
#include "util/stack_util.h"
#include "util/uid_util.h"
//using std::shared_ptr;
//using std::weak_ptr;
@ -50,7 +50,7 @@ const std::string MemTracker::COUNTER_NAME = "PeakMemoryUsage";
const std::string REQUEST_POOL_MEM_TRACKER_LABEL_FORMAT = "RequestPool=$0";
MemTracker::MemTracker(
int64_t byte_limit, const std::string& label, MemTracker* parent, bool log_usage_if_zero)
int64_t byte_limit, const std::string& label, MemTracker* parent, bool auto_unregister, bool log_usage_if_zero)
: _limit(byte_limit),
_label(label),
_parent(parent),
@ -61,7 +61,8 @@ MemTracker::MemTracker(
_num_gcs_metric(NULL),
_bytes_freed_by_last_gc_metric(NULL),
_bytes_over_limit_metric(NULL),
_limit_metric(NULL) {
_limit_metric(NULL),
_auto_unregister(auto_unregister) {
if (parent != NULL) _parent->add_child_tracker(this);
Init();
}
@ -113,8 +114,7 @@ void MemTracker::Init() {
}
// TODO chenhao , set MemTracker close state
void MemTracker::close() {
}
void MemTracker::close() {}
void MemTracker::enable_reservation_reporting(const ReservationTrackerCounters& counters) {
ReservationTrackerCounters* new_counters = new ReservationTrackerCounters(counters);
@ -124,7 +124,7 @@ void MemTracker::enable_reservation_reporting(const ReservationTrackerCounters&
int64_t MemTracker::GetPoolMemReserved() const {
// Pool trackers should have a _pool_name and no limit.
DCHECK(!_pool_name.empty());
DCHECK_EQ(_limit, -1) << LogUsage("");
DCHECK_EQ(_limit, -1) << LogUsage(UNLIMITED_DEPTH);
int64_t mem_reserved = 0L;
std::lock_guard<std::mutex> l(_child_trackers_lock);
@ -134,7 +134,7 @@ int64_t MemTracker::GetPoolMemReserved() const {
// Make sure we don't overflow if the query limits are set to ridiculous values.
mem_reserved += std::min(child_limit, MemInfo::physical_mem());
} else {
DCHECK_EQ(child_limit, -1) << child->LogUsage("");
DCHECK_EQ(child_limit, -1) << child->LogUsage(UNLIMITED_DEPTH);
mem_reserved += child->consumption();
}
}
@ -195,6 +195,10 @@ MemTracker::~MemTracker() {
}
}
delete _reservation_counters.load();
if (_auto_unregister && parent()) {
unregister_from_parent();
}
}
// Calling this on the query tracker results in output like:
@ -218,7 +222,8 @@ MemTracker::~MemTracker() {
// TrackerName: Limit=5.00 MB BufferPoolUsed/Reservation=0/5.00 MB OtherMemory=1.04 MB
// Total=6.04 MB Peak=6.45 MB
//
std::string MemTracker::LogUsage(const std::string& prefix, int64_t* logged_consumption) const {
std::string MemTracker::LogUsage(int max_recursive_depth, const std::string& prefix,
int64_t* logged_consumption) const {
int64_t curr_consumption = consumption();
int64_t peak_consumption = _consumption->value();
if (logged_consumption != nullptr) *logged_consumption = curr_consumption;
@ -234,32 +239,33 @@ std::string MemTracker::LogUsage(const std::string& prefix, int64_t* logged_cons
ReservationTrackerCounters* reservation_counters = _reservation_counters.load();
if (reservation_counters != nullptr) {
int64_t reservation = reservation_counters->peak_reservation->current_value();
int64_t used_reservation =
reservation_counters->peak_used_reservation->current_value();
int64_t used_reservation = reservation_counters->peak_used_reservation->current_value();
int64_t reservation_limit = 0;
//TODO chenhao, reservation_limit is null when ReservationTracker
//TODO chenhao, reservation_limit is null when ReservationTracker
// does't have reservation limit
if (reservation_counters->reservation_limit != nullptr) {
reservation_limit = reservation_counters->reservation_limit->value();
reservation_limit = reservation_counters->reservation_limit->value();
}
ss << " BufferPoolUsed/Reservation="
<< PrettyPrinter::print(used_reservation, TUnit::BYTES) << "/"
<< PrettyPrinter::print(reservation, TUnit::BYTES);
ss << " BufferPoolUsed/Reservation=" << PrettyPrinter::print(used_reservation, TUnit::BYTES)
<< "/" << PrettyPrinter::print(reservation, TUnit::BYTES);
if (reservation_limit != std::numeric_limits<int64_t>::max()) {
ss << " BufferPoolLimit=" << PrettyPrinter::print(reservation_limit, TUnit::BYTES);
}
ss << " OtherMemory="
<< PrettyPrinter::print(curr_consumption - reservation, TUnit::BYTES);
ss << " OtherMemory=" << PrettyPrinter::print(curr_consumption - reservation, TUnit::BYTES);
}
ss << " Total=" << PrettyPrinter::print(curr_consumption, TUnit::BYTES)
<< " Peak=" << PrettyPrinter::print(peak_consumption, TUnit::BYTES);
<< " Peak=" << PrettyPrinter::print(peak_consumption, TUnit::BYTES);
// This call does not need the children, so return early.
if (max_recursive_depth == 0) return ss.str();
std::string new_prefix = strings::Substitute(" $0", prefix);
int64_t child_consumption;
std::string child_trackers_usage;
{
std::lock_guard<std::mutex> l(_child_trackers_lock);
child_trackers_usage = LogUsage(new_prefix, _child_trackers, &child_consumption);
child_trackers_usage =
LogUsage(max_recursive_depth - 1, new_prefix, _child_trackers, &child_consumption);
}
if (!child_trackers_usage.empty()) ss << "\n" << child_trackers_usage;
@ -269,23 +275,24 @@ std::string MemTracker::LogUsage(const std::string& prefix, int64_t* logged_cons
// we did not necessarily get a consistent snapshot of the consumption values for all
// children at a single moment in time, but is good enough for our purposes.
int64_t untracked_bytes = curr_consumption - child_consumption;
ss << "\n" << new_prefix << "Untracked Memory: Total=";
ss << "\n"
<< new_prefix << "Untracked Memory: Total=";
ss << "\n"
<< new_prefix << "Untracked Memory: Total="
<< PrettyPrinter::print(untracked_bytes, TUnit::BYTES);
<< new_prefix
<< "Untracked Memory: Total=" << PrettyPrinter::print(untracked_bytes, TUnit::BYTES);
}
return ss.str();
}
std::string MemTracker::LogUsage(const std::string& prefix, const std::list<MemTracker*>& trackers,
int64_t* logged_consumption) {
std::string MemTracker::LogUsage(int max_recursive_depth, const std::string& prefix,
const std::list<MemTracker*>& trackers,
int64_t* logged_consumption) {
*logged_consumption = 0;
std::vector<std::string> usage_strings;
for (MemTracker* tracker : trackers) {
int64_t tracker_consumption;
std::string usage_string = tracker->LogUsage(prefix, &tracker_consumption);
std::string usage_string =
tracker->LogUsage(max_recursive_depth, prefix, &tracker_consumption);
if (!usage_string.empty()) usage_strings.push_back(usage_string);
*logged_consumption += tracker_consumption;
}
@ -363,4 +370,3 @@ bool MemTracker::GcMemory(int64_t max_consumption) {
}
} // end namespace doris

86
be/src/runtime/mem_tracker.h
View File

@ -24,11 +24,11 @@
#include <mutex>
#include <unordered_map>
#include "common/status.h"
#include "gen_cpp/Types_types.h"
#include "util/metrics.h"
#include "util/runtime_profile.h"
#include "util/spinlock.h"
#include "common/status.h"
namespace doris {
@ -68,10 +68,11 @@ class MemTracker {
public:
/// 'byte_limit' < 0 means no limit
/// 'label' is the label used in the usage string (LogUsage())
/// If 'auto_unregister' is true, never call unregister_from_parent().
/// If 'log_usage_if_zero' is false, this tracker (and its children) will not be included
/// in LogUsage() output if consumption is 0.
MemTracker(int64_t byte_limit = -1, const std::string& label = std::string(),
MemTracker* parent = NULL, bool log_usage_if_zero = true);
MemTracker* parent = NULL, bool auto_unregister = false, bool log_usage_if_zero = true);
/// C'tor for tracker for which consumption counter is created as part of a profile.
/// The counter is created with name COUNTER_NAME.
@ -227,7 +228,8 @@ public:
/// consistent.)
if ((*tracker)->_consumption_metric == NULL) {
DCHECK_GE((*tracker)->_consumption->current_value(), 0)
<< std::endl << (*tracker)->LogUsage();
<< std::endl
<< (*tracker)->LogUsage(UNLIMITED_DEPTH);
}
}
@ -277,22 +279,13 @@ public:
_consumption->set(_consumption_metric->value());
}
bool limit_exceeded() const { return _limit >= 0 && _limit < consumption(); }
bool limit_exceeded() const{
return _limit >= 0 && _limit < consumption();
}
int64_t limit() const { return _limit; }
int64_t limit() const {
return _limit;
}
bool has_limit() const { return _limit >= 0; }
bool has_limit() const {
return _limit >= 0;
}
const std::string& label() const {
return _label;
}
const std::string& label() const { return _label; }
/// Returns the lowest limit for this tracker and its ancestors. Returns
/// -1 if there is no limit.
@ -312,19 +305,14 @@ public:
/// admission control). Otherwise the current consumption is used.
int64_t GetPoolMemReserved() const;
int64_t consumption() const {
return _consumption->current_value();;
}
int64_t consumption() const { return _consumption->current_value(); }
/// Note that if _consumption is based on _consumption_metric, this will the max value
/// we've recorded in consumption(), not necessarily the highest value
/// _consumption_metric has ever reached.
int64_t peak_consumption() const { return _consumption->value(); }
MemTracker* parent() const {
return _parent;
}
MemTracker* parent() const { return _parent; }
/// Signature for function that can be called to free some memory after limit is
/// reached. The function should try to free at least 'bytes_to_free' bytes of
@ -342,12 +330,24 @@ public:
/// "<prefix>.<metric name>".
void RegisterMetrics(MetricRegistry* metrics, const std::string& prefix);
/// Logs the usage of this tracker and all of its children (recursively).
/// Logs the usage of this tracker and optionally its children (recursively).
/// If 'logged_consumption' is non-NULL, sets the consumption value logged.
/// 'max_recursive_depth' specifies the maximum number of levels of children
/// to include in the dump. If it is zero, then no children are dumped.
/// Limiting the recursive depth reduces the cost of dumping, particularly
/// for the process MemTracker.
/// TODO: once all memory is accounted in ReservationTracker hierarchy, move
/// reporting there.
std::string LogUsage(
const std::string& prefix = "", int64_t* logged_consumption = nullptr) const;
std::string LogUsage(int max_recursive_depth, const std::string& prefix = "",
int64_t* logged_consumption = nullptr) const;
/// Dumping the process MemTracker is expensive. Limiting the recursive depth
/// to two levels limits the level of detail to a one-line summary for each query
/// MemTracker, avoiding all MemTrackers below that level. This provides a summary
/// of process usage with substantially lower cost than the full dump.
static const int PROCESS_MEMTRACKER_LIMITED_DEPTH = 2;
/// Unlimited dumping is useful for query memtrackers or error conditions that
/// are not performance sensitive
static const int UNLIMITED_DEPTH = INT_MAX;
/// Log the memory usage when memory limit is exceeded and return a status object with
/// details of the allocation which caused the limit to be exceeded.
@ -356,8 +356,6 @@ public:
Status MemLimitExceeded(RuntimeState* state, const std::string& details,
int64_t failed_allocation = 0);
static const int UNLIMITED_DEPTH = INT_MAX;
static const std::string COUNTER_NAME;
static void update_limits(int64_t bytes, std::vector<MemTracker*>* limits) {
@ -367,8 +365,7 @@ public:
}
static bool limit_exceeded(const std::vector<MemTracker*>& limits) {
for (std::vector<MemTracker*>::const_iterator i = limits.begin(); i != limits.end();
++i) {
for (std::vector<MemTracker*>::const_iterator i = limits.begin(); i != limits.end(); ++i) {
if ((*i)->limit_exceeded()) {
// TODO: remove logging
LOG(WARNING) << "exceeded limit: limit=" << (*i)->limit() << " consumption="
@ -391,10 +388,8 @@ public:
return msg.str();
}
bool is_consumption_metric_null() {
return _consumption_metric == nullptr;
}
bool is_consumption_metric_null() { return _consumption_metric == nullptr; }
private:
friend class PoolMemTrackerRegistry;
@ -413,9 +408,11 @@ private:
}
/// Log consumption of all the trackers provided. Returns the sum of consumption in
/// 'logged_consumption'.
static std::string LogUsage(const std::string& prefix,
const std::list<MemTracker*>& trackers, int64_t* logged_consumption);
/// 'logged_consumption'. 'max_recursive_depth' specifies the maximum number of levels
/// of children to include in the dump. If it is zero, then no children are dumped.
static std::string LogUsage(int max_recursive_depth, const std::string& prefix,
const std::list<MemTracker*>& trackers,
int64_t* logged_consumption);
/// Lock to protect GcMemory(). This prevents many GCs from occurring at once.
std::mutex _gc_lock;
@ -427,7 +424,7 @@ private:
// contains only weak ptrs. MemTrackers that are handed out via get_query_mem_tracker()
// are shared ptrs. When all the shared ptrs are no longer referenced, the MemTracker
// d'tor will be called at which point the weak ptr will be removed from the map.
typedef std::unordered_map<TUniqueId, std::weak_ptr<MemTracker> > RequestTrackersMap;
typedef std::unordered_map<TUniqueId, std::weak_ptr<MemTracker>> RequestTrackersMap;
static RequestTrackersMap _s_request_to_mem_trackers;
// Only valid for MemTrackers returned from get_query_mem_tracker()
@ -437,7 +434,7 @@ private:
/// Only valid for MemTrackers returned from GetRequestPoolMemTracker()
std::string _pool_name;
int64_t _limit; // in bytes
int64_t _limit; // in bytes
//int64_t _consumption; // in bytes
std::string _label;
@ -459,8 +456,8 @@ private:
/// are owned by the fragment's RuntimeProfile.
AtomicPtr<ReservationTrackerCounters> _reservation_counters;
std::vector<MemTracker*> _all_trackers; // this tracker plus all of its ancestors
std::vector<MemTracker*> _limit_trackers; // _all_trackers with valid limits
std::vector<MemTracker*> _all_trackers; // this tracker plus all of its ancestors
std::vector<MemTracker*> _limit_trackers; // _all_trackers with valid limits
// All the child trackers of this tracker. Used for error reporting only.
// i.e., Updating a parent tracker does not update the children.
@ -497,7 +494,7 @@ private:
// process tracker never gets deleted so it is safe to reference it in the dtor.
// The query tracker has lifetime shared by multiple plan fragments so it's hard
// to do cleanup another way.
bool _auto_unregister;
bool _auto_unregister = false;
};
/// Global registry for query and pool MemTrackers. Owned by ExecEnv.
@ -510,8 +507,7 @@ class PoolMemTrackerRegistry {
/// with the process tracker as its parent. There is no explicit per-pool byte_limit
/// set at any particular impalad, so newly created trackers will always have a limit
/// of -1.
MemTracker* GetRequestPoolMemTracker(
const std::string& pool_name, bool create_if_not_present);
MemTracker* GetRequestPoolMemTracker(const std::string& pool_name, bool create_if_not_present);
private:
/// All per-request pool MemTracker objects. It is assumed that request pools will live
@ -524,6 +520,6 @@ class PoolMemTrackerRegistry {
SpinLock _pool_to_mem_trackers_lock;
};
}
} // namespace doris
#endif