database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Metrics] Support tablet level metrics (#4428)

Browse Source 
Sometimes we want to detect the hotspot of a cluster, for example, hot scanned tablet, hot wrote tablet,
but we have no insight about tablets in the cluster.
This patch introduce tablet level metrics to help to achieve this object, now support 4 metrics on tablets: `query_scan_bytes `, `query_scan_rows `, `flush_bytes `, `flush_count `. 
However, one BE may holds hundreds of thousands of tablets, so I add a parameter for the metrics HTTP request,
and not return tablet level metrics by default.
This commit is contained in:
Yingchun Lai
2020-09-02 10:39:41 +08:00
committed by GitHub
parent f3a9f3f87c
commit 498b06fbe2
51 changed files with 729 additions and 615 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
be/src/runtime/memory/chunk_allocator.cpp
View File

@ -41,12 +41,12 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_system_free_count, MetricUnit::N
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_system_alloc_cost_ns, MetricUnit::NANOSECONDS);
DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_system_free_cost_ns, MetricUnit::NANOSECONDS);
static IntCounter chunk_pool_local_core_alloc_count;
static IntCounter chunk_pool_other_core_alloc_count;
static IntCounter chunk_pool_system_alloc_count;
static IntCounter chunk_pool_system_free_count;
static IntCounter chunk_pool_system_alloc_cost_ns;
static IntCounter chunk_pool_system_free_cost_ns;
static IntCounter* chunk_pool_local_core_alloc_count;
static IntCounter* chunk_pool_other_core_alloc_count;
static IntCounter* chunk_pool_system_alloc_count;
static IntCounter* chunk_pool_system_free_count;
static IntCounter* chunk_pool_system_alloc_cost_ns;
static IntCounter* chunk_pool_system_free_cost_ns;
#ifdef BE_TEST
static std::mutex s_mutex;
@ -118,13 +118,13 @@ ChunkAllocator::ChunkAllocator(size_t reserve_limit)
_arenas[i].reset(new ChunkArena());
}
_chunk_allocator_metric_entity = DorisMetrics::instance()->metric_registry()->register_entity("chunk_allocator", {});
METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_local_core_alloc_count);
METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_other_core_alloc_count);
METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_alloc_count);
METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_free_count);
METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_alloc_cost_ns);
METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_free_cost_ns);
_chunk_allocator_metric_entity = DorisMetrics::instance()->metric_registry()->register_entity("chunk_allocator");
INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_local_core_alloc_count);
INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_other_core_alloc_count);
INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_alloc_count);
INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_free_count);
INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_alloc_cost_ns);
INT_COUNTER_METRIC_REGISTER(_chunk_allocator_metric_entity, chunk_pool_system_free_cost_ns);
}
bool ChunkAllocator::allocate(size_t size, Chunk* chunk) {
@ -135,7 +135,7 @@ bool ChunkAllocator::allocate(size_t size, Chunk* chunk) {
if (_arenas[core_id]->pop_free_chunk(size, &chunk->data)) {
_reserved_bytes.fetch_sub(size);
chunk_pool_local_core_alloc_count.increment(1);
chunk_pool_local_core_alloc_count->increment(1);
return true;
}
if (_reserved_bytes > size) {
@ -144,7 +144,7 @@ bool ChunkAllocator::allocate(size_t size, Chunk* chunk) {
for (int i = 1; i < _arenas.size(); ++i, ++core_id) {
if (_arenas[core_id % _arenas.size()]->pop_free_chunk(size, &chunk->data)) {
_reserved_bytes.fetch_sub(size);
chunk_pool_other_core_alloc_count.increment(1);
chunk_pool_other_core_alloc_count->increment(1);
// reset chunk's core_id to other
chunk->core_id = core_id % _arenas.size();
return true;
@ -158,8 +158,8 @@ bool ChunkAllocator::allocate(size_t size, Chunk* chunk) {
// allocate from system allocator
chunk->data = SystemAllocator::allocate(size);
}
chunk_pool_system_alloc_count.increment(1);
chunk_pool_system_alloc_cost_ns.increment(cost_ns);
chunk_pool_system_alloc_count->increment(1);
chunk_pool_system_alloc_cost_ns->increment(cost_ns);
if (chunk->data == nullptr) {
return false;
}
@ -177,8 +177,8 @@ void ChunkAllocator::free(const Chunk& chunk) {
SCOPED_RAW_TIMER(&cost_ns);
SystemAllocator::free(chunk.data, chunk.size);
}
chunk_pool_system_free_count.increment(1);
chunk_pool_system_free_cost_ns.increment(cost_ns);
chunk_pool_system_free_count->increment(1);
chunk_pool_system_free_cost_ns->increment(cost_ns);
return;
}