[enhancement](memory) Add tablet schema cache metrics (#14742)

be/src/olap/tablet_schema.cpp

@@ -441,18 +441,6 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
     }
 }
 
-uint32_t TabletColumn::mem_size() const {
-    auto size = sizeof(TabletColumn);
-    size += _col_name.size();
-    if (_has_default_value) {
-        size += _default_value.size();
-    }
-    for (auto& sub_column : _sub_columns) {
-        size += sub_column.mem_size();
-    }
-    return size;
-}
-
 void TabletColumn::add_sub_column(TabletColumn& sub_column) {
     _sub_columns.push_back(sub_column);
     sub_column._parent = this;
@@ -557,6 +545,7 @@ void TabletSchema::clear_columns() {
 }
 
 void TabletSchema::init_from_pb(const TabletSchemaPB& schema) {
+    SCOPED_MEM_COUNT(&_mem_size);
     _keys_type = schema.keys_type();
     _num_columns = 0;
     _num_key_columns = 0;
@@ -729,19 +718,6 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const {
     tablet_schema_pb->set_compression_type(_compression_type);
 }
 
-uint32_t TabletSchema::mem_size() const {
-    auto size = sizeof(TabletSchema);
-    for (auto& col : _cols) {
-        size += col.mem_size();
-    }
-
-    for (auto& pair : _field_name_to_index) {
-        size += pair.first.size();
-        size += sizeof(pair.second);
-    }
-    return size;
-}
-
 size_t TabletSchema::row_size() const {
     size_t size = 0;
     for (auto& column : _cols) {

be/src/olap/tablet_schema.h

@@ -45,7 +45,6 @@ public:
     void init_from_pb(const ColumnPB& column);
     void init_from_thrift(const TColumn& column);
     void to_schema_pb(ColumnPB* column) const;
-    uint32_t mem_size() const;
 
     int32_t unique_id() const { return _unique_id; }
     std::string name() const { return _col_name; }
@@ -150,7 +149,7 @@ public:
     void append_column(TabletColumn column, bool is_dropped_column = false);
     void copy_from(const TabletSchema& tablet_schema);
     std::string to_key() const;
-    uint32_t mem_size() const;
+    int64_t mem_size() const { return _mem_size; };
 
     size_t row_size() const;
     int32_t field_index(const std::string& field_name) const;
@@ -243,6 +242,7 @@ private:
     int32_t _sequence_col_idx = -1;
     int32_t _schema_version = -1;
     bool _disable_auto_compaction = false;
+    int64_t _mem_size = 0;
 };
 
 bool operator==(const TabletSchema& a, const TabletSchema& b);

be/src/olap/tablet_schema_cache.h

@@ -24,6 +24,7 @@
 #include <unordered_map>
 
 #include "olap/tablet_schema.h"
+#include "util/doris_metrics.h"
 
 namespace doris {
 
@@ -49,6 +50,9 @@ public:
             pb.ParseFromString(key);
             tablet_schema_ptr->init_from_pb(pb);
             _cache[key] = tablet_schema_ptr;
+            DorisMetrics::instance()->tablet_schema_cache_count->increment(1);
+            DorisMetrics::instance()->tablet_schema_cache_memory_bytes->increment(
+                    tablet_schema_ptr->mem_size());
             return tablet_schema_ptr;
         }
         return iter->second;
@@ -63,8 +67,12 @@ private:
         for (;;) {
             std::this_thread::sleep_for(std::chrono::seconds(tablet_schema_cache_recycle_interval));
             std::lock_guard guard(_mtx);
+            LOG(INFO) << "Tablet Schema Cache Capacity " << _cache.size();
             for (auto iter = _cache.begin(), last = _cache.end(); iter != last;) {
                 if (iter->second.unique()) {
+                    DorisMetrics::instance()->tablet_schema_cache_memory_bytes->increment(
+                            -iter->second->mem_size());
+                    DorisMetrics::instance()->tablet_schema_cache_count->increment(-1);
                     iter = _cache.erase(iter);
                 } else {
                     ++iter;

be/src/runtime/memory/mem_tracker_limiter.cpp

@@ -68,6 +68,7 @@ MemTrackerLimiter::MemTrackerLimiter(Type type, const std::string& label, int64_
 }
 
 MemTrackerLimiter::~MemTrackerLimiter() {
+    if (_type == Type::GLOBAL) return;
     consume(_untracked_mem);
     // mem hook record tracker cannot guarantee that the final consumption is 0,
     // nor can it guarantee that the memory alloc and free are recorded in a one-to-one correspondence.

be/src/util/doris_metrics.cpp

@@ -161,6 +161,9 @@ DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_memory_total_byte, MetricUni
 DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_sql_total_count, MetricUnit::NOUNIT);
 DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_cache_partition_total_count, MetricUnit::NOUNIT);
 
+DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(tablet_schema_cache_count, MetricUnit::NOUNIT);
+DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(tablet_schema_cache_memory_bytes, MetricUnit::BYTES);
+
 DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(upload_total_byte, MetricUnit::BYTES);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(upload_rowset_count, MetricUnit::ROWSETS);
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(upload_fail_count, MetricUnit::ROWSETS);
@@ -290,6 +293,9 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) {
     INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, query_cache_sql_total_count);
     INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, query_cache_partition_total_count);
 
+    INT_COUNTER_METRIC_REGISTER(_server_metric_entity, tablet_schema_cache_count);
+    INT_UGAUGE_METRIC_REGISTER(_server_metric_entity, tablet_schema_cache_memory_bytes);
+
     INT_COUNTER_METRIC_REGISTER(_server_metric_entity, local_file_reader_total);
     INT_COUNTER_METRIC_REGISTER(_server_metric_entity, s3_file_reader_total);
     INT_COUNTER_METRIC_REGISTER(_server_metric_entity, local_file_writer_total);

be/src/util/doris_metrics.h

@@ -199,6 +199,9 @@ public:
     UIntGauge* query_cache_sql_total_count;
     UIntGauge* query_cache_partition_total_count;
 
+    IntCounter* tablet_schema_cache_count;
+    UIntGauge* tablet_schema_cache_memory_bytes;
+
     UIntGauge* scanner_thread_pool_queue_size;
     UIntGauge* add_batch_task_queue_size;
     UIntGauge* send_batch_thread_pool_thread_num;