[Metric] Add metrics of tablet version num distribution (#5665)

Add metrics (P50, P75, P90, P95, P99, etc.) to show the distribution of tablets version count.

```
# TYPE doris_be_tablet_version_num_distribution histogram
doris_be_tablet_version_num_distribution{quantile="0.50"} 9.21429
doris_be_tablet_version_num_distribution{quantile="0.75"} 11.7949
doris_be_tablet_version_num_distribution{quantile="0.90"} 13
doris_be_tablet_version_num_distribution{quantile="0.95"} 13
doris_be_tablet_version_num_distribution{quantile="0.99"} 13
doris_be_tablet_version_num_distribution_sum 950
doris_be_tablet_version_num_distribution_count 100
```

be/src/olap/tablet_manager.cpp

@@ -47,6 +47,7 @@
 #include "olap/utils.h"
 #include "util/doris_metrics.h"
 #include "util/file_utils.h"
+#include "util/histogram.h"
 #include "util/path_util.h"
 #include "util/pretty_printer.h"
 #include "util/scoped_cleanup.h"
@@ -964,7 +965,7 @@ OLAPStatus TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTab
     LOG(INFO) << "find expired transactions for " << expire_txn_map.size() << " tablets";
 
     DorisMetrics::instance()->report_all_tablets_requests_total->increment(1);
-
+    HistogramStat tablet_version_num_hist;
     for (const auto& tablets_shard : _tablets_shards) {
         ReadLock rlock(tablets_shard.lock.get());
         for (const auto& item : tablets_shard.tablet_map) {
@@ -986,6 +987,9 @@ OLAPStatus TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTab
                     expire_txn_map.erase(find);
                 }
                 t_tablet.tablet_infos.push_back(tablet_info);
+                if (tablet_ptr->tablet_id() == tablet_id) {
+                    tablet_version_num_hist.add(tablet_ptr->version_count());
+                }
             }
 
             if (!t_tablet.tablet_infos.empty()) {
@@ -993,6 +997,7 @@ OLAPStatus TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTab
             }
         }
     }
+    DorisMetrics::instance()->tablet_version_num_distribution->set_histogram(tablet_version_num_hist);
     LOG(INFO) << "success to build all report tablets info. tablet_count=" << tablets_info->size();
     return OLAP_SUCCESS;
 }

be/src/util/doris_metrics.cpp

@@ -143,6 +143,8 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_base_max_compaction_score, MetricUnit:
 DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_used_permits, MetricUnit::NOUNIT);
 DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_waitting_permits, MetricUnit::NOUNIT);
 
+DEFINE_HISTOGRAM_METRIC_PROTOTYPE_2ARG(tablet_version_num_distribution, MetricUnit::NOUNIT);
+
 DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(push_request_write_bytes_per_second, MetricUnit::BYTES);
 DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(query_scan_bytes_per_second, MetricUnit::BYTES);
 DEFINE_GAUGE_CORE_METRIC_PROTOTYPE_2ARG(max_disk_io_util_percent, MetricUnit::PERCENT);
@@ -250,6 +252,8 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) {
     INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_used_permits);
     INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_waitting_permits);
 
+    HISTOGRAM_METRIC_REGISTER(_server_metric_entity, tablet_version_num_distribution);
+
     INT_GAUGE_METRIC_REGISTER(_server_metric_entity, push_request_write_bytes_per_second);
     INT_GAUGE_METRIC_REGISTER(_server_metric_entity, query_scan_bytes_per_second);
     INT_GAUGE_METRIC_REGISTER(_server_metric_entity, max_disk_io_util_percent);

be/src/util/doris_metrics.h

@@ -140,6 +140,8 @@ public:
     // permits required by the compaction task which is waitting for permits
     IntGauge* compaction_waitting_permits;
 
+    HistogramMetric* tablet_version_num_distribution;
+
     // The following metrics will be calculated
     // by metric calculator
     IntGauge* push_request_write_bytes_per_second;

be/src/util/metrics.cpp

@@ -138,6 +138,12 @@ void HistogramMetric::merge(const HistogramMetric& other) {
     _stats.merge(other._stats);
 }
 
+void HistogramMetric::set_histogram(const HistogramStat& stats) {
+    std::lock_guard<SpinLock> l(_lock);
+    _stats.clear();
+    _stats.merge(stats);
+}
+
 double HistogramMetric::median() const {
     return _stats.median();
 }

be/src/util/metrics.h

@@ -177,6 +177,7 @@ public:
     bool is_empty() const;
     void add(const uint64_t& value);
     void merge(const HistogramMetric& other);
+    void set_histogram(const HistogramStat& stats);
 
     uint64_t min() const { return _stats.min(); }
     uint64_t max() const { return _stats.max(); }
@@ -286,6 +287,9 @@ public:
 #define DEFINE_GAUGE_METRIC_PROTOTYPE_5ARG(name, unit, desc, group, labels) \
     DEFINE_METRIC_PROTOTYPE(name, MetricType::GAUGE, unit, desc, #group, labels, false)
 
+#define DEFINE_HISTOGRAM_METRIC_PROTOTYPE_2ARG(name, unit) \
+    DEFINE_METRIC_PROTOTYPE(name, MetricType::HISTOGRAM, unit, "", "", Labels(), false)
+
 #define INT_COUNTER_METRIC_REGISTER(entity, metric) \
     metric = (IntCounter*)(entity->register_metric<IntCounter>(&METRIC_##metric))
 
@@ -301,6 +305,9 @@ public:
 #define INT_ATOMIC_COUNTER_METRIC_REGISTER(entity, metric) \
     metric = (IntAtomicCounter*)(entity->register_metric<IntAtomicCounter>(&METRIC_##metric))
 
+#define HISTOGRAM_METRIC_REGISTER(entity, metric) \
+    metric = (HistogramMetric*)(entity->register_metric<HistogramMetric>(&METRIC_##metric))
+
 #define METRIC_DEREGISTER(entity, metric) entity->deregister_metric(&METRIC_##metric)
 
 // For 'metrics' in MetricEntity.