From 79544d39cb5f9614bf8b68d079a3ef170977e87e Mon Sep 17 00:00:00 2001
From: weizuo93 <68884553+weizuo93@users.noreply.github.com>
Date: Thu, 8 Apr 2021 09:10:40 +0800
Subject: [PATCH] [Metrics][LOG] Update metrics of 'max_compaction_score' and
 log for compaction (#5592)

* optimize compaction metrics and log

Co-authored-by: weizuo <weizuo@xiaomi.com>
---
 be/src/olap/compaction.cpp     |  5 +++--
 be/src/olap/olap_server.cpp    | 12 +++++++++++-
 be/src/olap/tablet_manager.cpp | 11 ++---------
 be/src/olap/tablet_manager.h   |  3 ++-
 4 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 35dcbb8e91..a9255dd3ed 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -133,8 +133,9 @@ OLAPStatus Compaction::do_compaction_impl(int64_t permits) {
 
     LOG(INFO) << "succeed to do " << compaction_name() << ". tablet=" << _tablet->full_name()
               << ", output_version=" << _output_version.first << "-" << _output_version.second
-              << ", segments=" << segments_num << ". elapsed time=" << watch.get_elapse_second()
-              << "s.";
+              << ", current_max_version=" << _tablet->rowset_with_max_version()->end_version()
+              << ", disk=" << _tablet->data_dir()->path() << ", segments=" << segments_num
+              << ". elapsed time=" << watch.get_elapse_second() << "s.";
 
     return OLAP_SUCCESS;
 }
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 19d2865075..97bbf8483a 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -386,6 +386,7 @@ void StorageEngine::_compaction_tasks_producer_callback() {
 std::vector<TabletSharedPtr> StorageEngine::_compaction_tasks_generator(
         CompactionType compaction_type, std::vector<DataDir*> data_dirs) {
     std::vector<TabletSharedPtr> tablets_compaction;
+    uint32_t max_compaction_score = 0;
     std::random_shuffle(data_dirs.begin(), data_dirs.end());
     for (auto data_dir : data_dirs) {
         std::unique_lock<std::mutex> lock(_tablet_submitted_compaction_mutex);
@@ -393,13 +394,22 @@ std::vector<TabletSharedPtr> StorageEngine::_compaction_tasks_generator(
             continue;
         }
         if (!data_dir->reach_capacity_limit(0)) {
+            uint32_t disk_max_score = 0;
             TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction(
-                    compaction_type, data_dir, _tablet_submitted_compaction[data_dir]);
+                    compaction_type, data_dir, _tablet_submitted_compaction[data_dir], &disk_max_score);
             if (tablet != nullptr) {
                 tablets_compaction.emplace_back(tablet);
+                max_compaction_score = std::max(max_compaction_score, disk_max_score);
             }
         }
     }
+    if (!tablets_compaction.empty()) {
+        if (compaction_type == CompactionType::BASE_COMPACTION) {
+            DorisMetrics::instance()->tablet_base_max_compaction_score->set_value(max_compaction_score);
+        } else {
+            DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value(max_compaction_score);
+        }
+    }
     return tablets_compaction;
 }
 
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index df81c8bcc1..92a4724b5f 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -681,7 +681,7 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) {
 
 TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
         CompactionType compaction_type, DataDir* data_dir,
-        std::vector<TTabletId>& tablet_submitted_compaction) {
+        std::vector<TTabletId>& tablet_submitted_compaction, uint32_t* score) {
     int64_t now_ms = UnixMillis();
     const string& compaction_type_str =
             compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative";
@@ -776,14 +776,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
                 << ", compaction_score=" << compaction_score
                 << ", tablet_scan_frequency=" << tablet_scan_frequency
                 << ", highest_score=" << highest_score;
-        // TODO(lingbin): Remove 'max' from metric name, it would be misunderstood as the
-        // biggest in history(like peak), but it is really just the value at current moment.
-        if (compaction_type == CompactionType::BASE_COMPACTION) {
-            DorisMetrics::instance()->tablet_base_max_compaction_score->set_value(compaction_score);
-        } else {
-            DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value(
-                    compaction_score);
-        }
+        *score = compaction_score;
     }
     return best_tablet;
 }
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index bbb417d1a2..75eebc594c 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -71,7 +71,8 @@ public:
 
     TabletSharedPtr find_best_tablet_to_compaction(CompactionType compaction_type,
                                                    DataDir* data_dir,
-                                                   vector<TTabletId>& tablet_submitted_compaction);
+                                                   vector<TTabletId>& tablet_submitted_compaction,
+                                                   uint32_t* score);
 
     TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash,
                                bool include_deleted = false, std::string* err = nullptr);