From ded60e59f98e3713033170ffb34841b8726f7a64 Mon Sep 17 00:00:00 2001
From: Mingyu Chen <morningman@163.com>
Date: Tue, 9 Jul 2019 10:36:13 +0800
Subject: [PATCH] Add a configuration to modify the reverse time of load error
 log (#1433)

Currently, the load error log on BE will be cleaned along with the
intermediate data of load, configured by 'load_data_reserve_hours'.
Sometimes user want to reserve the error log for longer time.
---
 be/src/common/config.h           |  4 +++-
 be/src/runtime/load_path_mgr.cpp | 19 +++++++++----------
 be/src/runtime/load_path_mgr.h   |  4 ++--
 3 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/be/src/common/config.h b/be/src/common/config.h
index 2ab793c578..7283d4ac04 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -257,8 +257,10 @@ namespace config {
     // Period to update rate counters and sampling counters in ms.
     CONF_Int32(periodic_counter_update_period_ms, "500");
 
-    // Used for mini Load
+    // Used for mini Load. mini load data file will be removed after this time.
     CONF_Int64(load_data_reserve_hours, "4");
+    // log error log will be removed after this time
+    CONF_Int64(load_error_log_reserve_hours, "48");
     CONF_Int64(mini_load_max_mb, "2048");
     CONF_Int32(number_tablet_writer_threads, "16");
 
diff --git a/be/src/runtime/load_path_mgr.cpp b/be/src/runtime/load_path_mgr.cpp
index 27a9c20daa..d928819b11 100644
--- a/be/src/runtime/load_path_mgr.cpp
+++ b/be/src/runtime/load_path_mgr.cpp
@@ -59,9 +59,8 @@ Status LoadPathMgr::init() {
 void* LoadPathMgr::cleaner(void* param) {
     // TODO(zc): add this thread to cgroup for control resource it use
     LoadPathMgr* mgr = (LoadPathMgr*)param;
-    int64_t sleep_time = std::max(3600L / 4, config::load_data_reserve_hours * 3600 / 4);
     while (true) {
-        sleep(sleep_time);
+        sleep(3600); // clean every one hour
         mgr->clean();
     }
     return nullptr;
@@ -98,7 +97,7 @@ Status LoadPathMgr::allocate_dir(
     return status;
 }
 
-bool LoadPathMgr::is_too_old(time_t cur_time, const std::string& label_dir) {
+bool LoadPathMgr::is_too_old(time_t cur_time, const std::string& label_dir, int64_t reserve_hours) {
     struct stat dir_stat;
     if (stat(label_dir.c_str(), &dir_stat)) {
         char buf[64];
@@ -108,7 +107,7 @@ bool LoadPathMgr::is_too_old(time_t cur_time, const std::string& label_dir) {
         return false;
     }
 
-    if ((cur_time - dir_stat.st_mtime) < _reserved_hours * 3600) {
+    if ((cur_time - dir_stat.st_mtime) < reserve_hours * 3600) {
         return false;
     }
 
@@ -155,8 +154,8 @@ std::string LoadPathMgr::get_load_error_absolute_path(const std::string& file_pa
     return path;
 }
 
-void LoadPathMgr::process_path(time_t now, const std::string& path) {
-    if (!is_too_old(now, path)) {
+void LoadPathMgr::process_path(time_t now, const std::string& path, int64_t reserve_hours) {
+    if (!is_too_old(now, path, reserve_hours)) {
         return;
     }
     LOG(INFO) << "Going to remove path. path=" << path;
@@ -200,11 +199,11 @@ void LoadPathMgr::clean_one_path(const std::string& path) {
                 }
                 for (auto& label : labels) {
                     std::string label_dir = sub_path + "/" + label;
-                    process_path(now, label_dir);
+                    process_path(now, label_dir, config::load_data_reserve_hours);
                 }
             } else {
                 // process label dir
-                process_path(now, sub_path);
+                process_path(now, sub_path, config::load_data_reserve_hours);
             }
         }
     }
@@ -240,11 +239,11 @@ void LoadPathMgr::clean_error_log() {
             }
             for (auto& error_log : error_log_files) {
                 std::string error_log_path = sub_path + "/" + error_log;
-                process_path(now, error_log_path);
+                process_path(now, error_log_path, config::load_error_log_reserve_hours);
             }
         } else {
             // process error log file
-            process_path(now, sub_path);
+            process_path(now, sub_path, config::load_error_log_reserve_hours);
         }
     }
 }
diff --git a/be/src/runtime/load_path_mgr.h b/be/src/runtime/load_path_mgr.h
index d1affc76d2..d03e71cd83 100644
--- a/be/src/runtime/load_path_mgr.h
+++ b/be/src/runtime/load_path_mgr.h
@@ -55,11 +55,11 @@ public:
     }
 
 private:
-    bool is_too_old(time_t cur_time, const std::string& label_dir);
+    bool is_too_old(time_t cur_time, const std::string& label_dir, int64_t reserve_hours);
     void clean_one_path(const std::string& path);
     void clean_error_log();
     void clean();
-    void process_path(time_t now, const std::string& path);
+    void process_path(time_t now, const std::string& path, int64_t reserve_hours);
 
     static void* cleaner(void* param);