database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

Add a configuration to modify the reverse time of load error log (#1433)

Browse Source 
Currently, the load error log on BE will be cleaned along with the
intermediate data of load, configured by 'load_data_reserve_hours'.
Sometimes user want to reserve the error log for longer time.
This commit is contained in:
Mingyu Chen
2019-07-09 10:36:13 +08:00
committed by ZHAO Chun
parent bde362c3cd
commit ded60e59f9
3 changed files with 14 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
be/src/common/config.h
View File

@ -257,8 +257,10 @@ namespace config {
// Period to update rate counters and sampling counters in ms.
CONF_Int32(periodic_counter_update_period_ms, "500");
// Used for mini Load
// Used for mini Load. mini load data file will be removed after this time.
CONF_Int64(load_data_reserve_hours, "4");
// log error log will be removed after this time
CONF_Int64(load_error_log_reserve_hours, "48");
CONF_Int64(mini_load_max_mb, "2048");
CONF_Int32(number_tablet_writer_threads, "16");

19
be/src/runtime/load_path_mgr.cpp
View File

@ -59,9 +59,8 @@ Status LoadPathMgr::init() {
void* LoadPathMgr::cleaner(void* param) {
// TODO(zc): add this thread to cgroup for control resource it use
LoadPathMgr* mgr = (LoadPathMgr*)param;
int64_t sleep_time = std::max(3600L / 4, config::load_data_reserve_hours * 3600 / 4);
while (true) {
sleep(sleep_time);
sleep(3600); // clean every one hour
mgr->clean();
}
return nullptr;
@ -98,7 +97,7 @@ Status LoadPathMgr::allocate_dir(
return status;
}
bool LoadPathMgr::is_too_old(time_t cur_time, const std::string& label_dir) {
bool LoadPathMgr::is_too_old(time_t cur_time, const std::string& label_dir, int64_t reserve_hours) {
struct stat dir_stat;
if (stat(label_dir.c_str(), &dir_stat)) {
char buf[64];
@ -108,7 +107,7 @@ bool LoadPathMgr::is_too_old(time_t cur_time, const std::string& label_dir) {
return false;
}
if ((cur_time - dir_stat.st_mtime) < _reserved_hours * 3600) {
if ((cur_time - dir_stat.st_mtime) < reserve_hours * 3600) {
return false;
}
@ -155,8 +154,8 @@ std::string LoadPathMgr::get_load_error_absolute_path(const std::string& file_pa
return path;
}
void LoadPathMgr::process_path(time_t now, const std::string& path) {
if (!is_too_old(now, path)) {
void LoadPathMgr::process_path(time_t now, const std::string& path, int64_t reserve_hours) {
if (!is_too_old(now, path, reserve_hours)) {
return;
}
LOG(INFO) << "Going to remove path. path=" << path;
@ -200,11 +199,11 @@ void LoadPathMgr::clean_one_path(const std::string& path) {
}
for (auto& label : labels) {
std::string label_dir = sub_path + "/" + label;
process_path(now, label_dir);
process_path(now, label_dir, config::load_data_reserve_hours);
}
} else {
// process label dir
process_path(now, sub_path);
process_path(now, sub_path, config::load_data_reserve_hours);
}
}
}
@ -240,11 +239,11 @@ void LoadPathMgr::clean_error_log() {
}
for (auto& error_log : error_log_files) {
std::string error_log_path = sub_path + "/" + error_log;
process_path(now, error_log_path);
process_path(now, error_log_path, config::load_error_log_reserve_hours);
}
} else {
// process error log file
process_path(now, sub_path);
process_path(now, sub_path, config::load_error_log_reserve_hours);
}
}
}

4
be/src/runtime/load_path_mgr.h
View File

@ -55,11 +55,11 @@ public:
}
private:
bool is_too_old(time_t cur_time, const std::string& label_dir);
bool is_too_old(time_t cur_time, const std::string& label_dir, int64_t reserve_hours);
void clean_one_path(const std::string& path);
void clean_error_log();
void clean();
void process_path(time_t now, const std::string& path);
void process_path(time_t now, const std::string& path, int64_t reserve_hours);
static void* cleaner(void* param);