database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Compaction] Fix a bug that CumulativeCompaction compares time of different precision (#2693)

Browse Source 
time(NULL) returns second-resolution timestamp, however all compaction related time in Tablet are in millis-resolution. Therefore should use UnixMillis() instead.
This commit is contained in:
Dayue Gao
2020-01-07 21:31:36 +08:00
committed by ZHAO Chun
parent 844ccaafc9
commit 4e2f01a9fa
4 changed files with 26 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
be/src/olap/cumulative_compaction.cpp
View File

@ -17,6 +17,7 @@
#include "olap/cumulative_compaction.h"
#include "util/doris_metrics.h"
#include "util/time.h"
namespace doris {
@ -129,13 +130,10 @@ OLAPStatus CumulativeCompaction::pick_rowsets_to_compact() {
// the cumulative point after waiting for a long time, to ensure that the base compaction can continue.
// check both last success time of base and cumulative compaction
int64_t last_cumu_compaction_success_time = _tablet->last_cumu_compaction_success_time();
int64_t last_base_compaction_success_time = _tablet->last_base_compaction_success_time();
int64_t interval_threshold = config::base_compaction_interval_seconds_since_last_operation;
int64_t now = time(NULL);
int64_t cumu_interval = now - last_cumu_compaction_success_time;
int64_t base_interval = now - last_base_compaction_success_time;
int64_t interval_threshold = config::base_compaction_interval_seconds_since_last_operation * 1000;
int64_t now = UnixMillis();
int64_t cumu_interval = now - _tablet->last_cumu_compaction_success_time();
int64_t base_interval = now - _tablet->last_base_compaction_success_time();
if (cumu_interval > interval_threshold && base_interval > interval_threshold) {
// before increasing cumulative point, we should make sure all rowsets are non-overlapping.
// if at least one rowset is overlapping, we should compact them first.

16
be/src/olap/tablet.cpp
View File

@ -67,10 +67,10 @@ Tablet::Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir)
_schema(tablet_meta->tablet_schema()),
_data_dir(data_dir),
_is_bad(false),
_last_cumu_compaction_failure_time(0),
_last_base_compaction_failure_time(0),
_last_cumu_compaction_success_time(0),
_last_base_compaction_success_time(0) {
_last_cumu_compaction_failure_millis(0),
_last_base_compaction_failure_millis(0),
_last_cumu_compaction_success_millis(0),
_last_base_compaction_success_millis(0) {
_tablet_path.append(_data_dir->path());
_tablet_path.append(DATA_PREFIX);
@ -1045,19 +1045,19 @@ OLAPStatus Tablet::get_compaction_status(std::string* json_result) {
root.AddMember("cumulative point", _cumulative_point.load(), root.GetAllocator());
rapidjson::Value cumu_value;
std::string format_str = ToStringFromUnixMillis(_last_cumu_compaction_failure_time.load());
std::string format_str = ToStringFromUnixMillis(_last_cumu_compaction_failure_millis.load());
cumu_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
root.AddMember("last cumulative failure time", cumu_value, root.GetAllocator());
rapidjson::Value base_value;
format_str = ToStringFromUnixMillis(_last_base_compaction_failure_time.load());
format_str = ToStringFromUnixMillis(_last_base_compaction_failure_millis.load());
base_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
root.AddMember("last base failure time", base_value, root.GetAllocator());
rapidjson::Value cumu_success_value;
format_str = ToStringFromUnixMillis(_last_cumu_compaction_success_time.load());
format_str = ToStringFromUnixMillis(_last_cumu_compaction_success_millis.load());
cumu_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
root.AddMember("last cumulative success time", cumu_success_value, root.GetAllocator());
rapidjson::Value base_success_value;
format_str = ToStringFromUnixMillis(_last_base_compaction_success_time.load());
format_str = ToStringFromUnixMillis(_last_base_compaction_success_millis.load());
base_success_value.SetString(format_str.c_str(), format_str.length(), root.GetAllocator());
root.AddMember("last base success time", base_success_value, root.GetAllocator());

24
be/src/olap/tablet.h
View File

@ -210,17 +210,17 @@ public:
void set_io_error();
void set_bad(bool is_bad) { _is_bad = is_bad; }
int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_time; }
void set_last_cumu_compaction_failure_time(int64_t time) { _last_cumu_compaction_failure_time = time; }
int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; }
void set_last_cumu_compaction_failure_time(int64_t millis) { _last_cumu_compaction_failure_millis = millis; }
int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_time; }
void set_last_base_compaction_failure_time(int64_t time) { _last_base_compaction_failure_time = time; }
int64_t last_base_compaction_failure_time() { return _last_base_compaction_failure_millis; }
void set_last_base_compaction_failure_time(int64_t millis) { _last_base_compaction_failure_millis = millis; }
int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_time; }
void set_last_cumu_compaction_success_time(int64_t time) { _last_cumu_compaction_success_time = time; }
int64_t last_cumu_compaction_success_time() { return _last_cumu_compaction_success_millis; }
void set_last_cumu_compaction_success_time(int64_t millis) { _last_cumu_compaction_success_millis = millis; }
int64_t last_base_compaction_success_time() { return _last_base_compaction_success_time; }
void set_last_base_compaction_success_time(int64_t time) { _last_base_compaction_success_time = time; }
int64_t last_base_compaction_success_time() { return _last_base_compaction_success_millis; }
void set_last_base_compaction_success_time(int64_t millis) { _last_base_compaction_success_millis = millis; }
void delete_all_files();
@ -284,10 +284,10 @@ private:
std::unordered_map<Version, RowsetSharedPtr, HashOfVersion> _inc_rs_version_map;
std::atomic<bool> _is_bad; // if this tablet is broken, set to true. default is false
std::atomic<int64_t> _last_cumu_compaction_failure_time; // timestamp of last cumulative compaction failure
std::atomic<int64_t> _last_base_compaction_failure_time; // timestamp of last base compaction failure
std::atomic<int64_t> _last_cumu_compaction_success_time; // timestamp of last cumu compaction success
std::atomic<int64_t> _last_base_compaction_success_time; // timestamp of last base compaction success
std::atomic<int64_t> _last_cumu_compaction_failure_millis; // timestamp of last cumu compaction failure
std::atomic<int64_t> _last_base_compaction_failure_millis; // timestamp of last base compaction failure
std::atomic<int64_t> _last_cumu_compaction_success_millis; // timestamp of last cumu compaction success
std::atomic<int64_t> _last_base_compaction_success_millis; // timestamp of last base compaction success
std::atomic<int64_t> _cumulative_point;
std::atomic<int32_t> _newly_created_rowset_num;

2
be/src/olap/tablet_manager.cpp
View File

@ -679,10 +679,10 @@ void TabletManager::get_tablet_stat(TTabletStatResult& result) {
TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
CompactionType compaction_type, DataDir* data_dir) {
int64_t now = UnixMillis();
ReadLock tablet_map_rdlock(&_tablet_map_lock);
uint32_t highest_score = 0;
TabletSharedPtr best_tablet;
int64_t now = UnixMillis();
for (tablet_map_t::value_type& table_ins : _tablet_map){
for (TabletSharedPtr& table_ptr : table_ins.second.table_arr) {
AlterTabletTaskSharedPtr cur_alter_task = table_ptr->alter_task();