database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Metrics][LOG] Update metrics of 'max_compaction_score' and log for compaction (#5592)

Browse Source 
* optimize compaction metrics and log

Co-authored-by: weizuo <weizuo@xiaomi.com>
This commit is contained in:
weizuo93
2021-04-08 09:10:40 +08:00
committed by GitHub
parent 621c89f7b9
commit 79544d39cb
4 changed files with 18 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
be/src/olap/compaction.cpp
View File

@ -133,8 +133,9 @@ OLAPStatus Compaction::do_compaction_impl(int64_t permits) {
LOG(INFO) << "succeed to do " << compaction_name() << ". tablet=" << _tablet->full_name()
<< ", output_version=" << _output_version.first << "-" << _output_version.second
<< ", segments=" << segments_num << ". elapsed time=" << watch.get_elapse_second()
<< "s.";
<< ", current_max_version=" << _tablet->rowset_with_max_version()->end_version()
<< ", disk=" << _tablet->data_dir()->path() << ", segments=" << segments_num
<< ". elapsed time=" << watch.get_elapse_second() << "s.";
return OLAP_SUCCESS;
}

12
be/src/olap/olap_server.cpp
View File

@ -386,6 +386,7 @@ void StorageEngine::_compaction_tasks_producer_callback() {
std::vector<TabletSharedPtr> StorageEngine::_compaction_tasks_generator(
CompactionType compaction_type, std::vector<DataDir*> data_dirs) {
std::vector<TabletSharedPtr> tablets_compaction;
uint32_t max_compaction_score = 0;
std::random_shuffle(data_dirs.begin(), data_dirs.end());
for (auto data_dir : data_dirs) {
std::unique_lock<std::mutex> lock(_tablet_submitted_compaction_mutex);
@ -393,13 +394,22 @@ std::vector<TabletSharedPtr> StorageEngine::_compaction_tasks_generator(
continue;
}
if (!data_dir->reach_capacity_limit(0)) {
uint32_t disk_max_score = 0;
TabletSharedPtr tablet = _tablet_manager->find_best_tablet_to_compaction(
compaction_type, data_dir, _tablet_submitted_compaction[data_dir]);
compaction_type, data_dir, _tablet_submitted_compaction[data_dir], &disk_max_score);
if (tablet != nullptr) {
tablets_compaction.emplace_back(tablet);
max_compaction_score = std::max(max_compaction_score, disk_max_score);
}
}
}
if (!tablets_compaction.empty()) {
if (compaction_type == CompactionType::BASE_COMPACTION) {
DorisMetrics::instance()->tablet_base_max_compaction_score->set_value(max_compaction_score);
} else {
DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value(max_compaction_score);
}
}
return tablets_compaction;
}

11
be/src/olap/tablet_manager.cpp
View File

@ -681,7 +681,7 @@ void TabletManager::get_tablet_stat(TTabletStatResult* result) {
TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
CompactionType compaction_type, DataDir* data_dir,
std::vector<TTabletId>& tablet_submitted_compaction) {
std::vector<TTabletId>& tablet_submitted_compaction, uint32_t* score) {
int64_t now_ms = UnixMillis();
const string& compaction_type_str =
compaction_type == CompactionType::BASE_COMPACTION ? "base" : "cumulative";
@ -776,14 +776,7 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
<< ", compaction_score=" << compaction_score
<< ", tablet_scan_frequency=" << tablet_scan_frequency
<< ", highest_score=" << highest_score;
// TODO(lingbin): Remove 'max' from metric name, it would be misunderstood as the
// biggest in history(like peak), but it is really just the value at current moment.
if (compaction_type == CompactionType::BASE_COMPACTION) {
DorisMetrics::instance()->tablet_base_max_compaction_score->set_value(compaction_score);
} else {
DorisMetrics::instance()->tablet_cumulative_max_compaction_score->set_value(
compaction_score);
}
*score = compaction_score;
}
return best_tablet;
}

3
be/src/olap/tablet_manager.h
View File

@ -71,7 +71,8 @@ public:
TabletSharedPtr find_best_tablet_to_compaction(CompactionType compaction_type,
DataDir* data_dir,
vector<TTabletId>& tablet_submitted_compaction);
vector<TTabletId>& tablet_submitted_compaction,
uint32_t* score);
TabletSharedPtr get_tablet(TTabletId tablet_id, SchemaHash schema_hash,
bool include_deleted = false, std::string* err = nullptr);