database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](compaction) add some trigger and delete useless log (#14796)

Browse Source 
1.add a vertical compaction segment file size config, make it more
flexible to set segment file size
2.add a config to close skip tablet compaction. If current skip logic
has some bug so we can still use old logic
3.delete some useless log
This commit is contained in:
yixiutt
2022-12-07 18:53:55 +08:00
committed by GitHub
parent 6b44039d58
commit 204ab4c951
3 changed files with 8 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/common/config.h
View File

@ -263,6 +263,8 @@ CONF_mBool(enable_ordered_data_compaction, "false");
CONF_mInt32(vertical_compaction_num_columns_per_group, "5");
// In vertical compaction, max memory usage for row_source_buffer
CONF_Int32(vertical_compaction_max_row_source_memory_mb, "200");
// In vertical compaction, max dest segment file size
CONF_mInt64(max_segment_size_in_vertical_compaction, "268435456");
// In ordered data compaction, min segment size for input rowset
CONF_mInt32(ordered_data_compaction_min_segment_size, "10485760");
@ -276,6 +278,7 @@ CONF_mInt64(base_compaction_min_rowset_num, "5");
CONF_mDouble(base_compaction_min_data_ratio, "0.3");
CONF_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024");
CONF_Bool(enable_skip_tablet_compaction, "true");
// output rowset of cumulative compaction total disk size exceed this config size,
// this rowset will be given to base compaction, unit is m byte.
CONF_mInt64(compaction_promotion_size_mbytes, "1024");

5
be/src/olap/compaction.cpp
View File

@ -101,8 +101,9 @@ int64_t Compaction::get_avg_segment_rows() {
// take care of empty rowset
// input_rowsets_size is total disk_size of input_rowset, this size is the
// final size after codec and compress, so expect dest segment file size
// in disk is config::writer_buffer_size
return config::write_buffer_size / (_input_rowsets_size / (_input_row_num + 1) + 1);
// in disk is config::max_segment_size_in_vertical_compaction
return config::max_segment_size_in_vertical_compaction /
(_input_rowsets_size / (_input_row_num + 1) + 1);
}
bool Compaction::is_rowset_tidy(std::string& pre_max_key, const RowsetSharedPtr& rhs) {

5
be/src/olap/tablet_manager.cpp
View File

@ -648,7 +648,8 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
std::shared_lock rdlock(tablets_shard.lock);
for (const auto& tablet_map : tablets_shard.tablet_map) {
const TabletSharedPtr& tablet_ptr = tablet_map.second;
if (tablet_ptr->should_skip_compaction(compaction_type, UnixSeconds())) {
if (config::enable_skip_tablet_compaction &&
tablet_ptr->should_skip_compaction(compaction_type, UnixSeconds())) {
continue;
}
if (!tablet_ptr->can_do_compaction(data_dir->path_hash(), compaction_type)) {
@ -691,8 +692,6 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction(
uint32_t current_compaction_score = tablet_ptr->calc_compaction_score(
compaction_type, cumulative_compaction_policy);
if (current_compaction_score < 5) {
VLOG_CRITICAL << "tablet set skip compaction, tablet_id: "
<< tablet_ptr->tablet_id();
tablet_ptr->set_skip_compaction(true, compaction_type, UnixSeconds());
}
if (current_compaction_score > highest_score) {