diff --git a/be/src/common/config.h b/be/src/common/config.h index fc06faf168..aeb9dc2fad 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -263,6 +263,8 @@ CONF_mBool(enable_ordered_data_compaction, "false"); CONF_mInt32(vertical_compaction_num_columns_per_group, "5"); // In vertical compaction, max memory usage for row_source_buffer CONF_Int32(vertical_compaction_max_row_source_memory_mb, "200"); +// In vertical compaction, max dest segment file size +CONF_mInt64(max_segment_size_in_vertical_compaction, "268435456"); // In ordered data compaction, min segment size for input rowset CONF_mInt32(ordered_data_compaction_min_segment_size, "10485760"); @@ -276,6 +278,7 @@ CONF_mInt64(base_compaction_min_rowset_num, "5"); CONF_mDouble(base_compaction_min_data_ratio, "0.3"); CONF_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024"); +CONF_Bool(enable_skip_tablet_compaction, "true"); // output rowset of cumulative compaction total disk size exceed this config size, // this rowset will be given to base compaction, unit is m byte. CONF_mInt64(compaction_promotion_size_mbytes, "1024"); diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 364fb976ed..8b2875afbd 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -101,8 +101,9 @@ int64_t Compaction::get_avg_segment_rows() { // take care of empty rowset // input_rowsets_size is total disk_size of input_rowset, this size is the // final size after codec and compress, so expect dest segment file size - // in disk is config::writer_buffer_size - return config::write_buffer_size / (_input_rowsets_size / (_input_row_num + 1) + 1); + // in disk is config::max_segment_size_in_vertical_compaction + return config::max_segment_size_in_vertical_compaction / + (_input_rowsets_size / (_input_row_num + 1) + 1); } bool Compaction::is_rowset_tidy(std::string& pre_max_key, const RowsetSharedPtr& rhs) { diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index c601891e79..f8e7200d10 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -648,7 +648,8 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction( std::shared_lock rdlock(tablets_shard.lock); for (const auto& tablet_map : tablets_shard.tablet_map) { const TabletSharedPtr& tablet_ptr = tablet_map.second; - if (tablet_ptr->should_skip_compaction(compaction_type, UnixSeconds())) { + if (config::enable_skip_tablet_compaction && + tablet_ptr->should_skip_compaction(compaction_type, UnixSeconds())) { continue; } if (!tablet_ptr->can_do_compaction(data_dir->path_hash(), compaction_type)) { @@ -691,8 +692,6 @@ TabletSharedPtr TabletManager::find_best_tablet_to_compaction( uint32_t current_compaction_score = tablet_ptr->calc_compaction_score( compaction_type, cumulative_compaction_policy); if (current_compaction_score < 5) { - VLOG_CRITICAL << "tablet set skip compaction, tablet_id: " - << tablet_ptr->tablet_id(); tablet_ptr->set_skip_compaction(true, compaction_type, UnixSeconds()); } if (current_compaction_score > highest_score) {