database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[refactor](conf) add be conf for partition topn partitions threshold (#23220)

Browse Source 
add be conf for partition topn partitions threshold
This commit is contained in:
zhangstar333
2023-08-21 10:52:41 +08:00
committed by GitHub
parent 33dfa0c454
commit 37b49f60b7
3 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/common/config.cpp
View File

@ -1059,6 +1059,8 @@ DEFINE_mBool(enable_merge_on_write_correctness_check, "true");
// The secure path with user files, used in the `local` table function.
DEFINE_mString(user_files_secure_path, "${DORIS_HOME}");
DEFINE_Int32(partition_topn_partition_threshold, "1024");
#ifdef BE_TEST
// test s3
DEFINE_String(test_s3_resource, "resource");

4
be/src/common/config.h
View File

@ -1116,6 +1116,10 @@ DECLARE_mBool(enable_merge_on_write_correctness_check);
// The secure path with user files, used in the `local` table function.
DECLARE_mString(user_files_secure_path);
// This threshold determines how many partitions will be allocated for window function get topn.
// and if this threshold is exceeded, the remaining data will be pass through to other node directly.
DECLARE_Int32(partition_topn_partition_threshold);
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);

3
be/src/vec/exec/vpartition_sort_node.cpp
View File

@ -181,7 +181,8 @@ Status VPartitionSortNode::sink(RuntimeState* state, vectorized::Block* input_bl
_value_places[0]->append_whole_block(input_block, child(0)->row_desc());
} else {
//just simply use partition num to check
if (_num_partition > 512 && child_input_rows < 10000 * _num_partition) {
if (_num_partition > config::partition_topn_partition_threshold &&
child_input_rows < 10000 * _num_partition) {
{
std::lock_guard<std::mutex> lock(_buffer_mutex);
_blocks_buffer.push(std::move(*input_block));