From 37b49f60b7861cc39b29d7bbbe979f8794ceeec4 Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Mon, 21 Aug 2023 10:52:41 +0800 Subject: [PATCH] [refactor](conf) add be conf for partition topn partitions threshold (#23220) add be conf for partition topn partitions threshold --- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 4 ++++ be/src/vec/exec/vpartition_sort_node.cpp | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index d2be829578..7998e15374 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1059,6 +1059,8 @@ DEFINE_mBool(enable_merge_on_write_correctness_check, "true"); // The secure path with user files, used in the `local` table function. DEFINE_mString(user_files_secure_path, "${DORIS_HOME}"); +DEFINE_Int32(partition_topn_partition_threshold, "1024"); + #ifdef BE_TEST // test s3 DEFINE_String(test_s3_resource, "resource"); diff --git a/be/src/common/config.h b/be/src/common/config.h index bd009f2b4e..a9f69e2a94 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1116,6 +1116,10 @@ DECLARE_mBool(enable_merge_on_write_correctness_check); // The secure path with user files, used in the `local` table function. DECLARE_mString(user_files_secure_path); +// This threshold determines how many partitions will be allocated for window function get topn. +// and if this threshold is exceeded, the remaining data will be pass through to other node directly. +DECLARE_Int32(partition_topn_partition_threshold); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/vec/exec/vpartition_sort_node.cpp b/be/src/vec/exec/vpartition_sort_node.cpp index f9c767276e..b9907accbb 100644 --- a/be/src/vec/exec/vpartition_sort_node.cpp +++ b/be/src/vec/exec/vpartition_sort_node.cpp @@ -181,7 +181,8 @@ Status VPartitionSortNode::sink(RuntimeState* state, vectorized::Block* input_bl _value_places[0]->append_whole_block(input_block, child(0)->row_desc()); } else { //just simply use partition num to check - if (_num_partition > 512 && child_input_rows < 10000 * _num_partition) { + if (_num_partition > config::partition_topn_partition_threshold && + child_input_rows < 10000 * _num_partition) { { std::lock_guard lock(_buffer_mutex); _blocks_buffer.push(std::move(*input_block));