diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 409cb6a9b3..cc1df07c3f 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -519,8 +519,6 @@ void StorageEngine::stop() { } \ } - THREADS_JOIN(_base_compaction_threads); - THREADS_JOIN(_cumulative_compaction_threads); THREADS_JOIN(_path_gc_threads); THREADS_JOIN(_path_scan_threads); THREADS_JOIN(_tablet_checkpoint_threads); diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index fb7de1d566..e5aaf4e939 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -303,10 +303,7 @@ private: scoped_refptr _garbage_sweeper_thread; // thread to monitor disk stat scoped_refptr _disk_stat_monitor_thread; - // threads to run base compaction - std::vector> _base_compaction_threads; - // threads to check cumulative - std::vector> _cumulative_compaction_threads; + // thread to produce both base and cumulative compaction tasks scoped_refptr _compaction_tasks_producer_thread; scoped_refptr _fd_cache_clean_thread; // threads to clean all file descriptor not actively in use diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md index e1d290d6a2..2b26b8647c 100644 --- a/docs/en/administrator-guide/config/be_config.md +++ b/docs/en/administrator-guide/config/be_config.md @@ -183,12 +183,19 @@ User can set this configuration to a larger value to get better QPS performance. ### `buffer_pool_clean_pages_limit` ### `buffer_pool_limit` + * Type: string * Description: The largest allocatable memory of the buffer pool * Default value: 80G The maximum amount of memory available in the BE buffer pool. The buffer pool is a new memory management structure of BE, which manages the memory by the buffer page and enables spill data to disk. The memory for all concurrent queries will be allocated from the buffer pool. The current buffer pool only works on **AggregationNode** and **ExchangeNode**. +### `check_auto_compaction_interval_seconds` + +* Type: int32 +* Description: Check the configuration of auto compaction in seconds when auto compaction disabled. +* Default value: 5 + ### `check_consistency_worker_count` ### `chunk_reserved_bytes_limit` @@ -226,6 +233,12 @@ Tablet score can be calculated like this: tablet_score = compaction_tablet_scan_frequency_factor * tablet_scan_frequency + compaction_tablet_scan_frequency_factor * compaction_score +### `compaction_task_num_per_disk` + +* Type: int32 +* Description: The number of compaction tasks which execute in parallel for a disk. +* Default value: 2 + ### `compress_rowbatches` * Type: bool @@ -234,6 +247,12 @@ tablet_score = compaction_tablet_scan_frequency_factor * tablet_scan_frequency + ### `create_tablet_worker_count` +### `cumulative_compaction_rounds_for_each_base_compaction_round` + +* Type: int32 +* Description: How many rounds of cumulative compaction for each round of base compaction when compaction tasks generation. +* Default value: 9 + ### `disable_auto_compaction` * Type: bool @@ -503,7 +522,11 @@ Indicates how many tablets in this data directory failed to load. At the same ti ### `max_client_cache_size_per_host` -### `max_compaction_concurrency` +### `max_compaction_threads` + +* Type: int32 +* Description: The maximum of thread number in compaction thread pool. +* Default value: 10 ### `max_consumer_num_per_group` @@ -568,6 +591,12 @@ Indicates how many tablets in this data directory failed to load. At the same ti * Default value: 600 * Unit: seconds +### `min_compaction_threads` + +* Type: int32 +* Description: The minimum of thread number in compaction thread pool. +* Default value: 10 + ### `min_cumulative_compaction_num_singleton_deltas` ### `min_file_descriptor_number` diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md index 4deb54877b..1f4a45d0af 100644 --- a/docs/zh-CN/administrator-guide/config/be_config.md +++ b/docs/zh-CN/administrator-guide/config/be_config.md @@ -168,6 +168,7 @@ Metrics: {"filtered_rows":0,"input_row_num":3346807,"input_rowsets_count":42,"in 用户可以将该配置的值调大来获取更好的QPS性能。更多的信息可以参考`https://github.com/apache/incubator-brpc/blob/master/docs/cn/benchmark.md`。 ### `brpc_port` + * 类型：int32 * 描述：BE 上的 brpc 的端口，用于 BE 之间通讯 * 默认值：8060 @@ -175,12 +176,19 @@ Metrics: {"filtered_rows":0,"input_row_num":3346807,"input_rowsets_count":42,"in ### `buffer_pool_clean_pages_limit` ### `buffer_pool_limit` + * 类型：string * 描述：buffer pool之中最大的可分配内存 * 默认值：80G BE缓存池最大的内存可用量，buffer pool是BE新的内存管理结构，通过buffer page来进行内存管理，并能够实现数据的落盘。并发的所有查询的内存申请都会通过buffer pool来申请。当前buffer pool仅作用在**AggregationNode**与**ExchangeNode**。 +### `check_auto_compaction_interval_seconds` + +* 类型：int32 +* 描述：当自动执行compaction的功能关闭时，检查自动compaction开关是否被开启的时间间隔。 +* 默认值：5 + ### `check_consistency_worker_count` ### `chunk_reserved_bytes_limit` @@ -220,6 +228,11 @@ tablet score可以通过以下公式计算： tablet_score = compaction_tablet_scan_frequency_factor * tablet_scan_frequency + compaction_tablet_scan_frequency_factor * compaction_score +### `compaction_task_num_per_disk` + +* 类型：int32 +* 描述：每个磁盘可以并发执行的compaction任务数量。 +* 默认值：2 ### `compress_rowbatches` * 类型：bool @@ -230,6 +243,12 @@ tablet_score = compaction_tablet_scan_frequency_factor * tablet_scan_frequency + ### `create_tablet_worker_count` +### `cumulative_compaction_rounds_for_each_base_compaction_round` + +* 类型：int32 +* 描述：Compaction任务的生产者每次连续生产多少轮cumulative compaction任务后生产一轮base compaction。 +* 默认值：9 + ### `disable_auto_compaction` * 类型：bool @@ -502,7 +521,11 @@ load tablets from header failed, failed tablets size: xxx, path=xxx ### `max_client_cache_size_per_host` -### `max_compaction_concurrency` +### `max_compaction_threads` + +* 类型：int32 +* 描述：Compaction线程池中线程数量的最大值。 +* 默认值：10 ### `max_consumer_num_per_group` @@ -567,6 +590,12 @@ load tablets from header failed, failed tablets size: xxx, path=xxx * 默认值：600 * 单位：秒 +### `min_compaction_threads` + +* 类型：int32 +* 描述：Compaction线程池中线程数量的最小值。 +* 默认值：10 + ### `min_cumulative_compaction_num_singleton_deltas` ### `min_file_descriptor_number`