database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[enhancement](merge-on-write) update publish/streamload/compaction co… (#21040)

Browse Source
This commit is contained in:
zhannngchen
2023-06-21 14:49:51 +08:00
committed by GitHub
parent 62fb0e642e
commit 564b3533cf
3 changed files with 7 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
be/src/agent/task_worker_pool.cpp
View File

@ -90,7 +90,6 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(agent_task_queue_size, MetricUnit::NOUNIT);
const uint32_t TASK_FINISH_MAX_RETRY = 3;
const uint32_t PUBLISH_VERSION_MAX_RETRY = 3;
const int64_t PUBLISH_TIMEOUT_SEC = 10;
std::atomic_ulong TaskWorkerPool::_s_report_version(time(nullptr) * 10000);
std::mutex TaskWorkerPool::_s_task_signatures_lock;
@ -1461,7 +1460,7 @@ void PublishVersionTaskPool::_publish_version_worker_thread_callback() {
break;
} else if (status.is<PUBLISH_VERSION_NOT_CONTINUOUS>()) {
int64_t time_elapsed = time(nullptr) - agent_task_req.recv_time;
if (time_elapsed > PUBLISH_TIMEOUT_SEC) {
if (time_elapsed > config::publish_version_task_timeout_s) {
LOG(INFO) << "task elapsed " << time_elapsed
<< " seconds since it is inserted to queue, it is timeout";
is_task_timeout = true;

6
be/src/common/config.cpp
View File

@ -145,6 +145,8 @@ DEFINE_Int32(push_worker_count_high_priority, "3");
DEFINE_Int32(publish_version_worker_count, "8");
// the count of tablet thread to publish version
DEFINE_Int32(tablet_publish_txn_max_thread, "32");
// the timeout of EnginPublishVersionTask
DEFINE_Int32(publish_version_task_timeout_s, "8");
// the count of thread to calc delete bitmap
DEFINE_Int32(calc_delete_bitmap_max_thread, "32");
// the count of thread to clear transaction task
@ -353,7 +355,7 @@ DEFINE_mDouble(compaction_promotion_ratio, "0.05");
// the smallest size of rowset promotion. When the rowset is less than this config, this
// rowset will be not given to base compaction. The unit is m byte.
DEFINE_mInt64(compaction_promotion_min_size_mbytes, "64");
DEFINE_mInt64(compaction_promotion_min_size_mbytes, "128");
// The lower bound size to do cumulative compaction. When total disk size of candidate rowsets is less than
// this size, size_based policy may not do to cumulative compaction. The unit is m byte.
@ -589,7 +591,7 @@ DEFINE_mInt32(priority_queue_remaining_tasks_increased_frequency, "512");
DEFINE_mBool(sync_tablet_meta, "false");
// default thrift rpc timeout ms
DEFINE_mInt32(thrift_rpc_timeout_ms, "10000");
DEFINE_mInt32(thrift_rpc_timeout_ms, "20000");
// txn commit rpc timeout
DEFINE_mInt32(txn_commit_rpc_timeout_ms, "10000");

2
be/src/common/config.h
View File

@ -185,6 +185,8 @@ DECLARE_Int32(push_worker_count_high_priority);
DECLARE_Int32(publish_version_worker_count);
// the count of tablet thread to publish version
DECLARE_Int32(tablet_publish_txn_max_thread);
// the timeout of EnginPublishVersionTask
DECLARE_Int32(publish_version_task_timeout_s);
// the count of thread to calc delete bitmap
DECLARE_Int32(calc_delete_bitmap_max_thread);
// the count of thread to clear transaction task