database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[Feature] compaction quickly for small data import (#9804)

Browse Source 
* compaction quickly for small data import #9791
1.merge small versions of rowset as soon as possible to increase the import frequency of small version data
2.small version means that the number of rows is less than config::small_compaction_rowset_rows  default 1000
This commit is contained in:
chenlinzhong
2022-06-15 21:48:34 +08:00
committed by GitHub
parent c4871fb306
commit 4dfebb9852
12 changed files with 195 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
be/src/agent/task_worker_pool.cpp
View File

@ -692,11 +692,13 @@ void TaskWorkerPool::_publish_version_worker_thread_callback() {
VLOG_NOTICE << "get publish version task, signature:" << agent_task_req.signature;
std::vector<TTabletId> error_tablet_ids;
std::vector<TTabletId> succ_tablet_ids;
uint32_t retry_time = 0;
Status res = Status::OK();
while (retry_time < PUBLISH_VERSION_MAX_RETRY) {
error_tablet_ids.clear();
EnginePublishVersionTask engine_task(publish_version_req, &error_tablet_ids);
EnginePublishVersionTask engine_task(publish_version_req, &error_tablet_ids,
&succ_tablet_ids);
res = _env->storage_engine()->execute_task(&engine_task);
if (res.ok()) {
break;
@ -718,7 +720,29 @@ void TaskWorkerPool::_publish_version_worker_thread_callback() {
<< ", error_code=" << res;
finish_task_request.__set_error_tablet_ids(error_tablet_ids);
} else {
LOG(INFO) << "publish_version success. signature:" << agent_task_req.signature;
int submit_tablets = 0;
if (config::enable_quick_compaction && config::quick_compaction_batch_size > 0) {
for (int i = 0; i < succ_tablet_ids.size(); i++) {
TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(
succ_tablet_ids[i]);
if (tablet != nullptr) {
submit_tablets++;
tablet->publised_count++;
if (tablet->publised_count % config::quick_compaction_batch_size == 0) {
StorageEngine::instance()->submit_quick_compaction_task(tablet);
LOG(INFO) << "trigger quick compaction succ, tabletid:"
<< succ_tablet_ids[i]
<< ", publised:" << tablet->publised_count;
}
} else {
LOG(WARNING) << "trigger quick compaction failed, tabletid:"
<< succ_tablet_ids[i];
}
}
LOG(INFO) << "publish_version success. signature:" << agent_task_req.signature
<< ", size:" << succ_tablet_ids.size();
}
}
res.to_thrift(&finish_task_request.task_status);