hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

split compaction parallel range for big table

Browse Source
This commit is contained in:
obdev
2024-02-27 03:45:23 +00:00
committed by ob-robot
parent 1f5b89e9d6
commit 51fe091ef2
10 changed files with 97 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
src/storage/compaction/ob_medium_compaction_func.cpp
View File

@ -713,6 +713,13 @@ int ObMediumCompactionScheduleFunc::init_parallel_range_and_schema_changed(
expected_task_count = 2;
LOG_INFO("ERRSIM EN_COMPACTION_MEDIUM_INIT_PARALLEL_RANGE", KPC(this), K(expected_task_count));
ret = OB_SUCCESS;
} else {
ret = OB_E(EventTable::EN_COMPACTION_MEDIUM_INIT_LARGE_PARALLEL_RANGE) ret;
if (OB_FAIL(ret)) {
expected_task_count = 64;
LOG_INFO("ERRSIM EN_COMPACTION_MEDIUM_INIT_LARGE_PARALLEL_RANGE", KPC(this), K(expected_task_count));
ret = OB_SUCCESS;
}
}
}
#endif
@ -738,16 +745,50 @@ int ObMediumCompactionScheduleFunc::init_parallel_range_and_schema_changed(
LOG_WARN("failed to get table iter", K(ret), K(range_array));
} else if (OB_FAIL(input_range_array.push_back(range))) {
LOG_WARN("failed to push back range", K(ret), K(range));
} else if (OB_FAIL(range_spliter.get_split_multi_ranges(
input_range_array,
expected_task_count,
tablet->get_rowkey_read_info(),
table_iter,
allocator_,
range_array))) {
LOG_WARN("failed to get split multi range", K(ret), K(range_array));
} else if (OB_FAIL(medium_info.gene_parallel_info(allocator_, range_array))) {
LOG_WARN("failed to get parallel ranges", K(ret), K(range_array));
} else {
bool recalc_count_flag = false;
do {
if (OB_FAIL(range_spliter.get_split_multi_ranges(
input_range_array,
expected_task_count,
tablet->get_rowkey_read_info(),
table_iter,
allocator_,
range_array))) {
LOG_WARN("failed to get split multi range", K(ret), K(range_array));
} else if (OB_FAIL(medium_info.gene_parallel_info(allocator_, range_array))) {
LOG_WARN("failed to get parallel ranges", K(ret), K(range_array));
} else {
int64_t buf_len = ObTabletMediumCompactionInfoRecorder::cal_buf_len(tablet->get_tablet_meta().tablet_id_, medium_info, nullptr/*log_header*/);
#ifdef ERRSIM
ret = OB_E(EventTable::EN_COMPACTION_MEDIUM_INIT_LARGE_PARALLEL_RANGE) ret;
if (OB_FAIL(ret)) {
ret = OB_SUCCESS;
if (!recalc_count_flag) {
buf_len = common::OB_MAX_LOG_ALLOWED_SIZE;
}
}
#endif
if (buf_len < common::OB_MAX_LOG_ALLOWED_SIZE) {
LOG_TRACE("success to split ranges", KR(ret), K(buf_len), K(medium_info.parallel_merge_info_), K(range_array), K(medium_info.parallel_merge_info_.get_serialize_size()));
break;
} else if (recalc_count_flag) {
expected_task_count -= MAX(1, expected_task_count / 5);
} else {
recalc_count_flag = true;
// get parallel info serialize size
const int64_t parallel_size = medium_info.parallel_merge_info_.get_serialize_size();
const double avg_range_size = (parallel_size + 0.0) / range_array.count();
const int64_t rest_info_size = buf_len - parallel_size;
expected_task_count = MAX(1, (common::OB_MAX_LOG_ALLOWED_SIZE - 1 - rest_info_size) / avg_range_size);
expected_task_count = MIN(expected_task_count, MAX_MERGE_THREAD);
LOG_INFO("success to recalc ranges", KR(ret), K(buf_len), K(expected_task_count), K(avg_range_size), K(rest_info_size));
}
medium_info.clear_parallel_range();
table_iter.resume();
range_array.reuse();
}
} while (OB_SUCC(ret) && !medium_info.contain_parallel_range_ && expected_task_count > 1);
}
}
}
@ -944,7 +985,7 @@ int ObMediumCompactionScheduleFunc::submit_medium_clog(
int ObMediumCompactionScheduleFunc::batch_check_medium_meta_table(
const ObIArray<ObTabletCheckInfo> &tablet_ls_infos,
hash::ObHashMap<ObLSID, share::ObLSInfo> &ls_info_map,
const hash::ObHashMap<ObLSID, share::ObLSInfo> &ls_info_map,
ObIArray<ObTabletCheckInfo> &finish_tablet_ls,
ObCompactionTimeGuard &time_guard)
{
@ -993,7 +1034,7 @@ int ObMediumCompactionScheduleFunc::check_medium_meta_table(
const int64_t check_medium_snapshot,
const ObTabletInfo &tablet_info,
const share::ObTabletReplicaFilterHolder &filters,
hash::ObHashMap<ObLSID, share::ObLSInfo> &ls_info_map,
const hash::ObHashMap<ObLSID, share::ObLSInfo> &ls_info_map,
bool &merge_finish)
{
int ret = OB_SUCCESS;
@ -1167,7 +1208,7 @@ int ObMediumCompactionScheduleFunc::batch_check_medium_checksum(
// for Leader, clean wait_check_medium_scn
int ObMediumCompactionScheduleFunc::batch_check_medium_finish(
hash::ObHashMap<ObLSID, share::ObLSInfo> &ls_info_map,
const hash::ObHashMap<ObLSID, share::ObLSInfo> &ls_info_map,
ObIArray<ObTabletCheckInfo> &finish_tablet_ls_infos,
const ObIArray<ObTabletCheckInfo> &tablet_ls_infos,
ObCompactionTimeGuard &time_guard)