hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

prevent mini dag from being created when sstable count reaches up limit && optimize perf when getting tenant config

Browse Source
This commit is contained in:
obdev
2023-03-02 14:54:35 +00:00
committed by ob-robot
parent 9e0e3b8aa9
commit 3f67e0ac31
6 changed files with 40 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/storage/access/ob_multiple_merge.h
View File

@ -28,6 +28,7 @@
#include "ob_table_access_context.h"
#include "storage/meta_mem/ob_tablet_handle.h"
#include "storage/lob/ob_lob_data_reader.h"
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
namespace oceanbase
{
@ -171,14 +172,7 @@ OB_INLINE int ObMultipleMerge::check_need_refresh_table(bool &need_refresh)
OB_INLINE int ObMultipleMerge::update_and_report_tablet_stat()
{
int ret = OB_SUCCESS;
bool enable_adaptive_compaction = true;
{
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
}
}
if (enable_adaptive_compaction) {
if (MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
EVENT_ADD(ObStatEventIds::STORAGE_READ_ROW_COUNT, scan_cnt_);
access_ctx_->table_store_stat_.access_row_cnt_ += row_stat_.filt_del_count_;
if (NULL != access_ctx_->table_scan_stat_) {

6
src/storage/compaction/ob_partition_merge_policy.cpp
View File

@ -1326,15 +1326,21 @@ int ObAdaptiveMergePolicy::find_meta_major_tables(
if (OB_FAIL(ret)) {
} else if (tx_determ_table_cnt < 2) {
ret = OB_NO_NEED_MERGE;
if (REACH_TENANT_TIME_INTERVAL(60 * 1000 * 1000/*60s*/)) {
LOG_INFO("no enough table for meta merge", K(ret), K(result), K(table_store));
}
} else if (inc_row_cnt < TRANS_STATE_DETERM_ROW_CNT_THRESHOLD
|| inc_row_cnt < INC_ROW_COUNT_PERCENTAGE_THRESHOLD * base_row_cnt) {
ret = OB_NO_NEED_MERGE;
if (REACH_TENANT_TIME_INTERVAL(60 * 1000 * 1000/*60s*/)) {
LOG_INFO("found sstable could merge is not enough", K(ret), K(inc_row_cnt), K(base_row_cnt));
}
} else if (result.version_range_.snapshot_version_ < tablet.get_multi_version_start()) {
ret = OB_NO_NEED_MERGE;
if (REACH_TENANT_TIME_INTERVAL(60 * 1000 * 1000/*60s*/)) {
LOG_INFO("chosen snapshot is abandoned", K(ret), K(result), K(tablet.get_multi_version_start()));
}
}
#ifdef ERRSIM
ret = OB_E(EventTable::EN_SCHEDULE_MEDIUM_COMPACTION) ret;

23
src/storage/compaction/ob_tablet_merge_task.cpp
View File

@ -240,6 +240,21 @@ int ObBasicTabletMergeDag::get_tablet_and_compat_mode()
compat_mode_ = tmp_tablet_handle.get_obj()->get_tablet_meta().compat_mode_;
}
if (OB_SUCC(ret) && is_mini_merge(merge_type_)) {
int64_t inc_sstable_cnt = tmp_tablet_handle.get_obj()->get_table_store().get_minor_sstables().count() + 1/*major table*/;
bool is_exist = false;
if (OB_FAIL(MTL(ObTenantDagScheduler *)->check_dag_exist(this, is_exist))) {
LOG_WARN("failed to check dag exist", K(ret), K_(param));
} else if (is_exist) {
++inc_sstable_cnt;
}
if (OB_SUCC(ret) && inc_sstable_cnt >= MAX_SSTABLE_CNT_IN_STORAGE) {
ret = OB_EAGAIN;
LOG_WARN("Too many sstables in tablet, cannot schdule mini compaction, retry later",
K(ret), K(inc_sstable_cnt), K(tmp_tablet_handle.get_obj()));
}
}
int tmp_ret = OB_SUCCESS;
if (OB_SUCC(ret)
&& typeid(*this) != typeid(ObTxTableMergeDag)
@ -1285,13 +1300,7 @@ int ObTabletMergeFinishTask::try_schedule_compaction_after_mini(
int tmp_ret = OB_SUCCESS;
const ObTabletID &tablet_id = ctx.param_.tablet_id_;
ObLSID ls_id = ctx.param_.ls_id_;
bool enable_adaptive_compaction = true;
{
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
}
}
bool enable_adaptive_compaction = MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction();
// report tablet stat
if (0 == ctx.get_merge_info().get_sstable_merge_info().macro_block_count_) {
// empty mini compaction, no need to reprot stat

16
src/storage/compaction/ob_tenant_tablet_scheduler.cpp
View File

@ -166,7 +166,8 @@ ObTenantTabletScheduler::ObTenantTabletScheduler()
merge_loop_task_(),
medium_loop_task_(),
sstable_gc_task_(),
fast_freeze_checker_()
fast_freeze_checker_(),
enable_adaptive_compaction_(false)
{
STATIC_ASSERT(static_cast<int64_t>(NO_MAJOR_MERGE_TYPE_CNT) == ARRAYSIZEOF(MERGE_TYPES), "merge type array len is mismatch");
}
@ -204,6 +205,7 @@ int ObTenantTabletScheduler::init()
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
schedule_interval = tenant_config->ob_compaction_schedule_interval;
enable_adaptive_compaction_ = tenant_config->_enable_adaptive_compaction;
fast_freeze_checker_.reload_config(tenant_config->_ob_enable_fast_freeze);
}
} // end of ObTenantConfigGuard
@ -266,6 +268,7 @@ int ObTenantTabletScheduler::reload_tenant_config()
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
merge_schedule_interval = tenant_config->ob_compaction_schedule_interval;
enable_adaptive_compaction_ = tenant_config->_enable_adaptive_compaction;
fast_freeze_checker_.reload_config(tenant_config->_ob_enable_fast_freeze);
}
} // end of ObTenantConfigGuard
@ -931,13 +934,6 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge(
bool is_leader = false;
bool could_major_merge = false;
const int64_t major_frozen_scn = get_frozen_version();
bool enable_adaptive_compaction = true;
{
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
}
}
if (MTL(ObTenantTabletScheduler *)->could_major_merge_start()) {
could_major_merge = true;
@ -1001,13 +997,13 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge(
} else if (ObTimeUtility::fast_current_time() <
tablet->get_medium_compaction_info_list().get_wait_check_medium_scn() + WAIT_MEDIUM_CHECK_THRESHOLD) {
// need wait 10 mins before schedule meta major
} else if (enable_adaptive_compaction && OB_TMP_FAIL(schedule_tablet_meta_major_merge(ls_handle, tablet_handle))) {
} else if (enable_adaptive_compaction_ && OB_TMP_FAIL(schedule_tablet_meta_major_merge(ls_handle, tablet_handle))) {
if (OB_SIZE_OVERFLOW != tmp_ret && OB_EAGAIN != tmp_ret) {
LOG_WARN("failed to schedule tablet merge", K(tmp_ret), K(ls_id), K(tablet_id));
}
}
} else if (could_major_merge
&& (!tablet_merge_finish || enable_adaptive_compaction)
&& (!tablet_merge_finish || enable_adaptive_compaction_)
&& OB_TMP_FAIL(func.schedule_next_medium_for_leader(
tablet_merge_finish ? 0 : merge_version, schedule_stats_))) { // schedule another round
LOG_WARN("failed to schedule next medium", K(tmp_ret), K(ls_id), K(tablet_id));

2
src/storage/compaction/ob_tenant_tablet_scheduler.h
View File

@ -103,6 +103,7 @@ public:
void wait();
bool is_stop() const { return is_stop_; }
int reload_tenant_config();
bool enable_adaptive_compaction() const { return enable_adaptive_compaction_; }
// major merge status control
void stop_major_merge();
@ -241,6 +242,7 @@ private:
MediumLoopTask medium_loop_task_;
SSTableGCTask sstable_gc_task_;
ObFastFreezeChecker fast_freeze_checker_;
bool enable_adaptive_compaction_;
};
} // namespace storage

23
src/storage/tablet/ob_tablet.cpp
View File

@ -39,6 +39,7 @@
#include "storage/blocksstable/ob_sstable.h"
#include "storage/blocksstable/ob_sstable_sec_meta_iterator.h"
#include "storage/compaction/ob_tenant_freeze_info_mgr.h"
#include "storage/compaction/ob_tenant_tablet_scheduler.h"
#include "storage/memtable/ob_memtable.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
#include "storage/meta_mem/ob_meta_obj_struct.h"
@ -1484,15 +1485,8 @@ int ObTablet::do_rowkey_exists(
// ROWKEY IN_ROW_CACHE / NOT EXIST
} else if (FALSE_IT(store_ctx.tablet_stat_.exist_row_read_table_cnt_ = check_table_cnt)) {
} else if (FALSE_IT(store_ctx.tablet_stat_.exist_row_total_table_cnt_ = table_iter.count())) {
} else {
bool enable_adaptive_compaction = true;
{
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
}
}
if (enable_adaptive_compaction && OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(store_ctx.tablet_stat_))) {
} else if (MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
if (OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(store_ctx.tablet_stat_))) {
LOG_WARN("failed to report tablet stat", K(tmp_ret), K(stat));
}
}
@ -1545,15 +1539,8 @@ int ObTablet::do_rowkeys_exist(ObTableStoreIterator &tables_iter, ObRowsInfo &ro
int tmp_ret = OB_SUCCESS;
if (0 == access_ctx.table_store_stat_.exist_row_.empty_read_cnt_) {
// ROWKEY IN_ROW_CACHE / NOT EXIST
} else {
bool enable_adaptive_compaction = true;
{
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
if (tenant_config.is_valid()) {
enable_adaptive_compaction = tenant_config->_enable_adaptive_compaction;
}
}
if (enable_adaptive_compaction && OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(tablet_stat))) {
} else if (MTL(ObTenantTabletScheduler *)->enable_adaptive_compaction()) {
if (OB_TMP_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(tablet_stat))) {
LOG_WARN("failed to report tablet stat", K(tmp_ret), K(tablet_stat));
}
}