hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Revert shrink memory usage of ddl kv mgr

Browse Source
This commit is contained in:
Hongqin-Li
2023-03-08 15:15:59 +00:00
committed by ob-robot
parent a0edc1b86f
commit a8fb238d49
7 changed files with 19 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/storage/ddl/ob_complement_data_task.cpp
View File

@ -1245,7 +1245,7 @@ int ObComplementMergeTask::add_build_hidden_table_sstable()
} else if (OB_FAIL(param_->get_hidden_table_key(hidden_table_key))) {
LOG_WARN("fail to get hidden table key", K(ret), K(hidden_table_key));
} else if (OB_FAIL(context_->data_sstable_redo_writer_.end_ddl_redo_and_create_ddl_sstable(
ls_handle, context_->ddl_kv_mgr_handle_, hidden_table_key, param_->dest_table_id_, param_->execution_id_, param_->task_id_))) {
ls_handle, hidden_table_key, param_->dest_table_id_, param_->execution_id_, param_->task_id_))) {
LOG_WARN("failed to end ddl redo", K(ret));
}
return ret;

6
src/storage/ddl/ob_ddl_merge_task.cpp
View File

@ -312,7 +312,7 @@ int ObDDLTableMergeTask::init(const ObDDLTableMergeDagParam &ddl_dag_param)
int ObDDLTableMergeTask::process()
{
int ret = OB_SUCCESS;
int64_t MAX_DDL_SSTABLE = ObTabletDDLKvMgr::MAX_DDL_KV_CNT_IN_STORAGE * 0.5;
int64_t MAX_DDL_SSTABLE = 128;
#ifdef ERRSIM
if (0 != GCONF.errsim_max_ddl_sstable_count) {
MAX_DDL_SSTABLE = GCONF.errsim_max_ddl_sstable_count;
@ -426,10 +426,6 @@ int ObDDLTableMergeTask::process()
}
} else {
LOG_INFO("commit ddl sstable succ", K(ddl_param), K(merge_param_));
int tmp_ret = OB_SUCCESS;
if (OB_TMP_FAIL(ddl_kv_mgr_handle.get_obj()->unregister_from_tablet(merge_param_.start_scn_, ddl_kv_mgr_handle))) {
LOG_WARN("unregister ddl kv mgr from tablet failed", K(tmp_ret), K(merge_param_));
}
}
}
}

38
src/storage/ddl/ob_ddl_redo_log_writer.cpp
View File

@ -689,27 +689,17 @@ int ObDDLRedoLogWriter::write_ddl_start_log(ObTabletHandle &tablet_handle,
ret = OB_TASK_EXPIRED;
LOG_INFO("receive a old execution id, don't do ddl start", K(ret), K(log));
} else if (ddl_kv_mgr_handle.get_obj()->get_commit_scn_nolock(tablet_handle.get_obj()->get_tablet_meta()).is_valid_and_not_min()) {
// ddl commit log already written
if (ddl_kv_mgr_handle.get_obj()->get_start_scn().is_valid_and_not_min()) {
start_scn = ddl_kv_mgr_handle.get_obj()->get_start_scn();
if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->set_execution_id_nolock(log.get_execution_id()))) {
LOG_WARN("failed to set execution id", K(ret));
}
LOG_INFO("already committed, use new execution id", K(ret), K(start_scn),
K(log.get_execution_id()), "tablet_meta", tablet_handle.get_obj()->get_tablet_meta());
} else {
// ddl kv mgr is freed when ddl merge task finished, this is a new one, so recover it
start_scn = ddl_kv_mgr_handle.get_obj()->get_start_scn();
if (!start_scn.is_valid_and_not_min()) {
start_scn = tablet_handle.get_obj()->get_tablet_meta().ddl_start_scn_;
SCN commit_scn = tablet_handle.get_obj()->get_tablet_meta().ddl_commit_scn_;
if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_recover_nolock(log.get_table_key(),
start_scn,
log.get_data_format_version(),
log.get_execution_id(),
commit_scn))) {
LOG_WARN("start ddl log failed", K(ret), K(start_scn), K(log));
}
LOG_INFO("already committed, recover to success status", K(ret), K(start_scn), K(commit_scn),
K(log.get_execution_id()), "tablet_meta", tablet_handle.get_obj()->get_tablet_meta());
}
if (!start_scn.is_valid_and_not_min()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("start scn must be valid after commit", K(ret), K(start_scn));
} else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->set_execution_id_nolock(log.get_execution_id()))) {
LOG_WARN("failed to set execution id", K(ret));
} else {
LOG_INFO("already committed, use previous start scn", K(ret), K(tablet_handle.get_obj()->get_tablet_meta()));
}
} else if (OB_ISNULL(cb = op_alloc(ObDDLStartClogCb))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -1089,7 +1079,6 @@ int ObDDLSSTableRedoWriter::start_ddl_redo(const ObITable::TableKey &table_key,
int ObDDLSSTableRedoWriter::end_ddl_redo_and_create_ddl_sstable(
ObLSHandle &ls_handle,
ObDDLKvMgrHandle &ddl_kv_mgr_handle,
const ObITable::TableKey &table_key,
const uint64_t table_id,
const int64_t execution_id,
@ -1097,17 +1086,20 @@ int ObDDLSSTableRedoWriter::end_ddl_redo_and_create_ddl_sstable(
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObDDLKvMgrHandle ddl_kv_mgr_handle;
const ObTabletID &tablet_id = table_key.tablet_id_;
ObLS *ls = nullptr;
ObLSID ls_id;
SCN ddl_start_scn = get_start_scn();
SCN commit_scn = SCN::min_scn();
if (OB_ISNULL(ls = ls_handle.get_ls()) || !ddl_kv_mgr_handle.is_valid() || OB_UNLIKELY(!table_key.is_valid())) {
if (OB_ISNULL(ls = ls_handle.get_ls()) || OB_UNLIKELY(!table_key.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid ls", K(ret), KP(ls), K(ddl_kv_mgr_handle), K(table_key));
LOG_WARN("invalid ls", K(ret), K(table_key));
} else if (OB_FALSE_IT(ls_id = ls->get_ls_id())) {
} else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, tablet_id, tablet_handle))) {
LOG_WARN("get tablet failed", K(ret));
} else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle))) {
LOG_WARN("get ddl kv manager failed", K(ret), K(ls_id), K(tablet_id));
} else if (OB_FALSE_IT(ddl_kv_mgr_handle.get_obj()->prepare_info_for_checksum_report(table_id, ddl_task_id))) {
} else if (OB_FAIL(write_commit_log(tablet_handle, ddl_kv_mgr_handle, table_key, table_id, execution_id, ddl_task_id, commit_scn))) {
if (OB_TASK_EXPIRED == ret) {

1
src/storage/ddl/ob_ddl_redo_log_writer.h
View File

@ -269,7 +269,6 @@ public:
const int64_t data_format_version,
ObDDLKvMgrHandle &ddl_kv_mgr_handle);
int end_ddl_redo_and_create_ddl_sstable(ObLSHandle &ls_handle,
ObDDLKvMgrHandle &ddl_kv_mgr_handle,
const ObITable::TableKey &table_key,
const uint64_t table_id,
const int64_t execution_id,

2
src/storage/ddl/ob_direct_insert_sstable_ctx.cpp
View File

@ -864,7 +864,7 @@ int ObSSTableInsertTabletContext::create_sstable_with_clog(
} else {
DEBUG_SYNC(AFTER_REMOTE_WRITE_DDL_PREPARE_LOG);
if (OB_FAIL(data_sstable_redo_writer_.end_ddl_redo_and_create_ddl_sstable(
ls_handle_, ddl_kv_mgr_handle_, table_key, table_id, build_param_.execution_id_, build_param_.ddl_task_id_))) {
ls_handle_, table_key, table_id, build_param_.execution_id_, build_param_.ddl_task_id_))) {
LOG_WARN("fail create ddl sstable", K(ret), K(table_key));
}
}

39
src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp
View File

@ -170,39 +170,6 @@ int ObTabletDDLKvMgr::ddl_start(ObTablet &tablet,
return ret;
}
int ObTabletDDLKvMgr::ddl_recover_nolock(const ObITable::TableKey &table_key,
const SCN &start_scn,
const int64_t data_format_version,
const int64_t execution_id,
const SCN &commit_scn)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret), K(is_inited_));
} else if (OB_UNLIKELY(!table_key.is_valid() || !start_scn.is_valid_and_not_min() || execution_id < 0 || data_format_version < 0
|| (commit_scn.is_valid_and_not_min() && commit_scn < start_scn))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(ret), K(table_key), K(start_scn), K(execution_id), K(data_format_version), K(commit_scn));
} else if (table_key.get_tablet_id() != tablet_id_) {
ret = OB_ERR_SYS;
LOG_WARN("tablet id not same", K(ret), K(table_key), K(tablet_id_));
} else if (start_scn_.is_valid_and_not_min()) {
ret = OB_ERR_SYS;
LOG_WARN("start scn has set", K(ret), K(start_scn_));
} else {
table_key_ = table_key;
data_format_version_ = data_format_version;
execution_id_ = execution_id;
start_scn_ = start_scn;
max_freeze_scn_ = commit_scn;
commit_scn_ = commit_scn;
success_start_scn_ = start_scn;
}
FLOG_INFO("ddl recover finished", K(ret), K(*this));
return ret;
}
int ObTabletDDLKvMgr::ddl_commit(const SCN &start_scn,
const SCN &commit_scn,
const uint64_t table_id,
@ -352,12 +319,6 @@ int ObTabletDDLKvMgr::wait_ddl_merge_success(const SCN &start_scn, const SCN &co
"wait_elpased_s", (ObTimeUtility::fast_current_time() - wait_start_ts) / 1000000L);
}
}
if (OB_TASK_EXPIRED == ret) { // maybe ddl merge task has finished
if (is_commit_success()) {
ret = OB_SUCCESS;
FLOG_INFO("ddl commit already succeed", K(start_scn), K(commit_scn), K(*this));
}
}
}
return ret;
}

4
src/storage/ddl/ob_tablet_ddl_kv_mgr.h
View File

@ -80,7 +80,6 @@ public:
int get_ddl_major_merge_param(const ObTabletMeta &tablet_meta, ObDDLTableMergeDagParam &merge_param);
int get_rec_scn(share::SCN &rec_scn);
void prepare_info_for_checksum_report(const uint64_t table_id, const int64_t ddl_task_id) { table_id_ = table_id; ddl_task_id_ = ddl_task_id; }
int ddl_recover_nolock(const ObITable::TableKey &table_key, const share::SCN &start_scn, const int64_t data_format_version, const int64_t execution_id, const share::SCN &commit_scn);
TO_STRING_KV(K_(is_inited), K_(success_start_scn), K_(ls_id), K_(tablet_id), K_(table_key),
K_(data_format_version), K_(start_scn), K_(commit_scn), K_(max_freeze_scn),
K_(table_id), K_(execution_id), K_(ddl_task_id), K_(head), K_(tail), K_(ref_cnt));
@ -98,9 +97,8 @@ private:
void cleanup_unlock();
void destroy();
bool is_commit_success_unlock() const;
public:
static const int64_t MAX_DDL_KV_CNT_IN_STORAGE = 64;
private:
static const int64_t MAX_DDL_KV_CNT_IN_STORAGE = 64;
bool is_inited_;
share::SCN success_start_scn_;
share::ObLSID ls_id_;