hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

avoid slow tablet load and dump operation to optimize ls offline procedure

Browse Source
This commit is contained in:
hiddenbomb
2023-09-27 09:40:19 +00:00
committed by ob-robot
parent ba2ac56c8e
commit faefeaa40f
10 changed files with 68 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -175,7 +175,7 @@ int ObLSTabletService::offline()
} else {
DestroyMemtableAndMemberAndMdsTableOperator clean_mem_op(this);
if (OB_FAIL(tablet_id_set_.foreach(clean_mem_op))) {
LOG_WARN("fail to clean memtables", K(ret), K(clean_mem_op.cur_tablet_id_));
LOG_WARN("fail to clean memtables", K(ret), "cur_tablet_id", clean_mem_op.cur_tablet_id_);
}
mds_table_mgr_.offline();
}
@ -5917,28 +5917,19 @@ int ObLSTabletService::GetAllTabletIDOperator::operator()(const common::ObTablet
int ObLSTabletService::DestroyMemtableAndMemberAndMdsTableOperator::operator()(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObTabletHandle handle;
const uint64_t tenant_id = MTL_ID();
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
cur_tablet_id_ = tablet_id;
if (OB_UNLIKELY(!tablet_id.is_valid()) ||
OB_ISNULL(tablet_svr_)) {
if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_ISNULL(tablet_svr_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(tablet_svr_));
} else if (OB_FAIL(tablet_svr_->get_tablet(tablet_id, handle, 0,
ObMDSGetTabletMode::READ_WITHOUT_CHECK))) {
if (OB_TABLET_NOT_EXIST == ret) {
LOG_WARN("failed to get tablet, skip clean memtable", K(ret), K(tablet_id));
ret = OB_SUCCESS;
} else {
LOG_ERROR("failed to get tablet", K(ret), K(tablet_id));
} else if (OB_ISNULL(tablet_svr_->ls_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls is null", K(ret));
} else {
const ObTabletMapKey key(tablet_svr_->ls_->get_ls_id(), tablet_id);
if (OB_FAIL(t3m->release_memtable_and_mds_table_for_ls_offline(key))) {
LOG_WARN("failed to release memtables and mds table", K(ret), K(key));
}
} else if (OB_FAIL(handle.get_obj()->release_memtables())) {
LOG_WARN("failed to release memtables", K(tenant_id), K(tablet_id));
} else if (OB_FAIL(handle.get_obj()->forcely_reset_mds_table("OFFLINE"))) {
LOG_WARN("failed to release mds_table", K(tenant_id), K(tablet_id));
} else if (!tablet_id.is_ls_inner_tablet() && OB_FAIL(handle.get_obj()->reset_storage_related_member())) {
LOG_WARN("failed to destroy storage related member", K(ret), K(tenant_id), K(tablet_id));
}
return ret;
}
@ -5946,12 +5937,9 @@ int ObLSTabletService::DestroyMemtableAndMemberAndMdsTableOperator::operator()(c
int ObLSTabletService::SetMemtableFrozenOperator::operator()(const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
ObTabletHandle handle;
const uint64_t tenant_id = MTL_ID();
cur_tablet_id_ = tablet_id;
if (OB_UNLIKELY(!tablet_id.is_valid()) ||
OB_ISNULL(tablet_svr_)) {
if (OB_UNLIKELY(!tablet_id.is_valid()) || OB_ISNULL(tablet_svr_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(tablet_svr_));
} else if (OB_FAIL(tablet_svr_->get_tablet(tablet_id,
@ -5965,7 +5953,7 @@ int ObLSTabletService::SetMemtableFrozenOperator::operator()(const common::ObTab
LOG_ERROR("failed to get tablet", K(ret), K(tablet_id));
}
} else if (OB_FAIL(handle.get_obj()->set_frozen_for_all_memtables())) {
LOG_WARN("failed to set frozen for all memtables", K(tenant_id), K(tablet_id));
LOG_WARN("failed to set frozen for all memtables", K(ret), K(tablet_id));
}
return ret;
}

25
src/storage/meta_mem/ob_tablet_pointer.cpp
View File

@ -341,6 +341,31 @@ void ObTabletPointer::mark_mds_table_deleted()
return mds_table_handler_.mark_removed_from_t3m(this);
}
int ObTabletPointer::release_memtable_and_mds_table_for_ls_offline()
{
int ret = OB_SUCCESS;
ObIMemtableMgr *memtable_mgr = memtable_mgr_handle_.get_memtable_mgr();
mds::MdsTableHandle mds_table;
if (OB_ISNULL(memtable_mgr)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("memtable mgr is null", K(ret));
} else if (OB_FAIL(memtable_mgr->release_memtables())) {
LOG_WARN("failed to release memtables", K(ret));
} else if (OB_FAIL(memtable_mgr->reset_storage_recorder())) {
LOG_WARN("failed to destroy storage recorder", K(ret), KPC(memtable_mgr));
} else if (OB_FAIL(get_mds_table(mds_table, false/*not_exist_create*/))) {
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to get mds table", K(ret));
}
} else if (OB_FAIL(mds_table.forcely_reset_mds_table("OFFLINE"))) {
LOG_WARN("fail to release mds nodes in mds table", K(ret));
}
return ret;
}
int ObTabletPointer::add_tablet_to_old_version_chain(ObTablet *tablet)
{
int ret = OB_SUCCESS;

1
src/storage/meta_mem/ob_tablet_pointer.h
View File

@ -69,6 +69,7 @@ public:
bool is_tablet_status_written() const;
int try_release_mds_nodes_below(const share::SCN &scn);
int try_gc_mds_table();
int release_memtable_and_mds_table_for_ls_offline();
int get_min_mds_ckpt_scn(share::SCN &scn);
ObLS *get_ls() const;
private:

30
src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp
View File

@ -1209,16 +1209,36 @@ void ObTenantMetaMemMgr::mark_mds_table_deleted_(const ObTabletMapKey &key)
if (OB_ENTRY_NOT_EXIST == ret) {
// do nothing
} else {
LOG_WARN_RET(OB_ERR_UNEXPECTED, "fail to get pointer from map", KR(ret), "resource_ptr", ptr_handle.get_resource_ptr());
LOG_WARN_RET(OB_ERR_UNEXPECTED, "fail to get pointer from map", KR(ret), K(key), "resource_ptr", ptr_handle.get_resource_ptr());
}
} else if (OB_ISNULL(ptr_handle.get_resource_ptr())) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "ptr_handle is null", KPC(ptr_handle.get_resource_ptr()));
} else {
if (OB_ISNULL(ptr_handle.get_resource_ptr())) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "ptr_handle is null", KPC(ptr_handle.get_resource_ptr()));
ObTabletPointer *tablet_pointer = static_cast<ObTabletPointer *>(ptr_handle.get_resource_ptr());
tablet_pointer->mark_mds_table_deleted();
}
}
int ObTenantMetaMemMgr::release_memtable_and_mds_table_for_ls_offline(const ObTabletMapKey &key)
{
int ret = OB_SUCCESS;
ObTabletPointerHandle ptr_handle(tablet_map_);
if (OB_FAIL(tablet_map_.get(key, ptr_handle))) {
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
ObTabletPointer *tablet_pointer = static_cast<ObTabletPointer *>(ptr_handle.get_resource_ptr());
tablet_pointer->mark_mds_table_deleted();
LOG_WARN_RET(OB_ERR_UNEXPECTED, "fail to get pointer from map", KR(ret), K(key), "resource_ptr", ptr_handle.get_resource_ptr());
}
} else if (OB_ISNULL(ptr_handle.get_resource_ptr())) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "ptr_handle is null", KPC(ptr_handle.get_resource_ptr()));
} else {
ObTabletPointer *tablet_pointer = static_cast<ObTabletPointer *>(ptr_handle.get_resource_ptr());
if (OB_FAIL(tablet_pointer->release_memtable_and_mds_table_for_ls_offline())) {
LOG_WARN("failed to release memtable and mds table", K(ret), K(key));
}
}
return ret;
}
int ObTenantMetaMemMgr::create_tmp_tablet(

1
src/storage/meta_mem/ob_tenant_meta_mem_mgr.h
View File

@ -275,6 +275,7 @@ public:
int has_meta_wait_gc(bool &is_wait);
int dump_tablet_info();
int release_memtable_and_mds_table_for_ls_offline(const ObTabletMapKey &key);
TO_STRING_KV(K_(tenant_id), K_(is_inited), "tablet count", tablet_map_.count());
private:

7
src/storage/ob_i_memtable_mgr.h
View File

@ -285,11 +285,8 @@ public:
UNUSED(log_handler);
return OB_NOT_SUPPORTED;
}
virtual int reset_storage_recorder()
{ // do nothing
return OB_NOT_SUPPORTED;
}
virtual int set_frozen_for_all_memtables() { return OB_SUCCESS; }
virtual int reset_storage_recorder() { return common::OB_SUCCESS; }
virtual int set_frozen_for_all_memtables() { return common::OB_SUCCESS; }
DECLARE_VIRTUAL_TO_STRING;
protected:
static int64_t get_memtable_idx(const int64_t pos) { return pos & (MAX_MEMSTORE_CNT - 1); }

2
src/storage/tablet/ob_i_tablet_mds_interface.h
View File

@ -41,8 +41,6 @@ public:
template <typename Key, typename Value>// general remove for multi key unit
int remove(const Key &key, mds::MdsCtx &ctx, const int64_t lock_timeout_us = 0);
// sometimes mds ndoes needed be forcely released, e.g.: ls offline
template <int N>
int forcely_reset_mds_table(const char (&reason)[N]);// reason must be compile-time str
template <typename T>
int is_locked_by_others(bool &is_locked, const mds::MdsWriter &self = mds::MdsWriter()) const;

30
src/storage/tablet/ob_i_tablet_mds_interface.ipp
View File

@ -134,32 +134,6 @@ inline int ObITabletMdsInterface::check_tablet_status_written(bool &written)
return ret;
}
template <int N>
int ObITabletMdsInterface::forcely_reset_mds_table(const char (&reason)[N])
{
#define PRINT_WRAPPER KR(ret), K(reason)
MDS_TG(10_ms);
int ret = OB_SUCCESS;
mds::MdsTableHandle handle;
if (OB_UNLIKELY(!check_is_inited_())) {
ret = OB_NOT_INIT;
MDS_LOG_GC(WARN, "not inited");
} else if (CLICK_FAIL(get_mds_table_handle_(handle, false))) {
if (OB_ENTRY_NOT_EXIST != ret) {
MDS_LOG_GC(WARN, "failed to get_mds_table");
} else {
ret = OB_SUCCESS;
}
} else if (!handle.is_valid()) {
ret = OB_ERR_UNEXPECTED;
MDS_LOG_GC(WARN, "mds cannot be NULL");
} else if (CLICK_FAIL(handle.forcely_reset_mds_table(reason))) {
MDS_LOG_GC(WARN, "fail to release mds nodes in mds table");
}
return ret;
#undef PRINT_WRAPPER
}
/**********************************IMPLEMENTATION WITH TEMPLATE************************************/
template <>
@ -173,7 +147,7 @@ inline int ObITabletMdsInterface::get_mds_data_from_tablet<ObTabletCreateDeleteM
const mds::MdsDumpKV *kv = nullptr;
ObArenaAllocator allocator("mds_reader");
const ObTabletComplexAddr<mds::MdsDumpKV> &tablet_status_addr = get_mds_data_().tablet_status_.committed_kv_;
const ObTabletCreateDeleteMdsUserData& tablet_status_cache = get_mds_data_().tablet_status_cache_;
const ObTabletCreateDeleteMdsUserData &tablet_status_cache = get_mds_data_().tablet_status_cache_;
// TODO(@chenqingxiang.cqx): remove read from IO after cache ready
if (tablet_status_cache.is_valid()) {
@ -217,7 +191,7 @@ inline int ObITabletMdsInterface::get_mds_data_from_tablet<ObTabletBindingMdsUse
const mds::MdsDumpKV *kv = nullptr;
ObArenaAllocator allocator("mds_reader");
const ObTabletComplexAddr<mds::MdsDumpKV> &aux_tablet_info_addr = get_mds_data_().aux_tablet_info_.committed_kv_;
const ObTabletBindingMdsUserData& aux_tablet_info_cache = get_mds_data_().aux_tablet_info_cache_;
const ObTabletBindingMdsUserData &aux_tablet_info_cache = get_mds_data_().aux_tablet_info_cache_;
if (aux_tablet_info_addr.is_memory_object()) {
if (CLICK_FAIL(read_op(aux_tablet_info_cache))) {

20
src/storage/tablet/ob_tablet.cpp
View File

@ -3598,26 +3598,6 @@ int ObTablet::mark_mds_table_switched_to_empty_shell_()
return ret;
}
int ObTablet::reset_storage_related_member()
{
int ret = OB_SUCCESS;
ObIMemtableMgr *memtable_mgr = nullptr;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret), K_(is_inited));
} else if (is_ls_inner_tablet()) {
// do nothing
} else if (is_empty_shell()) {
LOG_DEBUG("tablet is empty shell", K(ret));
} else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) {
LOG_WARN("failed to get memtable mgr", K(ret));
} else if (OB_FAIL(memtable_mgr->reset_storage_recorder())) {
LOG_WARN("failed to destroy storage recorder", K(ret), KPC(memtable_mgr));
}
return ret;
}
int ObTablet::get_memtable_mgr(ObIMemtableMgr *&memtable_mgr) const
{
int ret = OB_SUCCESS;

1
src/storage/tablet/ob_tablet.h
View File

@ -314,7 +314,6 @@ public:
// just for rebuild or migrate retry.
int release_memtables();
int wait_release_memtables();
int reset_storage_related_member();
// multi-source data operation
int get_storage_schema_for_transfer_in(