hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Tablet will not hold memtable ref when ls offline

Browse Source
This commit is contained in:
godyangfight 2023-10-12 02:10:01 +00:00 committed by ob-robot
parent 7d90783c71
commit fc85a979ab
4 changed files with 100 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
mittest/mtlenv/storage/test_ls_tablet_service.cpp
View File

@ -259,7 +259,7 @@ TEST_F(TestLSTabletService, test_create_tablet_without_index)
ASSERT_EQ(1 + inner_tablet_count, MTL(ObTenantMetaMemMgr*)->tablet_map_.map_.size());
ObTabletMapKey key(ls_id_, tablet_id);
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -323,7 +323,7 @@ TEST_F(TestLSTabletService, test_serialize_tablet)
ob_free(buf);
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -558,9 +558,9 @@ TEST_F(TestLSTabletService, test_create_tablet_with_index)
ObTabletMapKey key;
key.ls_id_ = ls_id_;
key.tablet_id_ = data_tablet_id;
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
key.tablet_id_ = index_tablet_id;
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -608,9 +608,9 @@ TEST_F(TestLSTabletService, test_create_index_tablet)
ObTabletMapKey key;
key.ls_id_ = ls_id_;
key.tablet_id_ = data_tablet_id;
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
key.tablet_id_ = index_tablet_id;
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -657,7 +657,7 @@ TEST_F(TestLSTabletService, test_get_ls_min_end_scn)
tablet_handle.get_obj()->tablet_meta_.clog_checkpoint_scn_ = orig_scn; // set orig_scn to del tablet
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -688,7 +688,7 @@ TEST_F(TestLSTabletService, test_replay_empty_shell)
ASSERT_EQ(nullptr, wrapper.get_member()->get_major_sstables().get_boundary_table(true));
ObTabletMapKey key(ls_id_, tablet_id);
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
ObLogCursor replay_start_cursor_;
@ -716,7 +716,7 @@ TEST_F(TestLSTabletService, test_replay_empty_shell)
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_EQ(true, empty_shell_tablet->is_empty_shell());
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -769,7 +769,7 @@ TEST_F(TestLSTabletService, test_cover_empty_shell)
ASSERT_EQ(ObTabletStatus::NORMAL, user_data.tablet_status_);
ObTabletMapKey key(ls_id_, tablet_id);
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -808,7 +808,7 @@ TEST_F(TestLSTabletService, test_migrate_empty_shell)
ASSERT_EQ(pos, tablet_meta.get_serialize_size());
ObTabletMapKey key(ls_id_, tablet_id);
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
ObMigrationTabletParam new_tablet_meta;
@ -827,7 +827,7 @@ TEST_F(TestLSTabletService, test_migrate_empty_shell)
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_EQ(true, empty_shell_tablet->is_empty_shell());
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -918,7 +918,7 @@ TEST_F(TestLSTabletService, test_migrate_param)
*/
ObTabletMapKey key(ls_id_, tablet_id);
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -976,7 +976,7 @@ TEST_F(TestLSTabletService, test_migrate_param_empty_shell)
*/
ObTabletMapKey key(ls_id_, tablet_id);
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -1002,11 +1002,11 @@ TEST_F(TestLSTabletService, test_update_empty_shell)
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_TRUE(empty_shell_tablet->is_empty_shell());
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}
TEST_F(TestLSTabletService, test_update_tablet_release_memtable)
TEST_F(TestLSTabletService, update_tablet_release_memtable_for_offline)
{
int ret = OB_SUCCESS;
const int64_t inner_tablet_count = INNER_TABLET_CNT;
@ -1033,15 +1033,13 @@ TEST_F(TestLSTabletService, test_update_tablet_release_memtable)
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->create_memtable(data_tablet_id, 100));
ASSERT_EQ(1, tablet_handle.get_obj()->memtable_count_);
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable(data_tablet_id, SCN::max_scn()));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable(data_tablet_id, SCN::max_scn()));
MTL(ObTenantCheckpointSlogHandler *)->shared_block_rwriter_.macro_handle_.reset();
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable_for_offline(data_tablet_id, SCN::max_scn()));
ASSERT_EQ(OB_TABLET_NOT_EXIST, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable_for_offline(data_tablet_id, SCN::max_scn()));
ObTabletMapKey key;
key.ls_id_ = ls_id_;
key.tablet_id_ = data_tablet_id;
ret = MTL(ObTenantMetaMemMgr*)->del_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(key);
ASSERT_EQ(OB_SUCCESS, ret);
}

106
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -172,11 +172,11 @@ int ObLSTabletService::offline()
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret), K_(is_inited));
} else if (OB_FAIL(offline_build_tablet_without_memtable_())) {
LOG_WARN("failed to offline build tablet without memtable", K(ret));
} else if (OB_FAIL(offline_destroy_memtable_and_mds_table_())) {
LOG_WARN("failed to offline destroy memtable and mds table", K(ret));
} else {
DestroyMemtableAndMemberAndMdsTableOperator clean_mem_op(this);
if (OB_FAIL(tablet_id_set_.foreach(clean_mem_op))) {
LOG_WARN("fail to clean memtables", K(ret), "cur_tablet_id", clean_mem_op.cur_tablet_id_);
}
mds_table_mgr_.offline();
}
return ret;
@ -1435,13 +1435,15 @@ int ObLSTabletService::build_new_tablet_from_mds_table(
return ret;
}
int ObLSTabletService::update_tablet_release_memtable(
int ObLSTabletService::update_tablet_release_memtable_for_offline(
const common::ObTabletID &tablet_id,
const SCN scn)
{
int ret = OB_SUCCESS;
const ObTabletMapKey key(ls_->get_ls_id(), tablet_id);
const ObLSID ls_id(ls_->get_ls_id());
const ObTabletMapKey key(ls_id, tablet_id);
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
ObTimeGuard time_guard("ObLSTabletService::update_tablet_release_memtable", 1_s);
ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash());
time_guard.click("Lock");
@ -1453,37 +1455,49 @@ int ObLSTabletService::update_tablet_release_memtable(
LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(scn));
} else if (OB_FAIL(ObTabletCreateDeleteHelper::get_tablet(key, tablet_handle))) {
LOG_WARN("fail to direct get tablet", K(ret), K(key));
} else if (OB_ISNULL(tablet = tablet_handle.get_obj())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet should not be NULL", K(ret), K(key));
} else if (tablet->is_empty_shell()) {
//do nothing
} else {
time_guard.click("get_tablet");
ObTablet *tablet = tablet_handle.get_obj();
ObTabletHandle new_tablet_handle;
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
const bool need_persist = nullptr != tablet_handle.get_allocator();
ObMetaDiskAddr disk_addr;
if (!need_persist) {
ObITable *table = nullptr;
ObTableStoreIterator iter;
const bool is_from_buf_pool = nullptr == tablet_handle.get_obj()->get_allocator();
if (is_from_buf_pool) {
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
ObTabletHandle new_tablet_handle;
if (OB_FAIL(ObTabletPersister::copy_from_old_tablet(*tablet, new_tablet_handle))) {
LOG_WARN("fail to copy from old tablet", K(ret), KPC(tablet));
} else if (FALSE_IT(time_guard.click("CpTablet"))) {
} else if (OB_FAIL(new_tablet_handle.get_obj()->rebuild_memtables(scn))) {
LOG_WARN("fail to rebuild memtables", K(ret), K(scn), K(new_tablet_handle));
} else if (OB_FAIL(t3m->compare_and_swap_tablet(key, tablet_handle, new_tablet_handle))) {
LOG_ERROR("failed to compare and swap tablet", K(ret), K(key), K(tablet_handle), K(new_tablet_handle));
} else {
time_guard.click("CpTablet");
time_guard.click("CASwap");
LOG_INFO("succeeded to copy tablet to release memtable", K(ret), K(key), K(tablet_handle), K(new_tablet_handle));
}
} else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(*tablet, new_tablet_handle))) {
LOG_WARN("fail to persist and transform tablet", K(ret), KPC(tablet), K(new_tablet_handle));
} else if (FALSE_IT(time_guard.click("Persist"))) {
} else if (FALSE_IT(disk_addr = new_tablet_handle.get_obj()->tablet_addr_)) {
} else if (OB_FAIL(ObTabletSlogHelper::write_update_tablet_slog(key.ls_id_, tablet_id, disk_addr))) {
LOG_WARN("failed to write update tablet slog", K(ret), K(key), K(disk_addr));
} else if (OB_UNLIKELY(!tablet->get_tablet_addr().is_memory())) {
ret = OB_NOT_SUPPORTED;
LOG_ERROR("This tablet is full tablet, but its addr isn't memory", K(ret), KPC(tablet));
} else if (OB_FAIL(tablet->get_all_sstables(iter))) {
LOG_WARN("fail to get all sstable", K(ret), K(iter));
} else if (1 == iter.count() && OB_FAIL(iter.get_next(table))) {
LOG_WARN("fail to get next table", K(ret), K(iter));
} else if (OB_UNLIKELY(iter.count() > 1)
|| (OB_NOT_NULL(table) && (!table->is_sstable()
|| static_cast<ObSSTable *>(table)->get_data_macro_block_count() != 0))) {
ret = OB_NOT_SUPPORTED;
LOG_ERROR("This tablet is full tablet, but all of its sstables isn't only one empty major",
K(ret), K(iter), KPC(table));
} else if (OB_FAIL(tablet_handle.get_obj()->wait_release_memtables())) {
LOG_ERROR("failed to release memtables", K(ret), K(tablet_id));
} else if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) {
LOG_ERROR("failed to do remove tablet", K(ret), K(ls_id), K(tablet_id));
} else {
time_guard.click("WrSlog");
}
if (FAILEDx(new_tablet_handle.get_obj()->rebuild_memtables(scn))) {
LOG_WARN("fail to rebuild memtables", K(ret), K(scn), K(new_tablet_handle));
} else if (OB_FAIL(t3m->compare_and_swap_tablet(key, tablet_handle, new_tablet_handle))) {
LOG_ERROR("failed to compare and swap tablet", K(ret), K(key), K(tablet_handle), K(new_tablet_handle));
ob_usleep(1000 * 1000);
ob_abort();
} else {
time_guard.click("CASwap");
LOG_INFO("succeeded to build new tablet", K(ret), K(key), K(disk_addr), K(tablet_handle));
time_guard.click("RmTablet");
}
}
return ret;
@ -6266,5 +6280,37 @@ int ObLSTabletService::check_parts_tx_state_in_transfer_for_4377_(transaction::O
return ret;
}
int ObLSTabletService::offline_build_tablet_without_memtable_()
{
int ret = OB_SUCCESS;
ObArray<ObTabletID> tablet_id_array;
const bool except_ls_inner_tablet = false;
const SCN scn(SCN::max_scn());
if (OB_FAIL(get_all_tablet_ids(except_ls_inner_tablet, tablet_id_array))) {
LOG_WARN("failed to get all tablet ids", K(ret));
} else {
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_id_array.count(); ++i) {
const ObTabletID &tablet_id = tablet_id_array.at(i);
if (OB_FAIL(update_tablet_release_memtable_for_offline(tablet_id, scn))) {
LOG_WARN("failed to update tablet release memtable", K(ret), K(tablet_id));
}
}
}
return ret;
}
int ObLSTabletService::offline_destroy_memtable_and_mds_table_()
{
int ret = OB_SUCCESS;
DestroyMemtableAndMemberAndMdsTableOperator clean_mem_op(this);
if (OB_FAIL(tablet_id_set_.foreach(clean_mem_op))) {
LOG_WARN("fail to clean memtables", K(ret), "cur_tablet_id", clean_mem_op.cur_tablet_id_);
}
return ret;
}
} // namespace storage
} // namespace oceanbase

4
src/storage/ls/ob_ls_tablet_service.h
View File

@ -242,7 +242,7 @@ public:
const common::ObTabletID &tablet_id,
const int64_t mds_construct_sequence,
const share::SCN &flush_scn);
int update_tablet_release_memtable(
int update_tablet_release_memtable_for_offline(
const common::ObTabletID &tablet_id,
const SCN scn);
int update_tablet_report_status(const common::ObTabletID &tablet_id);
@ -504,6 +504,8 @@ private:
const ObMigrationTabletParam &mig_tablet_param,
ObTabletHandle &handle);
int delete_all_tablets();
int offline_build_tablet_without_memtable_();
int offline_destroy_memtable_and_mds_table_();
private:
static int check_real_leader_for_4377_(const ObLSID ls_id);
static int check_need_rollback_in_transfer_for_4377_(const transaction::ObTxDesc *tx_desc,

2
src/storage/tablet/ob_tablet_persister.cpp
View File

@ -139,6 +139,8 @@ int ObTabletPersister::persist_and_transform_tablet(
LOG_WARN("fail to acqurie tablet", K(ret), K(type), K(new_handle));
} else if (OB_FAIL(transform_tablet_memory_footprint(old_tablet, new_handle.get_buf(), new_handle.get_buf_len()))) {
LOG_WARN("fail to transform tablet memory footprint", K(ret), K(old_tablet), K(type));
} else if (OB_FAIL(new_handle.get_obj()->inc_macro_ref_cnt())) {
LOG_WARN("fail to increase macro ref cnt for new tablet", K(ret), K(new_handle));
}
}
return ret;