hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix race condition for ls offline tablet remove and empty shell task

Browse Source
This commit is contained in:
hiddenbomb
2024-02-20 13:15:12 +00:00
committed by ob-robot
parent 178c2fb423
commit a9d18eadb8
3 changed files with 37 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
mittest/mtlenv/storage/test_ls_tablet_service.cpp
View File

@ -260,8 +260,7 @@ TEST_F(TestLSTabletService, test_create_tablet_without_index)
valid_tablet_num(inner_tablet_count);
ASSERT_EQ(1 + inner_tablet_count, MTL(ObTenantMetaMemMgr*)->tablet_map_.map_.size());
ObTabletMapKey key(ls_id_, tablet_id);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -325,7 +324,7 @@ TEST_F(TestLSTabletService, test_serialize_tablet)
ob_free(buf);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -557,12 +556,9 @@ TEST_F(TestLSTabletService, test_create_tablet_with_index)
valid_tablet_num(inner_tablet_count);
ASSERT_EQ(2 + INNER_TABLET_CNT, MTL(ObTenantMetaMemMgr*)->tablet_map_.map_.size());
ObTabletMapKey key;
key.ls_id_ = ls_id_;
key.tablet_id_ = data_tablet_id;
ret = ls_tablet_service_->do_remove_tablet(key);
key.tablet_id_ = index_tablet_id;
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, data_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, index_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -607,12 +603,9 @@ TEST_F(TestLSTabletService, test_create_index_tablet)
valid_tablet_num(inner_tablet_count);
ASSERT_EQ(2 + INNER_TABLET_CNT, MTL(ObTenantMetaMemMgr*)->tablet_map_.map_.size());
ObTabletMapKey key;
key.ls_id_ = ls_id_;
key.tablet_id_ = data_tablet_id;
ret = ls_tablet_service_->do_remove_tablet(key);
key.tablet_id_ = index_tablet_id;
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, data_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, index_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -659,7 +652,7 @@ TEST_F(TestLSTabletService, test_get_ls_min_end_scn)
tablet_handle.get_obj()->tablet_meta_.clog_checkpoint_scn_ = orig_scn; // set orig_scn to del tablet
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -718,8 +711,7 @@ TEST_F(TestLSTabletService, test_replay_empty_shell)
ASSERT_EQ(nullptr, wrapper.get_member()->get_major_sstables().get_boundary_table(true));
ASSERT_EQ(tablet_addr, test_tablet_handle.get_obj()->tablet_addr_);
ObTabletMapKey key(ls_id_, tablet_id);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ObLogCursor replay_start_cursor_;
@ -747,7 +739,7 @@ TEST_F(TestLSTabletService, test_replay_empty_shell)
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_EQ(true, empty_shell_tablet->is_empty_shell());
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -799,8 +791,7 @@ TEST_F(TestLSTabletService, test_cover_empty_shell)
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->get_tablet_status(share::SCN::max_scn(), user_data));
ASSERT_EQ(ObTabletStatus::NORMAL, user_data.tablet_status_);
ObTabletMapKey key(ls_id_, tablet_id);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -838,8 +829,7 @@ TEST_F(TestLSTabletService, test_migrate_empty_shell)
ASSERT_EQ(OB_SUCCESS, tablet_meta.serialize(buf, 4096, pos));
ASSERT_EQ(pos, tablet_meta.get_serialize_size());
ObTabletMapKey key(ls_id_, tablet_id);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ObMigrationTabletParam new_tablet_meta;
@ -858,7 +848,7 @@ TEST_F(TestLSTabletService, test_migrate_empty_shell)
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_EQ(true, empty_shell_tablet->is_empty_shell());
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -948,8 +938,7 @@ TEST_F(TestLSTabletService, test_migrate_param)
ASSERT_EQ(OB_SUCCESS, tablet_meta.build_deleted_tablet_info(ls_id_, tablet_id));
*/
ObTabletMapKey key(ls_id_, tablet_id);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -1006,8 +995,7 @@ TEST_F(TestLSTabletService, test_migrate_param_empty_shell)
ASSERT_EQ(OB_SUCCESS, tablet_meta.build_deleted_tablet_info(ls_id_, tablet_id));
*/
ObTabletMapKey key(ls_id_, tablet_id);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -1026,14 +1014,13 @@ TEST_F(TestLSTabletService, test_update_empty_shell)
ASSERT_EQ(OB_SUCCESS, ret);
ret = ls_tablet_service_->update_tablet_to_empty_shell(tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
ObTabletMapKey key(ls_id_, tablet_id);
ObTabletHandle tablet_handle;
ret = ls_tablet_service_->get_tablet(tablet_id, tablet_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK);
ASSERT_EQ(OB_SUCCESS, ret);
ObTablet *empty_shell_tablet = tablet_handle.get_obj();
ASSERT_TRUE(empty_shell_tablet->is_empty_shell());
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -1067,10 +1054,7 @@ TEST_F(TestLSTabletService, update_tablet_release_memtable_for_offline)
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable_for_offline(data_tablet_id, SCN::max_scn()));
ASSERT_EQ(OB_TABLET_NOT_EXIST, ls_handle.get_ls()->get_tablet_svr()->update_tablet_release_memtable_for_offline(data_tablet_id, SCN::max_scn()));
ObTabletMapKey key;
key.ls_id_ = ls_id_;
key.tablet_id_ = data_tablet_id;
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, data_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}
@ -1116,7 +1100,7 @@ TEST_F(TestLSTabletService, update_tablet_ddl_commit_scn)
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet_svr()->get_tablet(data_tablet_id, tablet_handle));
ASSERT_EQ(ddl_commit_scn, tablet_handle.get_obj()->tablet_meta_.ddl_commit_scn_);
ret = ls_tablet_service_->do_remove_tablet(key);
ret = ls_tablet_service_->do_remove_tablet(ls_id_, data_tablet_id);
ASSERT_EQ(OB_SUCCESS, ret);
}

27
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -496,14 +496,16 @@ int ObLSTabletService::remove_tablets(const common::ObIArray<common::ObTabletID>
return ret;
}
int ObLSTabletService::do_remove_tablet(const ObTabletMapKey &key)
int ObLSTabletService::do_remove_tablet(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
ObTimeGuard time_guard("RmTabletLock", 1_s);
ObBucketHashWLockGuard lock_guard(bucket_lock_, key.tablet_id_.hash());
ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash());
time_guard.click("Lock");
if (OB_FAIL(inner_remove_tablet(key.ls_id_, key.tablet_id_))) {
LOG_WARN("fail to remove tablet with lock", K(ret), K(key));
if (OB_FAIL(inner_remove_tablet(ls_id, tablet_id))) {
LOG_WARN("fail to remove tablet with lock", K(ret), K(ls_id), K(tablet_id));
}
return ret;
}
@ -2147,8 +2149,8 @@ int ObLSTabletService::create_transfer_in_tablet(
LOG_WARN("old tablet is not empty shell", K(ret), K(old_tablet_hdl));
} else if (FALSE_IT(cover_empty_shell = true)) {
// Called in mds trans callback, slog won't be written.
} else if (OB_FAIL(do_remove_tablet(key))) {
LOG_WARN("failed to remove tablet", K(ret), K(key));
} else if (OB_FAIL(do_remove_tablet(ls_id, tablet_id))) {
LOG_WARN("failed to remove tablet", K(ret), K(ls_id), K(tablet_id));
}
time_guard.click("Prepare");
@ -6067,7 +6069,7 @@ int ObLSTabletService::remove_ls_inner_tablet(
} else if (OB_UNLIKELY(ls_id != ls_->get_ls_id())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("ls id is unexpected", K(ret), "arg_ls_id", ls_id, "ls_id", ls_->get_ls_id());
} else if (OB_FAIL(do_remove_tablet(ObTabletMapKey(ls_id, tablet_id)))) {
} else if (OB_FAIL(do_remove_tablet(ls_id, tablet_id))) {
LOG_WARN("failed to remove tablet", K(ret), K(ls_id), K(tablet_id));
}
@ -6613,7 +6615,8 @@ int ObLSTabletService::check_tablet_no_active_memtable(const ObIArray<ObTabletID
int ObLSTabletService::offline_gc_tablet_for_create_or_transfer_in_abort_()
{
int ret = OB_SUCCESS;
LOG_INFO("start offline_gc_tablet_for_create_or_transfer_in_abort", K(ls_->get_ls_id()));
const share::ObLSID &ls_id = ls_->get_ls_id();
LOG_INFO("start offline_gc_tablet_for_create_or_transfer_in_abort", K(ret), K(ls_id));
ObTabletIDArray deleted_tablets;
ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK);
bool tablet_status_is_written = false;
@ -6663,11 +6666,11 @@ int ObLSTabletService::offline_gc_tablet_for_create_or_transfer_in_abort_()
// gc deleted_tablets
if (OB_SUCC(ret)) {
for (int64_t i = 0; OB_SUCC(ret) && i < deleted_tablets.count(); ++i) {
const ObTabletID tablet_id = deleted_tablets.at(i);
if (OB_FAIL(inner_remove_tablet(ls_->get_ls_id(), tablet_id))) {
LOG_WARN("failed to remove tablet", K(ret), K(tablet_id));
const common::ObTabletID &tablet_id = deleted_tablets.at(i);
if (OB_FAIL(do_remove_tablet(ls_id, tablet_id))) {
LOG_WARN("failed to remove tablet", K(ret), K(ls_id), K(tablet_id));
} else {
LOG_INFO("gc tablet finish", K(ret), K(tablet_id));
LOG_INFO("gc tablet finish", K(ret), K(ls_id), K(tablet_id));
}
}
}

4
src/storage/ls/ob_ls_tablet_service.h
View File

@ -281,7 +281,6 @@ public:
const int64_t buf_len,
const ObTabletID &tablet_id,
ObTabletTransferInfo &tablet_transfer_info);
int do_remove_tablet(const ObTabletMapKey &key);
int create_memtable(
const common::ObTabletID &tablet_id,
@ -542,6 +541,9 @@ private:
const common::ObTabletID &tablet_id,
const ObUpdateTabletPointerParam &param,
ObTabletHandle &tablet_handle);
int do_remove_tablet(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id);
int inner_remove_tablet(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id);