hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fix restore transfer in tablet mistake rollback bug.

Browse Source
This commit is contained in:
godyangfight 2024-03-22 08:46:43 +00:00 committed by ob-robot
parent d8e0974d8f
commit 0b9d356814
5 changed files with 96 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -2252,18 +2252,24 @@ int ObLSTabletService::create_empty_shell_tablet(
int ObLSTabletService::rollback_remove_tablet(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id)
const common::ObTabletID &tablet_id,
const share::SCN &transfer_start_scn)
{
int ret = OB_SUCCESS;
bool is_same = true;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ls tablet service do not init", K(ret));
} else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) {
} else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || !transfer_start_scn.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id));
LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id), K(transfer_start_scn));
} else {
ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash());
if (OB_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) {
if (OB_FAIL(check_rollback_tablet_is_same_(ls_id, tablet_id, transfer_start_scn, is_same))) {
LOG_WARN("failed to check rollback tablet is same", K(ret), K(ls_id), K(tablet_id), K(transfer_start_scn));
} else if (!is_same) {
//do nothing
} else if (OB_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) {
LOG_WARN("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id));
}
}
@ -6731,5 +6737,33 @@ int ObLSTabletService::offline_gc_tablet_for_create_or_transfer_in_abort_()
return ret;
}
int ObLSTabletService::check_rollback_tablet_is_same_(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const share::SCN &transfer_start_scn,
bool &is_same)
{
int ret = OB_SUCCESS;
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
is_same = true;
if (OB_FAIL(direct_get_tablet(tablet_id, tablet_handle))) {
if (OB_TABLET_NOT_EXIST == ret) {
is_same = true;
ret = OB_SUCCESS;
}
} else if (OB_ISNULL(tablet = tablet_handle.get_obj())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet should not be NULL", K(ret), K(ls_id), K(tablet_id));
} else if (transfer_start_scn != tablet->get_tablet_meta().transfer_info_.transfer_start_scn_) {
is_same = false;
LOG_ERROR("rollback tablet is not same, cannot rollback", K(ls_id), K(tablet_id), K(transfer_start_scn), KPC(tablet));
} else {
is_same = true;
}
return ret;
}
} // namespace storage
} // namespace oceanbase

9
src/storage/ls/ob_ls_tablet_service.h
View File

@ -196,7 +196,8 @@ public:
ObTabletHandle &tablet_handle);
int rollback_remove_tablet(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id);
const common::ObTabletID &tablet_id,
const share::SCN &transfer_start_scn);
int get_tablet(
const common::ObTabletID &tablet_id,
@ -782,7 +783,11 @@ private:
int create_empty_shell_tablet(
const ObMigrationTabletParam &param,
ObTabletHandle &tablet_handle);
int check_rollback_tablet_is_same_(
const share::ObLSID &ls_id,
const common::ObTabletID &tablet_id,
const share::SCN &transfer_start_scn,
bool &is_same);
private:
int direct_insert_rows(const uint64_t table_id,
const int64_t task_id,

15
src/storage/tablet/ob_tablet_create_mds_helper.cpp
View File

@ -40,6 +40,9 @@ namespace oceanbase
{
namespace storage
{
ERRSIM_POINT_DEF(EN_CREATE_TABLET_FAILED);
int ObTabletCreateMdsHelper::on_commit_for_old_mds(
const char* buf,
const int64_t len,
@ -373,6 +376,15 @@ int ObTabletCreateMdsHelper::create_tablets(
}
}
#ifdef ERRSIM
if (OB_SUCC(ret)) {
ret = EN_CREATE_TABLET_FAILED ? : OB_SUCCESS;
if (OB_FAIL(ret)) {
LOG_WARN("inject EN_CREATE_TABLET_FAILED", K(ret));
}
}
#endif
return ret;
}
@ -977,6 +989,7 @@ int ObTabletCreateMdsHelper::rollback_remove_tablets(
ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*);
ObLSHandle ls_handle;
ObLS *ls = nullptr;
const share::SCN transfer_start_scn(share::SCN::min_scn());
if (CLICK_FAIL(get_ls(ls_id, ls_handle))) {
LOG_WARN("failed to get ls", K(ret), K(ls_id));
@ -987,7 +1000,7 @@ int ObTabletCreateMdsHelper::rollback_remove_tablets(
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_id_array.count(); ++i) {
MDS_TG(10_ms);
const common::ObTabletID &tablet_id = tablet_id_array.at(i);
if (CLICK_FAIL(ls->get_tablet_svr()->rollback_remove_tablet(ls_id, tablet_id))) {
if (CLICK_FAIL(ls->get_tablet_svr()->rollback_remove_tablet(ls_id, tablet_id, transfer_start_scn))) {
LOG_WARN("failed to rollback remove tablet", K(ret), K(ls_id), K(tablet_id));
}
}

46
src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp
View File

@ -39,6 +39,8 @@ namespace storage
{
using namespace oceanbase::transaction;
ERRSIM_POINT_DEF(EN_CREATE_TRANSFER_IN_TABLET_FAILED);
/******************ObTabletStartTransferOutReplayExecutor*********************/
class ObTabletStartTransferOutReplayExecutor final : public logservice::ObTabletReplayExecutor
{
@ -1300,7 +1302,7 @@ int ObTabletStartTransferInHelper::create_transfer_in_tablets_(
// roll back operation
if (OB_FAIL(ret)) {
int tmp_ret = OB_SUCCESS;
if (CLICK() && OB_TMP_FAIL(rollback_transfer_in_tablets_(tablet_id_array, dest_ls))) {
if (CLICK() && OB_TMP_FAIL(rollback_transfer_in_tablets_(tx_start_transfer_in_info, tablet_id_array, dest_ls))) {
LOG_WARN("failed to roll back remove tablets", K(tmp_ret),
K(tx_start_transfer_in_info), K(lbt()));
ob_usleep(1000 * 1000);
@ -1408,20 +1410,45 @@ int ObTabletStartTransferInHelper::inner_create_transfer_in_tablet_(
UNUSED(scn);
UNUSED(for_replay);
int ret = OB_SUCCESS;
if (OB_FAIL(dest_ls->get_tablet_svr()->create_transfer_in_tablet(dest_ls->get_ls_id(), tablet_meta, tablet_handle))) {
if (OB_FAIL(ret)) {
} else if (OB_FAIL(dest_ls->get_tablet_svr()->create_transfer_in_tablet(dest_ls->get_ls_id(), tablet_meta, tablet_handle))) {
LOG_WARN("failed to create transfer in tablet", K(ret), K(tablet_meta), KPC(dest_ls));
}
#ifdef ERRSIM
if (OB_SUCC(ret)) {
ret = EN_CREATE_TRANSFER_IN_TABLET_FAILED ? : OB_SUCCESS;
if (OB_FAIL(ret)) {
LOG_WARN("inject EN_CREATE_TRANSFER_IN_TABLET_FAILED", K(ret));
}
}
#endif
return ret;
}
int ObTabletStartTransferInHelper::rollback_transfer_in_tablets_(
const ObTXStartTransferInInfo &tx_start_transfer_in_info,
const common::ObIArray<common::ObTabletID> &tablet_id_array,
ObLS *dest_ls)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; OB_SUCC(ret) && i < tablet_id_array.count(); ++i) {
const ObTabletID &tablet_id = tablet_id_array.at(i);
if (OB_FAIL(rollback_transfer_in_tablet_(tablet_id, dest_ls))) {
bool found = false;
share::SCN transfer_start_scn;
for (int64_t j = 0; OB_SUCC(ret) && j < tx_start_transfer_in_info.tablet_meta_list_.count() && !found; ++j) {
const ObTabletID &tmp_tablet_id = tx_start_transfer_in_info.tablet_meta_list_.at(j).tablet_id_;
if (tmp_tablet_id == tablet_id) {
transfer_start_scn = tx_start_transfer_in_info.tablet_meta_list_.at(j).transfer_info_.transfer_start_scn_;
found = true;
}
}
if (OB_FAIL(ret)) {
} else if (!found) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("rollback transfer in tablet do not exist, unexpected", K(ret), K(tablet_id), K(tx_start_transfer_in_info));
} else if (OB_FAIL(rollback_transfer_in_tablet_(tablet_id, transfer_start_scn, dest_ls))) {
LOG_WARN("failed to rollback transfer in tablet", K(ret), K(tablet_id_array));
}
}
@ -1430,6 +1457,7 @@ int ObTabletStartTransferInHelper::rollback_transfer_in_tablets_(
int ObTabletStartTransferInHelper::rollback_transfer_in_tablet_(
const common::ObTabletID &tablet_id,
const share::SCN &transfer_start_scn,
ObLS *dest_ls)
{
int ret = OB_SUCCESS;
@ -1445,8 +1473,8 @@ int ObTabletStartTransferInHelper::rollback_transfer_in_tablet_(
} else {
LOG_WARN("failed to get tablet", K(ret), KPC(dest_ls), K(tablet_id));
}
} else if (OB_FAIL(dest_ls->get_tablet_svr()->rollback_remove_tablet(dest_ls->get_ls_id(), tablet_id))) {
LOG_WARN("failed to rollback remove tablet", K(ret), K(dest_ls), K(tablet_id));
} else if (OB_FAIL(dest_ls->get_tablet_svr()->rollback_remove_tablet(dest_ls->get_ls_id(), tablet_id, transfer_start_scn))) {
LOG_WARN("failed to rollback remove tablet", K(ret), K(dest_ls), K(tablet_id), K(transfer_start_scn));
}
return ret;
}
@ -1754,12 +1782,8 @@ int ObTabletStartTransferInHelper::check_transfer_dest_tablet_ready_(
} else {
const ObTabletMapKey key(tablet_meta.ls_id_, tablet_meta.tablet_id_);
if (OB_FAIL(ObTabletCreateDeleteHelper::get_tablet(key, tablet_handle))) {
if (OB_TABLET_NOT_EXIST == ret) {
can_skip = false;
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to get tablet", K(ret), K(tablet_meta));
}
can_skip = false;
LOG_WARN("failed to get tablet", K(ret), K(tablet_meta));
} else if (OB_ISNULL(tablet = tablet_handle.get_obj())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet should not be NULL", K(ret), K(tablet_handle), KP(tablet));

2
src/storage/tablet/ob_tablet_start_transfer_mds_helper.h
View File

@ -188,10 +188,12 @@ private:
mds::BufferCtx &ctx,
common::ObIArray<common::ObTabletID> &tablet_id_array);
static int rollback_transfer_in_tablets_(
const ObTXStartTransferInInfo &tx_start_transfer_in_info,
const common::ObIArray<common::ObTabletID> &tablet_id_array,
ObLS *dest_ls);
static int rollback_transfer_in_tablet_(
const common::ObTabletID &tablet_id,
const share::SCN &transfer_start_scn,
ObLS *dest_ls);
static int inner_create_transfer_in_tablet_(