From c6ded9ad28071caec08258b11bdd3b54686b5649 Mon Sep 17 00:00:00 2001 From: hiddenbomb Date: Tue, 14 Feb 2023 03:12:24 +0000 Subject: [PATCH] fix incorrect operation when force killing tx --- .../test_tablet_create_delete_helper.cpp | 120 +++++++++++++++++- .../tablet/ob_tablet_create_delete_helper.cpp | 18 ++- 2 files changed, 134 insertions(+), 4 deletions(-) diff --git a/mittest/mtlenv/storage/test_tablet_create_delete_helper.cpp b/mittest/mtlenv/storage/test_tablet_create_delete_helper.cpp index 1d8f2d1d9..c7d8fa1d0 100644 --- a/mittest/mtlenv/storage/test_tablet_create_delete_helper.cpp +++ b/mittest/mtlenv/storage/test_tablet_create_delete_helper.cpp @@ -2912,13 +2912,131 @@ TEST_F(TestTabletCreateDeleteHelper, tablet_not_exist_commit) ASSERT_EQ(OB_SUCCESS, ret); } } + +TEST_F(TestTabletCreateDeleteHelper, force_kill_create_tablet_tx) +{ + int ret = OB_SUCCESS; + + ObMulSourceDataNotifyArg trans_flags; + trans_flags.for_replay_ = false; + trans_flags.redo_submitted_ = false; + trans_flags.redo_synced_ = false; + trans_flags.is_force_kill_ = false; + trans_flags.tx_id_ = 1; + trans_flags.scn_ = share::SCN::invalid_scn(); + + ObSArray tablet_id_array; + tablet_id_array.push_back(ObTabletID(100)); + ObBatchCreateTabletArg arg; + ret = TestTabletCreateDeleteHelper::build_create_pure_data_tablet_arg( + ls_id_, tablet_id_array, arg); + ASSERT_EQ(OB_SUCCESS, ret); + + ObLSHandle ls_handle; + ObLSService *ls_svr = MTL(ObLSService*); + ret = ls_svr->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD); + ASSERT_EQ(OB_SUCCESS, ret); + ObLS *ls = ls_handle.get_ls(); + ObLSTabletService &ls_tablet_service = ls->ls_tablet_svr_; + + ret = ls_tablet_service.on_prepare_create_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + + trans_flags.redo_submitted_ = true; + trans_flags.redo_synced_ = true; + trans_flags.scn_ = share::SCN::minus(share::SCN::max_scn(), 100); + ret = ls_tablet_service.on_redo_create_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + + // mock transaction force killed + trans_flags.is_force_kill_ = true; + trans_flags.scn_.set_invalid(); + ret = ls_tablet_service.on_abort_create_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); +} + +TEST_F(TestTabletCreateDeleteHelper, force_kill_remove_tablet_tx) +{ + int ret = OB_SUCCESS; + + ObMulSourceDataNotifyArg trans_flags; + trans_flags.for_replay_ = false; + trans_flags.redo_submitted_ = false; + trans_flags.redo_synced_ = false; + trans_flags.is_force_kill_ = false; + trans_flags.tx_id_ = 1; + trans_flags.scn_ = share::SCN::invalid_scn(); + + ObLSHandle ls_handle; + ObLSService *ls_svr = MTL(ObLSService*); + ret = ls_svr->get_ls(ls_id_, ls_handle, ObLSGetMod::STORAGE_MOD); + ASSERT_EQ(OB_SUCCESS, ret); + ObLS *ls = ls_handle.get_ls(); + ObLSTabletService &ls_tablet_service = ls->ls_tablet_svr_; + + { + ObSArray tablet_id_array; + tablet_id_array.push_back(ObTabletID(100)); + ObBatchCreateTabletArg arg; + ret = TestTabletCreateDeleteHelper::build_create_pure_data_tablet_arg( + ls_id_, tablet_id_array, arg); + ASSERT_EQ(OB_SUCCESS, ret); + + ret = ls_tablet_service.on_prepare_create_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + + trans_flags.redo_submitted_ = true; + trans_flags.redo_synced_ = true; + trans_flags.scn_ = share::SCN::minus(share::SCN::max_scn(), 100); + ret = ls_tablet_service.on_redo_create_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + + trans_flags.scn_ = share::SCN::plus(trans_flags.scn_, 1); + ret = ls_tablet_service.on_tx_end_create_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + ret = ls_tablet_service.on_commit_create_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + } + + // remove tablets + { + ObSArray tablet_id_array; + tablet_id_array.push_back(ObTabletID(100)); + + ObBatchRemoveTabletArg arg; + ret = TestTabletCreateDeleteHelper::build_remove_tablet_arg( + ls_id_, tablet_id_array, arg); + ASSERT_EQ(OB_SUCCESS, ret); + + trans_flags.for_replay_ = false; + trans_flags.redo_submitted_ = false; + trans_flags.redo_synced_ = false; + trans_flags.tx_id_ = 2; + trans_flags.scn_ = share::SCN::invalid_scn(); + + ret = ls_tablet_service.on_prepare_remove_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + + trans_flags.redo_submitted_ = true; + trans_flags.redo_synced_ = true; + trans_flags.scn_ = share::SCN::minus(share::SCN::max_scn(), 90); + ret = ls_tablet_service.on_redo_remove_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + + // mock transaction force killed + trans_flags.is_force_kill_ = true; + trans_flags.scn_.set_invalid(); + ret = ls_tablet_service.on_abort_remove_tablets(arg, trans_flags); + ASSERT_EQ(OB_SUCCESS, ret); + } +} } // namespace storage } // namespace oceanbase int main(int argc, char **argv) { system("rm -f test_tablet_create_delete_helper.log*"); - OB_LOGGER.set_file_name("test_tablet_create_delete_helper.log"); + OB_LOGGER.set_file_name("test_tablet_create_delete_helper.log", true); OB_LOGGER.set_log_level("INFO"); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/storage/tablet/ob_tablet_create_delete_helper.cpp b/src/storage/tablet/ob_tablet_create_delete_helper.cpp index 3f51c2dce..f8d2c91a8 100644 --- a/src/storage/tablet/ob_tablet_create_delete_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_delete_helper.cpp @@ -913,6 +913,7 @@ int ObTabletCreateDeleteHelper::do_abort_create_tablet( // replaying procedure, clog ts is smaller than tx log ts, just skip LOG_INFO("skip abort create tablet", K(ret), K(tablet_id), K(trans_flags), K(tx_data)); } else if (OB_UNLIKELY(!trans_flags.for_replay_ + && trans_flags.scn_.is_valid() && tx_data.tx_scn_ != SCN::max_scn() && trans_flags.scn_ <= tx_data.tx_scn_)) { // If tx log ts equals SCN::max_scn(), it means redo callback has not been called. @@ -926,9 +927,20 @@ int ObTabletCreateDeleteHelper::do_abort_create_tablet( ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet status is not CREATING", K(ret), K(tablet_id), K(trans_flags), K(tx_data)); } else { + share::SCN tx_scn; + share::SCN memtable_scn; + if (trans_flags.scn_.is_valid()) { + tx_scn = trans_flags.scn_; + memtable_scn = trans_flags.scn_; + } else { + // trans flags scn is invalid, maybe tx is force killed + tx_scn = tx_data.tx_scn_; + memtable_scn = tx_data.tx_scn_; + } + if (OB_FAIL(set_tablet_final_status(tablet_handle, ObTabletStatus::DELETED, - trans_flags.scn_, trans_flags.scn_, trans_flags.for_replay_))) { - LOG_WARN("failed to set tablet status to DELETED", K(ret), K(tablet_handle), K(trans_flags)); + tx_scn, memtable_scn, trans_flags.for_replay_))) { + LOG_WARN("failed to set tablet status to DELETED", K(ret), K(key), K(tablet_handle), K(trans_flags), K(tx_scn), K(memtable_scn)); } else if (OB_FAIL(t3m->erase_pinned_tablet(key))) { LOG_ERROR("failed to erase tablet handle", K(ret), K(key)); ob_usleep(1000 * 1000); @@ -1286,7 +1298,7 @@ int ObTabletCreateDeleteHelper::do_abort_remove_tablet( tx_scn = tx_data.tx_scn_; } - const SCN memtable_scn = (SCN::invalid_scn() == trans_flags.scn_) ? SCN::max_scn() : trans_flags.scn_; + const SCN &memtable_scn = (SCN::invalid_scn() == trans_flags.scn_) ? SCN::max_scn() : trans_flags.scn_; if (OB_FAIL(set_tablet_final_status(tablet_handle, ObTabletStatus::NORMAL, tx_scn, memtable_scn, trans_flags.for_replay_, ref_op))) {