diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index d8c78d48d..06a4ce1f0 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -927,13 +927,8 @@ int ObLSTabletService::migrate_create_tablet( handle = tablet_handle; } else { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(tablet_id_set_.erase(tablet_id))) { - if (OB_HASH_NOT_EXIST != tmp_ret) { - LOG_ERROR("fail to erase tablet id from set", K(tmp_ret), K(tablet_id)); - } - } - if (OB_TMP_FAIL(inner_remove_tablet(key.ls_id_, key.tablet_id_))) { - LOG_WARN("fail to erase tablet from meta memory manager", K(tmp_ret), K(key)); + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); } } @@ -1885,13 +1880,8 @@ int ObLSTabletService::replay_create_tablet( FLOG_INFO("succeeded to replay create one tablet", K(ret), K(ls_id), K(tablet_id), KPC(tablet)); } else { int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(tablet_id_set_.erase(tablet_id))) { - if (OB_HASH_NOT_EXIST != tmp_ret) { - LOG_ERROR("fail to erase tablet id from set", K(tmp_ret), K(tablet_id)); - } - } - if (OB_TMP_FAIL(inner_remove_tablet(key.ls_id_, key.tablet_id_))) { - LOG_WARN("fail to erase tablet from meta memory manager", K(tmp_ret), K(key)); + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); } } } @@ -2055,6 +2045,15 @@ int ObLSTabletService::create_tablet( } else { report_tablet_to_rs(tablet_id); } + + if (OB_SUCC(ret)) { + LOG_INFO("succeed to create tablet", K(ret), K(ls_id), K(tablet_id)); + } else { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); + } + } } return ret; @@ -2090,12 +2089,6 @@ int ObLSTabletService::create_inner_tablet( create_scn, snapshot_version, create_tablet_schema, freezer))) { LOG_WARN("failed to init tablet", K(ret), K(ls_id), K(tablet_id), K(data_tablet_id), K(create_scn), K(snapshot_version), K(create_tablet_schema)); - int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(inner_remove_tablet(ls_id, tablet_id))) { - LOG_ERROR("failed to delete tablet from t3m", K(ret), K(key), K(lbt())); - ob_usleep(1000 * 1000); - ob_abort(); - } } else if (FALSE_IT(time_guard.click("InitTablet"))) { } else if (OB_FAIL(ObTabletPersister::persist_and_transform_tablet(*tmp_tablet, tablet_handle))) { LOG_WARN("fail to persist and transform tablet", K(ret), K(tmp_tablet_hdl), K(tablet_handle)); @@ -2103,8 +2096,15 @@ int ObLSTabletService::create_inner_tablet( } else if (FALSE_IT(disk_addr = tablet_handle.get_obj()->get_tablet_addr())) { } else if (OB_FAIL(safe_create_cas_tablet(ls_id, tablet_id, disk_addr, tablet_handle, time_guard))) { LOG_WARN("fail to refresh tablet", K(ret), K(ls_id), K(tablet_id), K(disk_addr), K(tablet_handle)); + } + + if (OB_SUCC(ret)) { + LOG_INFO("create ls inner tablet success", K(ret), K(key), K(disk_addr)); } else { - LOG_INFO("create inner tablet success", K(key), K(disk_addr)); + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); + } } return ret; } @@ -2116,7 +2116,8 @@ int ObLSTabletService::create_transfer_in_tablet( { int ret = OB_SUCCESS; ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); - const ObTabletMapKey key(ls_id, tablet_meta.tablet_id_); + const common::ObTabletID &tablet_id = tablet_meta.tablet_id_; + const ObTabletMapKey key(ls_id, tablet_id); ObTablet *tablet = nullptr; ObFreezer *freezer = ls_->get_freezer(); common::ObArenaAllocator *allocator = nullptr; @@ -2132,12 +2133,12 @@ int ObLSTabletService::create_transfer_in_tablet( } else if (!ls_id.is_valid() || !tablet_meta.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("create transfer in tablet get invalid argument", K(ret), K(ls_id), K(tablet_meta)); - } else if (OB_FAIL(direct_get_tablet(tablet_meta.tablet_id_, old_tablet_hdl))) { + } else if (OB_FAIL(direct_get_tablet(tablet_id, old_tablet_hdl))) { if (OB_TABLET_NOT_EXIST == ret) { ret = OB_SUCCESS; - LOG_DEBUG("tablet not exist", K(ret), K(tablet_meta.tablet_id_)); + LOG_DEBUG("tablet not exist", K(ret), K(tablet_id)); } else { - LOG_WARN("fail to get tablet", K(ret), K(tablet_meta.tablet_id_)); + LOG_WARN("fail to get tablet", K(ret), K(tablet_id)); } } else if (!old_tablet_hdl.get_obj()->is_empty_shell()) { ret = OB_ERR_UNEXPECTED; @@ -2153,7 +2154,7 @@ int ObLSTabletService::create_transfer_in_tablet( LOG_WARN("failed to prepare create msd tablet", K(ret)); } else { ObUpdateTabletPointerParam param; - ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_meta.tablet_id_.hash()); + ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); time_guard.click("Lock"); if (FAILEDx(ObTabletCreateDeleteHelper::create_msd_tablet(key, tablet_handle))) { LOG_WARN("failed to create msd tablet", K(ret), K(key)); @@ -2167,12 +2168,20 @@ int ObLSTabletService::create_transfer_in_tablet( LOG_WARN("fail to get updating tablet pointer parameters", K(ret), KPC(tablet)); } else if (OB_FAIL(t3m->compare_and_swap_tablet(key, tablet_handle, tablet_handle, param))) { LOG_WARN("failed to compare and swap tablet", K(ret), K(key), K(tablet_handle), K(param)); - } else if (OB_FAIL(tablet_id_set_.set(tablet_meta.tablet_id_))) { - LOG_WARN("fail to insert tablet id", K(ret), K(ls_id), K(tablet_meta)); + } else if (OB_FAIL(tablet_id_set_.set(tablet_id))) { + LOG_WARN("fail to insert tablet id", K(ret), K(ls_id), K(tablet_id)); } else { - LOG_INFO("create transfer in tablet", K(ls_id), K(key), K(tablet_meta), K(cover_empty_shell)); time_guard.click("Swap"); } + + if (OB_SUCC(ret)) { + LOG_INFO("create transfer in tablet", K(ret), K(key), K(tablet_meta), K(cover_empty_shell)); + } else { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { + LOG_ERROR("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); + } + } } return ret; @@ -2237,35 +2246,56 @@ int ObLSTabletService::create_empty_shell_tablet( return ret; } -int ObLSTabletService::rollback_tablet( +int ObLSTabletService::rollback_remove_tablet( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id) { int ret = OB_SUCCESS; - ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); - ObLSHandle ls_handle; - ObLS *ls = nullptr; - ObTabletMapKey key(ls_id, tablet_id); if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ls tablet service do not init", K(ret)); } else if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("rollback tablet get invalid argument", K(ret), K(ls_id), K(tablet_id)); - } else if (OB_FAIL(do_remove_tablet(key))) { - LOG_WARN("failed to delete tablet", K(ret), K(key)); - } else if (OB_FAIL(tablet_id_set_.erase(key.tablet_id_))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - LOG_DEBUG("tablet id does not exist, maybe has not been inserted yet", K(ret), K(key)); - } else { - //the caller checks abort if is required. - LOG_WARN("failed to erase tablet id", K(ret), K(key)); + LOG_WARN("invalid args", K(ret), K(ls_id), K(tablet_id)); + } else { + ObBucketHashWLockGuard lock_guard(bucket_lock_, tablet_id.hash()); + if (OB_FAIL(rollback_remove_tablet_without_lock(ls_id, tablet_id))) { + LOG_WARN("fail to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); } } return ret; } +int ObLSTabletService::rollback_remove_tablet_without_lock( + const share::ObLSID &ls_id, + const common::ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); + const ObTabletMapKey key(ls_id, tablet_id); + + if (OB_FAIL(tablet_id_set_.erase(tablet_id))) { + if (OB_HASH_NOT_EXIST == ret) { + // tablet id is already erased + ret = OB_SUCCESS; + LOG_DEBUG("tablet id does not exist, maybe has not been inserted yet", K(ret), K(ls_id), K(tablet_id)); + } else { + LOG_WARN("fail to erase tablet id from set", K(ret), K(ls_id), K(tablet_id)); + } + } + + if (OB_SUCC(ret)) { + // loop retry to delete tablet from t3m + while (OB_FAIL(t3m->del_tablet(key))) { + if (REACH_TIME_INTERVAL(10_s)) { + LOG_ERROR("failed to delete tablet from t3m", K(ret), K(ls_id), K(tablet_id)); + } + } + } + + return ret; +} + int ObLSTabletService::create_memtable( const common::ObTabletID &tablet_id, const int64_t schema_version, diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index cd485a8f0..dd7dbd3b6 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -193,7 +193,7 @@ public: const share::ObLSID &ls_id, const ObMigrationTabletParam &tablet_meta, ObTabletHandle &tablet_handle); - int rollback_tablet( + int rollback_remove_tablet( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); @@ -546,6 +546,9 @@ private: int inner_remove_tablet( const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); + int rollback_remove_tablet_without_lock( + const share::ObLSID &ls_id, + const common::ObTabletID &tablet_id); int rollback_rebuild_tablet(const ObTabletID &tablet_id); int trim_old_tablets(const ObTabletID &tablet_id); int rebuild_tablet_with_old( diff --git a/src/storage/meta_mem/ob_meta_obj_struct.h b/src/storage/meta_mem/ob_meta_obj_struct.h index 6b91bd15e..40d17f67e 100644 --- a/src/storage/meta_mem/ob_meta_obj_struct.h +++ b/src/storage/meta_mem/ob_meta_obj_struct.h @@ -245,7 +245,6 @@ void ObMetaObjGuard::set_obj(ObMetaObj &obj) obj_pool_ = obj.pool_; allocator_ = obj.allocator_; t3m_ = obj.t3m_; - hold_start_time_ = ObClockGenerator::getClock(); } obj_ = obj.ptr_; obj_->inc_ref(); @@ -280,6 +279,7 @@ void ObMetaObjGuard::reset() reset_obj(); obj_pool_ = nullptr; allocator_ = nullptr; + hold_start_time_ = INT64_MAX; t3m_ = nullptr; } diff --git a/src/storage/tablet/ob_tablet_create_mds_helper.cpp b/src/storage/tablet/ob_tablet_create_mds_helper.cpp index 3357811b3..1fb73b09e 100644 --- a/src/storage/tablet/ob_tablet_create_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_mds_helper.cpp @@ -73,7 +73,7 @@ int ObTabletCreateMdsHelper::register_process( if (OB_FAIL(ret)) { // roll back int tmp_ret = OB_SUCCESS; - if (CLICK_TMP_FAIL(roll_back_remove_tablets(arg.id_, tablet_id_array))) { + if (CLICK_TMP_FAIL(rollback_remove_tablets(arg.id_, tablet_id_array))) { LOG_ERROR("failed to roll back remove tablets", K(tmp_ret)); ob_usleep(1 * 1000 * 1000); ob_abort(); @@ -150,7 +150,7 @@ int ObTabletCreateMdsHelper::replay_process( if (CLICK_FAIL(ret)) { // roll back int tmp_ret = OB_SUCCESS; - if (CLICK() && OB_TMP_FAIL(roll_back_remove_tablets(arg.id_, tablet_id_array))) { + if (CLICK() && OB_TMP_FAIL(rollback_remove_tablets(arg.id_, tablet_id_array))) { LOG_ERROR("failed to roll back remove tablets", K(tmp_ret)); ob_usleep(1 * 1000 * 1000); ob_abort(); @@ -950,7 +950,7 @@ int ObTabletCreateMdsHelper::build_bind_hidden_tablets( return ret; } -int ObTabletCreateMdsHelper::roll_back_remove_tablets( +int ObTabletCreateMdsHelper::rollback_remove_tablets( const share::ObLSID &ls_id, const common::ObIArray &tablet_id_array) { @@ -959,8 +959,6 @@ int ObTabletCreateMdsHelper::roll_back_remove_tablets( ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); ObLSHandle ls_handle; ObLS *ls = nullptr; - ObTabletMapKey key; - key.ls_id_ = ls_id; if (CLICK_FAIL(get_ls(ls_id, ls_handle))) { LOG_WARN("failed to get ls", K(ret), K(ls_id)); @@ -968,20 +966,11 @@ int ObTabletCreateMdsHelper::roll_back_remove_tablets( ret = OB_ERR_UNEXPECTED; LOG_WARN("ls is null", K(ret), K(ls_id), K(ls_handle)); } else { - ObTabletIDSet &tablet_id_set = ls->get_tablet_svr()->tablet_id_set_; - for (int64_t i = 0; OB_SUCC(ret) && i < tablet_id_array.count(); ++i) { MDS_TG(10_ms); - key.tablet_id_ = tablet_id_array.at(i); - if (CLICK_FAIL(ls->get_tablet_svr()->do_remove_tablet(key))) { - LOG_WARN("failed to delete tablet", K(ret), K(key)); - } else if (CLICK_FAIL(tablet_id_set.erase(key.tablet_id_))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - LOG_DEBUG("tablet id does not exist, maybe has not been inserted yet", K(ret), K(key)); - } else { - LOG_WARN("failed to erase tablet id", K(ret), K(key)); - } + const common::ObTabletID &tablet_id = tablet_id_array.at(i); + if (CLICK_FAIL(ls->get_tablet_svr()->rollback_remove_tablet(ls_id, tablet_id))) { + LOG_WARN("failed to rollback remove tablet", K(ret), K(ls_id), K(tablet_id)); } } } diff --git a/src/storage/tablet/ob_tablet_create_mds_helper.h b/src/storage/tablet/ob_tablet_create_mds_helper.h index 5786085b1..df01d3179 100644 --- a/src/storage/tablet/ob_tablet_create_mds_helper.h +++ b/src/storage/tablet/ob_tablet_create_mds_helper.h @@ -141,7 +141,7 @@ private: const share::SCN &scn, mds::BufferCtx &ctx, common::ObIArray &tablet_id_array); - static int roll_back_remove_tablets( + static int rollback_remove_tablets( const share::ObLSID &ls_id, const common::ObIArray &tablet_id_array); static int get_ls( diff --git a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp index e12524267..2581b277e 100644 --- a/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp +++ b/src/storage/tablet/ob_tablet_start_transfer_mds_helper.cpp @@ -1445,8 +1445,8 @@ int ObTabletStartTransferInHelper::rollback_transfer_in_tablet_( } else { LOG_WARN("failed to get tablet", K(ret), KPC(dest_ls), K(tablet_id)); } - } else if (OB_FAIL(dest_ls->get_tablet_svr()->rollback_tablet(dest_ls->get_ls_id(), tablet_id))) { - LOG_WARN("failed to rollback tablet", K(ret), K(dest_ls), K(tablet_id)); + } else if (OB_FAIL(dest_ls->get_tablet_svr()->rollback_remove_tablet(dest_ls->get_ls_id(), tablet_id))) { + LOG_WARN("failed to rollback remove tablet", K(ret), K(dest_ls), K(tablet_id)); } return ret; }