From 0c5c98a81dd703f80b8c882ceb7976ad057ee7e5 Mon Sep 17 00:00:00 2001 From: fengdeyiji <546976189@qq.com> Date: Mon, 14 Aug 2023 09:48:48 +0000 Subject: [PATCH] [MDS] fix read mds meet offline may get wrong data bug --- src/storage/CMakeLists.txt | 1 + src/storage/ls/ob_ls.cpp | 29 ++++++- src/storage/ls/ob_ls.h | 6 +- src/storage/ls/ob_ls_switch_checker.cpp | 55 ++++++++++++ src/storage/ls/ob_ls_switch_checker.h | 37 ++++++++ src/storage/ls/ob_ls_tablet_service.cpp | 4 +- src/storage/meta_mem/ob_tablet_pointer.cpp | 5 ++ src/storage/meta_mem/ob_tablet_pointer.h | 2 +- .../meta_mem/ob_tenant_meta_mem_mgr.cpp | 2 +- .../multi_data_source/mds_table_mgr.cpp | 2 +- .../ob_mds_table_merge_task.cpp | 2 +- .../runtime_utility/common_define.h | 2 +- .../tablet/ob_i_tablet_mds_interface.h | 2 +- .../tablet/ob_i_tablet_mds_interface.ipp | 86 +++++++++++++++---- src/storage/tablet/ob_tablet.cpp | 10 +-- .../tablet/ob_tablet_create_delete_helper.cpp | 4 +- src/storage/tablet/ob_tablet_persister.cpp | 16 ++-- src/storage/tablet/ob_tablet_slog_helper.cpp | 2 +- 18 files changed, 225 insertions(+), 42 deletions(-) create mode 100644 src/storage/ls/ob_ls_switch_checker.cpp create mode 100644 src/storage/ls/ob_ls_switch_checker.h diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 34d913bc33..8f79e7311a 100755 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -374,6 +374,7 @@ ob_set_subtarget(ob_storage ls ls/ob_ls_meta_package.cpp ls/ob_ls_role_handler.cpp ls/ob_ls_sync_tablet_seq_handler.cpp + ls/ob_ls_switch_checker.cpp ls/ob_ls_tablet_service.cpp ls/ob_ls_tx_service.cpp ls/ob_ls_saved_info.cpp diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index ad73f718fe..6f808c75ba 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -91,6 +91,7 @@ ObLS::ObLS() is_stopped_(false), is_offlined_(false), is_remove_(false), + switch_epoch_(0), ls_meta_(), rs_reporter_(nullptr), startup_transfer_info_() @@ -892,6 +893,7 @@ int ObLS::offline_(const int64_t start_ts) ret = OB_NOT_INIT; LOG_WARN("ls is not inited", K(ret)); } else if (FALSE_IT(is_offlined_ = true)) { + } else if (OB_FAIL(offline_advance_epoch_())) { } else if (FALSE_IT(checkpoint_executor_.offline())) { LOG_WARN("checkpoint executor offline failed", K(ret), K(ls_meta_)); } else if (OB_FAIL(ls_restore_handler_.offline())) { @@ -1004,6 +1006,31 @@ int ObLS::online_compaction_() return ret; } +int ObLS::offline_advance_epoch_() +{ + int ret = OB_SUCCESS; + if (ATOMIC_LOAD(&switch_epoch_) & 1) { + ATOMIC_AAF(&switch_epoch_, 1); + LOG_INFO("offline advance epoch", K(ret), K(ls_meta_), K_(switch_epoch)); + } else { + LOG_INFO("offline not advance epoch(maybe repeat call)", K(ret), K(ls_meta_), K_(switch_epoch)); + } + return ret; +} + +int ObLS::online_advance_epoch_() +{ + int ret = OB_SUCCESS; + if (ATOMIC_LOAD(&switch_epoch_) & 1) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("switch_epoch_ is odd, means online already", K(ret)); + } else { + ATOMIC_AAF(&switch_epoch_, 1); + LOG_INFO("online advance epoch", K(ret), K(ls_meta_), K_(switch_epoch)); + } + return ret; +} + int ObLS::online() { int ret = OB_SUCCESS; @@ -1036,9 +1063,9 @@ int ObLS::online() } else if (FALSE_IT(checkpoint_executor_.online())) { } else if (FALSE_IT(tablet_gc_handler_.online())) { } else if (FALSE_IT(tablet_empty_shell_handler_.online())) { + } else if (OB_FAIL(online_advance_epoch_())) { } else { is_offlined_ = false; - // do nothing } FLOG_INFO("ls online end", KR(ret), "ls_id", get_ls_id()); diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 2a042fc339..7221951871 100755 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -161,6 +161,7 @@ public: friend ObLSLockGuard; friend class ObFreezer; friend class checkpoint::ObDataCheckpoint; + friend class ObLSSwitchChecker; public: static constexpr int64_t TOTAL_INNER_TABLET_NUM = 3; static const uint64_t INNER_TABLET_ID_LIST[TOTAL_INNER_TABLET_NUM]; @@ -336,7 +337,7 @@ public: bool is_stopped() const { return is_stopped_; } int check_can_replay_clog(bool &can_replay); - TO_STRING_KV(K_(ls_meta), K_(log_handler), K_(restore_handler), K_(is_inited), K_(tablet_gc_handler), K_(startup_transfer_info)); + TO_STRING_KV(K_(ls_meta), K_(switch_epoch), K_(log_handler), K_(restore_handler), K_(is_inited), K_(tablet_gc_handler), K_(startup_transfer_info)); private: int ls_init_for_dup_table_(); int ls_destory_for_dup_table_(); @@ -349,6 +350,8 @@ private: int online_compaction_(); int offline_tx_(const int64_t start_ts); int online_tx_(); + int offline_advance_epoch_(); + int online_advance_epoch_(); public: // ObLSMeta interface: int update_ls_meta(const bool update_restore_status, @@ -894,6 +897,7 @@ private: bool is_stopped_; bool is_offlined_; bool is_remove_; + uint64_t switch_epoch_;// started from 0, odd means online, even means offline ObLSMeta ls_meta_; observer::ObIMetaReport *rs_reporter_; ObLSLock lock_; diff --git a/src/storage/ls/ob_ls_switch_checker.cpp b/src/storage/ls/ob_ls_switch_checker.cpp new file mode 100644 index 0000000000..ad0f2cb106 --- /dev/null +++ b/src/storage/ls/ob_ls_switch_checker.cpp @@ -0,0 +1,55 @@ +#include "ob_ls_switch_checker.h" +#include "lib/ob_errno.h" +#include "ob_ls.h" +#include "share/ob_errno.h" + +namespace oceanbase +{ +namespace storage +{ + +int ObLSSwitchChecker::check_online(ObLS *ls) +{ + int ret = OB_SUCCESS; + ls_ = ls; + if (OB_ISNULL(ls)) { + ret = OB_BAD_NULL_ERROR; + } else { + record_switch_epoch_ = ATOMIC_LOAD(&(ls_->switch_epoch_)); + if (!(record_switch_epoch_ & 1)) { + ret = OB_LS_OFFLINE; + } + } + return ret; +} + +int ObLSSwitchChecker::check_ls_switch_state(ObLS *ls, bool &online_state) +{ + int ret = OB_SUCCESS; + ls_ = ls; + if (OB_ISNULL(ls)) { + ret = OB_BAD_NULL_ERROR; + } else { + record_switch_epoch_ = ATOMIC_LOAD(&(ls_->switch_epoch_)); + if (!(record_switch_epoch_ & 1)) { + online_state = false; + } else { + online_state = true; + } + } + return ret; +} + +int ObLSSwitchChecker::double_check_epoch() const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ls_)) { + ret = OB_NOT_INIT; + } else if (record_switch_epoch_ != ATOMIC_LOAD(&(ls_->switch_epoch_))) { + ret = OB_VERSION_NOT_MATCH; + } + return ret; +} + +} +} \ No newline at end of file diff --git a/src/storage/ls/ob_ls_switch_checker.h b/src/storage/ls/ob_ls_switch_checker.h new file mode 100644 index 0000000000..54bd1c9e13 --- /dev/null +++ b/src/storage/ls/ob_ls_switch_checker.h @@ -0,0 +1,37 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEABASE_STORAGE_OB_LS_SWITCH_CHECKER_ +#define OCEABASE_STORAGE_OB_LS_SWITCH_CHECKER_ +#include + +namespace oceanbase +{ +namespace storage +{ +class ObLS; + +class ObLSSwitchChecker +{ +public: + ObLSSwitchChecker() : ls_(nullptr), record_switch_epoch_(UINT64_MAX) {} + int check_online(ObLS *ls); + int check_ls_switch_state(ObLS *ls, bool &online_state); + int double_check_epoch() const; +private: + ObLS *ls_; + uint64_t record_switch_epoch_; +}; + +} +} +#endif \ No newline at end of file diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 7f99a02371..021201df92 100755 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -1395,7 +1395,7 @@ int ObLSTabletService::build_new_tablet_from_mds_table( const common::ObTabletID &tablet_id, const share::SCN &flush_scn) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; common::ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "BuildMSD")); const share::ObLSID &ls_id = ls_->get_ls_id(); @@ -1864,7 +1864,7 @@ int ObLSTabletService::get_tablet_with_timeout( int ObLSTabletService::direct_get_tablet(const common::ObTabletID &tablet_id, ObTabletHandle &handle) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; const ObTabletMapKey key(ls_->get_ls_id(), tablet_id); diff --git a/src/storage/meta_mem/ob_tablet_pointer.cpp b/src/storage/meta_mem/ob_tablet_pointer.cpp index 257144a9db..c7af1c00dd 100644 --- a/src/storage/meta_mem/ob_tablet_pointer.cpp +++ b/src/storage/meta_mem/ob_tablet_pointer.cpp @@ -407,6 +407,11 @@ int ObTabletPointer::get_min_mds_ckpt_scn(share::SCN &scn) return ret; } +ObLS *ObTabletPointer::get_ls() const +{ + return ls_handle_.get_ls(); +} + int ObTabletPointer::acquire_obj(ObTablet *&t) { int ret = OB_SUCCESS; diff --git a/src/storage/meta_mem/ob_tablet_pointer.h b/src/storage/meta_mem/ob_tablet_pointer.h index 25b81dcfa5..f83f0de164 100644 --- a/src/storage/meta_mem/ob_tablet_pointer.h +++ b/src/storage/meta_mem/ob_tablet_pointer.h @@ -28,7 +28,6 @@ namespace storage { class ObTablet; class ObTabletDDLKvMgr; - typedef ObMetaObjGuard ObDDLKvMgrHandle; class ObTabletPointer : public ObMetaPointer @@ -71,6 +70,7 @@ public: int try_release_mds_nodes_below(const share::SCN &scn); int try_gc_mds_table(); int get_min_mds_ckpt_scn(share::SCN &scn); + ObLS *get_ls() const; private: int wash_obj(); int add_tablet_to_old_version_chain(ObTablet *tablet); diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp index 490b6258aa..5c0380c4bc 100755 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp @@ -1781,7 +1781,7 @@ int ObTenantMetaMemMgr::compare_and_swap_tablet( const ObTabletHandle &old_handle, ObTabletHandle &new_handle) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; const ObMetaDiskAddr &new_addr = new_handle.get_obj()->get_tablet_addr(); const ObTablet *old_tablet = old_handle.get_obj(); diff --git a/src/storage/multi_data_source/mds_table_mgr.cpp b/src/storage/multi_data_source/mds_table_mgr.cpp index 56046e8eac..24f7ae2887 100644 --- a/src/storage/multi_data_source/mds_table_mgr.cpp +++ b/src/storage/multi_data_source/mds_table_mgr.cpp @@ -247,7 +247,7 @@ int ObMdsTableMgr::flush(SCN recycle_scn, bool need_freeze) MDS_LOG_FREEZE(WARN, "fail to get max_consequent_callbacked_scn", KR(ret), K(*this)); } else if (!max_consequent_callbacked_scn.is_valid() || max_consequent_callbacked_scn.is_max() || max_consequent_callbacked_scn.is_min()) { ret = OB_ERR_UNEXPECTED; - MDS_LOG_FREEZE(ERROR, "invalid max_consequent_callbacked_scn", KR(ret), K(*this)); + MDS_LOG_FREEZE(WARN, "invalid max_consequent_callbacked_scn", KR(ret), K(*this)); } else { if (need_freeze) { if (recycle_scn.is_max() || !recycle_scn.is_valid()) { diff --git a/src/storage/multi_data_source/ob_mds_table_merge_task.cpp b/src/storage/multi_data_source/ob_mds_table_merge_task.cpp index 2f6c7b16f6..31a4c0d8db 100644 --- a/src/storage/multi_data_source/ob_mds_table_merge_task.cpp +++ b/src/storage/multi_data_source/ob_mds_table_merge_task.cpp @@ -66,7 +66,7 @@ int ObMdsTableMergeTask::init() int ObMdsTableMergeTask::process() { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; DEBUG_SYNC(AFTER_EMPTY_SHELL_TABLET_CREATE); diff --git a/src/storage/multi_data_source/runtime_utility/common_define.h b/src/storage/multi_data_source/runtime_utility/common_define.h index c6d7b13bb0..5559568802 100644 --- a/src/storage/multi_data_source/runtime_utility/common_define.h +++ b/src/storage/multi_data_source/runtime_utility/common_define.h @@ -172,7 +172,7 @@ enum LogPhase }; constexpr int64_t INVALID_VALUE = -1; -#define MDS_TG(ms) TIMEGUARD_INIT(MDS, ms, 5_s) +#define MDS_TG(ms) TIMEGUARD_INIT(MDS, ms) #define _MDS_LOG_PHASE(level, phase, info, args...) \ do {\ if (phase == mds::LogPhase::NONE) {\ diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.h b/src/storage/tablet/ob_i_tablet_mds_interface.h index 67dd7f68a8..3f8d3e5f67 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.h +++ b/src/storage/tablet/ob_i_tablet_mds_interface.h @@ -18,7 +18,7 @@ #include "storage/meta_mem/ob_tablet_pointer.h" #include "storage/tablet/ob_tablet_mds_data.h" #include "storage/tablet/ob_tablet_member_wrapper.h" -#include "storage/tablet/ob_tablet_meta.h" +#include "storage/ls/ob_ls_switch_checker.h" namespace oceanbase { diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.ipp b/src/storage/tablet/ob_i_tablet_mds_interface.ipp index f8daea687b..8c7f62966d 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.ipp +++ b/src/storage/tablet/ob_i_tablet_mds_interface.ipp @@ -1,10 +1,6 @@ #ifndef INCLUDE_OB_TABLET_MDS_PART_IPP #define INCLUDE_OB_TABLET_MDS_PART_IPP -#include "lib/ob_errno.h" #include "ob_i_tablet_mds_interface.h" -#include "storage/multi_data_source/compile_utility/compile_mapper.h" -#include "storage/multi_data_source/mds_node.h" -#include "storage/multi_data_source/mds_unit.h" #endif namespace oceanbase { @@ -286,16 +282,16 @@ int ObITabletMdsInterface::set(T &&data, mds::MdsCtx &ctx, const int64_t lock_ti MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; - if (CLICK_FAIL(get_mds_table_handle_(handle, true))) { + if (MDS_FAIL(get_mds_table_handle_(handle, true))) { MDS_LOG_SET(WARN, "failed to get_mds_table"); } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "mds cannot be NULL"); - } else if (CLICK_FAIL(handle.set(std::forward(data), ctx, lock_timeout_us))) { - MDS_LOG_SET(WARN, "failed to set dummy key unit data"); } else if (OB_ISNULL(get_tablet_ponter_())) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(handle.set(std::forward(data), ctx, lock_timeout_us))) { + MDS_LOG_SET(WARN, "failed to set dummy key unit data"); } else { get_tablet_ponter_()->set_mds_written(); MDS_LOG_SET(TRACE, "success to set dummy key unit data"); @@ -319,11 +315,11 @@ int ObITabletMdsInterface::replay(T &&data, mds::MdsCtx &ctx, const share::SCN & } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "mds cannot be NULL"); - } else if (CLICK_FAIL(handle.replay(std::forward(data), ctx, scn))) { - MDS_LOG_SET(WARN, "failed to replay dummy key unit data"); } else if (OB_ISNULL(get_tablet_ponter_())) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (CLICK_FAIL(handle.replay(std::forward(data), ctx, scn))) { + MDS_LOG_SET(WARN, "failed to replay dummy key unit data"); } else { get_tablet_ponter_()->set_mds_written(); MDS_LOG_SET(TRACE, "success to replay dummy key unit data"); @@ -345,11 +341,11 @@ int ObITabletMdsInterface::set(const Key &key, Value &&data, mds::MdsCtx &ctx, c } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "mds cannot be NULL"); - } else if (CLICK_FAIL(handle.set(key, std::forward(data), ctx, lock_timeout_us))) { - MDS_LOG_SET(WARN, "failed to set multi key unit data"); } else if (OB_ISNULL(get_tablet_ponter_())) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (CLICK_FAIL(handle.set(key, std::forward(data), ctx, lock_timeout_us))) { + MDS_LOG_SET(WARN, "failed to set multi key unit data"); } else { get_tablet_ponter_()->set_mds_written(); MDS_LOG_SET(TRACE, "success to set multi key unit data"); @@ -376,11 +372,11 @@ int ObITabletMdsInterface::replay(const Key &key, } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "mds cannot be NULL"); - } else if (CLICK_FAIL(handle.replay(key, std::forward(mds), ctx, scn))) { - MDS_LOG_SET(WARN, "failed to replay multi key unit data"); } else if (OB_ISNULL(get_tablet_ponter_())) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (CLICK_FAIL(handle.replay(key, std::forward(mds), ctx, scn))) { + MDS_LOG_SET(WARN, "failed to replay multi key unit data"); } else { get_tablet_ponter_()->set_mds_written(); MDS_LOG_SET(TRACE, "success to replay multi key unit data"); @@ -397,11 +393,15 @@ int ObITabletMdsInterface::remove(const Key &key, mds::MdsCtx &ctx, const int64_ MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; if (CLICK_FAIL(get_mds_table_handle_(handle, true))) { MDS_LOG_SET(WARN, "failed to get_mds_table"); } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_SET(WARN, "tablet pointer is null", K(ret), KPC(this)); } else if (CLICK_FAIL(handle.remove(key, ctx, lock_timeout_us))) { MDS_LOG_SET(WARN, "failed to remove multi key unit data"); } else if (OB_ISNULL(get_tablet_ponter_())) { @@ -422,16 +422,17 @@ int ObITabletMdsInterface::replay_remove(const Key &key, mds::MdsCtx &ctx, const MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; if (CLICK_FAIL(get_mds_table_handle_(handle, true))) { MDS_LOG_SET(WARN, "failed to get_mds_table"); } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "mds cannot be NULL"); - } else if (CLICK() && OB_SUCCESS != (ret = handle.replay_remove(key, ctx, scn))) { - MDS_LOG_SET(WARN, "failed to replay remove multi key unit data"); } else if (OB_ISNULL(get_tablet_ponter_())) { ret = OB_ERR_UNEXPECTED; MDS_LOG_SET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (CLICK() && OB_SUCCESS != (ret = handle.replay_remove(key, ctx, scn))) { + MDS_LOG_SET(WARN, "failed to replay remove multi key unit data"); } else { get_tablet_ponter_()->set_mds_written(); MDS_LOG_SET(TRACE, "success to remove multi key unit data"); @@ -447,6 +448,8 @@ int ObITabletMdsInterface::is_locked_by_others(bool &is_locked, const mds::MdsWr MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + bool is_online = false; if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { if (OB_ENTRY_NOT_EXIST != ret) { MDS_LOG_GET(WARN, "failed to get_mds_table"); @@ -458,6 +461,11 @@ int ObITabletMdsInterface::is_locked_by_others(bool &is_locked, const mds::MdsWr } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); } else if (CLICK_FAIL(handle.is_locked_by_others(is_locked, self))) { if (OB_SNAPSHOT_DISCARDED != ret) { MDS_LOG_GET(WARN, "failed to check lock unit data"); @@ -468,7 +476,11 @@ int ObITabletMdsInterface::is_locked_by_others(bool &is_locked, const mds::MdsWr } } if (OB_SUCC(ret)) { - MDS_LOG_GET(TRACE, "success to get is locked by others state"); + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get is locked by others state"); + } } return ret; #undef PRINT_WRAPPER @@ -481,6 +493,8 @@ int ObITabletMdsInterface::get_latest(OP &&read_op, bool &is_committed, const in MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + bool is_online = false; if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { if (OB_ENTRY_NOT_EXIST != ret) { MDS_LOG_GET(WARN, "failed to get_mds_table"); @@ -490,6 +504,11 @@ int ObITabletMdsInterface::get_latest(OP &&read_op, bool &is_committed, const in } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); } else if (CLICK_FAIL(handle.get_latest(read_op, is_committed, read_seq))) { if (OB_SNAPSHOT_DISCARDED != ret) { MDS_LOG_GET(WARN, "failed to get mds data"); @@ -508,6 +527,13 @@ int ObITabletMdsInterface::get_latest(OP &&read_op, bool &is_committed, const in } } } + if (OB_SUCC(ret)) { + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get_latest"); + } + } return ret; #undef PRINT_WRAPPER } @@ -522,6 +548,8 @@ int ObITabletMdsInterface::get_snapshot(OP &&read_op, MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + bool is_online = false; if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { if (OB_ENTRY_NOT_EXIST != ret) { MDS_LOG_GET(WARN, "failed to get_mds_table"); @@ -531,6 +559,11 @@ int ObITabletMdsInterface::get_snapshot(OP &&read_op, } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); } else if (CLICK_FAIL(handle.get_snapshot(read_op, snapshot, read_seq, timeout_us))) { if (OB_SNAPSHOT_DISCARDED != ret) { MDS_LOG_GET(WARN, "failed to get mds data"); @@ -552,6 +585,13 @@ int ObITabletMdsInterface::get_snapshot(OP &&read_op, } } } + if (OB_SUCC(ret)) { + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get_snapshot"); + } + } return ret; #undef PRINT_WRAPPER } @@ -567,6 +607,8 @@ int ObITabletMdsInterface::get_snapshot(const Key &key, MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + bool is_online = false; if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { if (OB_ENTRY_NOT_EXIST != ret) { MDS_LOG_GET(WARN, "failed to get_mds_table"); @@ -576,6 +618,11 @@ int ObITabletMdsInterface::get_snapshot(const Key &key, } else if (!handle.is_valid()) { ret = OB_ERR_UNEXPECTED; MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); } else if (CLICK() && OB_SUCCESS != (ret = handle.get_snapshot(key, read_op, snapshot, read_seq, timeout_us))) { if (OB_SNAPSHOT_DISCARDED != ret) { MDS_LOG_GET(WARN, "failed to get mds data"); @@ -597,6 +644,13 @@ int ObITabletMdsInterface::get_snapshot(const Key &key, } } } + if (OB_SUCC(ret)) { + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get_snapshot"); + } + } return ret; #undef PRINT_WRAPPER } diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index d613ea7849..c58e1e8bf5 100755 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -373,7 +373,7 @@ int ObTablet::init( const ObTabletMdsData &mds_table_data, const ObTabletMdsData &base_data) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; allocator_ = &allocator; common::ObArenaAllocator tmp_arena_allocator(common::ObMemAttr(MTL_ID(), "InitTabletMDS")); @@ -709,7 +709,7 @@ int ObTablet::init( int ObTablet::fetch_table_store(ObTabletMemberWrapper &wrapper) const { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; if (OB_UNLIKELY(!table_store_addr_.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -5106,7 +5106,7 @@ int ObTablet::build_memtable(common::ObIArray &handle_array, co int ObTablet::read_mds_table(common::ObIAllocator &allocator, ObTabletMdsData &mds_data, const bool for_flush) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; @@ -5142,7 +5142,7 @@ int ObTablet::read_mds_table_medium_info_list( common::ObIAllocator &allocator, ObTabletDumpedMediumInfo &medium_info_list) const { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; @@ -5175,7 +5175,7 @@ int ObTablet::notify_mds_table_flush_ret( const share::SCN &flush_scn, const int flush_ret) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle mds_table; const share::ObLSID &ls_id = tablet_meta_.ls_id_; diff --git a/src/storage/tablet/ob_tablet_create_delete_helper.cpp b/src/storage/tablet/ob_tablet_create_delete_helper.cpp index ebd55e6758..f9ad858560 100755 --- a/src/storage/tablet/ob_tablet_create_delete_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_delete_helper.cpp @@ -58,7 +58,7 @@ int ObTabletCreateDeleteHelper::get_tablet( ObTabletHandle &handle, const int64_t timeout_us) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; static const int64_t SLEEP_TIME_US = 10; ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); @@ -491,7 +491,7 @@ int ObTabletCreateDeleteHelper::acquire_tmp_tablet( common::ObArenaAllocator &allocator, ObTabletHandle &handle) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; ObTenantMetaMemMgr *t3m = MTL(ObTenantMetaMemMgr*); if (OB_UNLIKELY(!key.is_valid())) { diff --git a/src/storage/tablet/ob_tablet_persister.cpp b/src/storage/tablet/ob_tablet_persister.cpp index 676121078a..426f48fd71 100644 --- a/src/storage/tablet/ob_tablet_persister.cpp +++ b/src/storage/tablet/ob_tablet_persister.cpp @@ -90,7 +90,7 @@ int ObTabletPersister::persist_and_transform_tablet( const ObTablet &old_tablet, ObTabletHandle &new_handle) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; common::ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "PesistTranf")); common::ObSEArray tablet_meta_write_ctxs; @@ -119,7 +119,7 @@ int ObTabletPersister::recursively_persist( common::ObIArray &sstable_meta_write_ctxs, ObTabletHandle &new_handle) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; if (CLICK_FAIL(persist_and_fill_tablet( old_tablet, allocator, tablet_meta_write_ctxs, sstable_meta_write_ctxs, new_handle))) { @@ -143,7 +143,7 @@ int ObTabletPersister::convert_tablet_to_mem_arg( ObTabletMemberWrapper &auto_inc_seq, ObTabletTransformArg &arg) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; arg.reset(); if (OB_UNLIKELY(!tablet.is_valid())) { @@ -185,7 +185,7 @@ int ObTabletPersister::convert_tablet_to_disk_arg( ObTabletMemberWrapper &auto_inc_seq, ObTabletTransformArg &arg) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; arg.reset(); @@ -264,7 +264,7 @@ int ObTabletPersister::persist_and_fill_tablet( common::ObIArray &sstable_meta_write_ctxs, ObTabletHandle &new_handle) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; common::ObSEArray write_infos; ObTabletTransformArg arg; @@ -299,7 +299,7 @@ int ObTabletPersister::check_tablet_meta_ids( const common::ObIArray &tablet_meta_write_ctxs, const ObTablet &tablet) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; ObSArray meta_ids; ObSArray ctx_ids; @@ -363,7 +363,7 @@ int ObTabletPersister::acquire_tablet( int ObTabletPersister::persist_4k_tablet(common::ObArenaAllocator &allocator, ObTabletHandle &new_handle) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; ObTablet *new_tablet = new_handle.get_obj(); ObTenantCheckpointSlogHandler *ckpt_slog_hanlder = MTL(ObTenantCheckpointSlogHandler*); @@ -424,7 +424,7 @@ int ObTabletPersister::transform( char *buf, const int64_t len) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; ObTablet *tiny_tablet = reinterpret_cast(buf); ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), "TmpPullMemTbl")); diff --git a/src/storage/tablet/ob_tablet_slog_helper.cpp b/src/storage/tablet/ob_tablet_slog_helper.cpp index e1932b245a..462ea03b37 100755 --- a/src/storage/tablet/ob_tablet_slog_helper.cpp +++ b/src/storage/tablet/ob_tablet_slog_helper.cpp @@ -35,7 +35,7 @@ int ObTabletSlogHelper::write_update_tablet_slog( const common::ObTabletID &tablet_id, const ObMetaDiskAddr &disk_addr) { - TIMEGUARD_INIT(STORAGE, 10_ms, 5_s); + TIMEGUARD_INIT(STORAGE, 10_ms); int ret = OB_SUCCESS; const ObTabletMapKey tablet_key(ls_id, tablet_id); if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || !disk_addr.is_valid())) {