From f1d3f3e76f0fdfbfc1a89ad58009e7d8087f1b90 Mon Sep 17 00:00:00 2001 From: fengdeyiji <546976189@qq.com> Date: Mon, 4 Dec 2023 09:16:24 +0000 Subject: [PATCH] [MDS] if read mds data meet ls state changed: 1. retry if still online. 2.OB_LS_OFFLINE if offline. --- src/storage/ls/ob_ls_switch_checker.cpp | 13 +- src/storage/ls/ob_ls_switch_checker.h | 4 +- .../tablet/ob_i_tablet_mds_interface.ipp | 319 ++++++++++-------- 3 files changed, 179 insertions(+), 157 deletions(-) diff --git a/src/storage/ls/ob_ls_switch_checker.cpp b/src/storage/ls/ob_ls_switch_checker.cpp index 7e4258d179..b53e9d78f2 100644 --- a/src/storage/ls/ob_ls_switch_checker.cpp +++ b/src/storage/ls/ob_ls_switch_checker.cpp @@ -35,7 +35,7 @@ int ObLSSwitchChecker::check_online(ObLS *ls) return ret; } -int ObLSSwitchChecker::check_ls_switch_state(ObLS *ls, bool &online_state) +int ObLSSwitchChecker::check_ls_switch_state(ObLS *ls, bool &is_online) { int ret = OB_SUCCESS; ls_ = ls; @@ -44,21 +44,24 @@ int ObLSSwitchChecker::check_ls_switch_state(ObLS *ls, bool &online_state) } else { record_switch_epoch_ = ATOMIC_LOAD(&(ls_->switch_epoch_)); if (!(record_switch_epoch_ & 1)) { - online_state = false; + is_online = false; } else { - online_state = true; + is_online = true; } } return ret; } -int ObLSSwitchChecker::double_check_epoch() const +int ObLSSwitchChecker::double_check_epoch(bool &is_online) const { int ret = OB_SUCCESS; + int64_t switch_state = 0; if (OB_ISNULL(ls_)) { ret = OB_NOT_INIT; - } else if (record_switch_epoch_ != ATOMIC_LOAD(&(ls_->switch_epoch_))) { + } else if (FALSE_IT(switch_state = ATOMIC_LOAD(&(ls_->switch_epoch_)))) { + } else if (OB_UNLIKELY(record_switch_epoch_ != switch_state)) { ret = OB_VERSION_NOT_MATCH; + is_online = (switch_state & 1) ? false : true; } return ret; } diff --git a/src/storage/ls/ob_ls_switch_checker.h b/src/storage/ls/ob_ls_switch_checker.h index 54bd1c9e13..0ef998b34a 100644 --- a/src/storage/ls/ob_ls_switch_checker.h +++ b/src/storage/ls/ob_ls_switch_checker.h @@ -25,8 +25,8 @@ class ObLSSwitchChecker public: ObLSSwitchChecker() : ls_(nullptr), record_switch_epoch_(UINT64_MAX) {} int check_online(ObLS *ls); - int check_ls_switch_state(ObLS *ls, bool &online_state); - int double_check_epoch() const; + int check_ls_switch_state(ObLS *ls, bool &is_online); + int double_check_epoch(bool &is_online) const; private: ObLS *ls_; uint64_t record_switch_epoch_; diff --git a/src/storage/tablet/ob_i_tablet_mds_interface.ipp b/src/storage/tablet/ob_i_tablet_mds_interface.ipp index d0b4cf8952..b233d98e75 100644 --- a/src/storage/tablet/ob_i_tablet_mds_interface.ipp +++ b/src/storage/tablet/ob_i_tablet_mds_interface.ipp @@ -1,6 +1,7 @@ #ifndef INCLUDE_OB_TABLET_MDS_PART_IPP #define INCLUDE_OB_TABLET_MDS_PART_IPP #include "ob_i_tablet_mds_interface.h" +#include "share/ob_errno.h" #endif namespace oceanbase { @@ -353,7 +354,6 @@ int ObITabletMdsInterface::remove(const Key &key, mds::MdsCtx &ctx, const int64_ MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; - ObLSSwitchChecker ls_switch_checker; if (CLICK_FAIL(get_mds_table_handle_(handle, true))) { MDS_LOG_SET(WARN, "failed to get_mds_table"); } else if (!handle.is_valid()) { @@ -382,7 +382,6 @@ int ObITabletMdsInterface::replay_remove(const Key &key, mds::MdsCtx &ctx, const MDS_TG(10_ms); int ret = OB_SUCCESS; mds::MdsTableHandle handle; - ObLSSwitchChecker ls_switch_checker; if (CLICK_FAIL(get_mds_table_handle_(handle, true))) { MDS_LOG_SET(WARN, "failed to get_mds_table"); } else if (!handle.is_valid()) { @@ -407,41 +406,46 @@ int ObITabletMdsInterface::is_locked_by_others(bool &is_locked, const mds::MdsWr #define PRINT_WRAPPER KR(ret), K(*this), K(is_locked), K(self) MDS_TG(10_ms); int ret = OB_SUCCESS; - mds::MdsTableHandle handle; - ObLSSwitchChecker ls_switch_checker; bool is_online = false; - if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { - if (OB_ENTRY_NOT_EXIST != ret) { - MDS_LOG_GET(WARN, "failed to get_mds_table"); - } else { - MDS_LOG_GET(TRACE, "failed to get_mds_table"); - ret = OB_SUCCESS; - is_locked = false; + do { + mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { + if (OB_ENTRY_NOT_EXIST != ret) { + MDS_LOG_GET(WARN, "failed to get_mds_table"); + } else { + MDS_LOG_GET(TRACE, "failed to get_mds_table"); + ret = OB_SUCCESS; + is_locked = false; + } + } else if (!handle.is_valid()) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); + } else if (CLICK_FAIL(handle.is_locked_by_others(is_locked, self))) { + if (OB_SNAPSHOT_DISCARDED != ret) { + MDS_LOG_GET(WARN, "failed to check lock unit data"); + } else { + MDS_LOG_GET(TRACE, "failed to check lock unit data"); + ret = OB_SUCCESS; + is_locked = false; + } } - } else if (!handle.is_valid()) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "mds cannot be NULL"); - } else if (OB_ISNULL(get_tablet_ponter_())) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); - } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { - MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); - } else if (CLICK_FAIL(handle.is_locked_by_others(is_locked, self))) { - if (OB_SNAPSHOT_DISCARDED != ret) { - MDS_LOG_GET(WARN, "failed to check lock unit data"); - } else { - MDS_LOG_GET(TRACE, "failed to check lock unit data"); - ret = OB_SUCCESS; - is_locked = false; + if (OB_SUCC(ret)) { + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch(is_online))) { + if (!is_online) { + ret = OB_LS_OFFLINE; + } + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get is locked by others state"); + } } - } - if (OB_SUCC(ret)) { - if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { - MDS_LOG_GET(WARN, "failed to double check ls online"); - } else { - MDS_LOG_GET(TRACE, "success to get is locked by others state"); - } - } + } while (ret == OB_VERSION_NOT_MATCH && is_online); return ret; #undef PRINT_WRAPPER } @@ -452,48 +456,53 @@ int ObITabletMdsInterface::get_latest(OP &&read_op, bool &is_committed, const in #define PRINT_WRAPPER KR(ret), K(*this), K(read_seq), K(typeid(OP).name()) MDS_TG(10_ms); int ret = OB_SUCCESS; - mds::MdsTableHandle handle; - ObLSSwitchChecker ls_switch_checker; bool is_online = false; - if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { - if (OB_ENTRY_NOT_EXIST != ret) { - MDS_LOG_GET(WARN, "failed to get_mds_table"); - } else { - MDS_LOG_GET(TRACE, "failed to get_mds_table"); - } - } else if (!handle.is_valid()) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "mds cannot be NULL"); - } else if (OB_ISNULL(get_tablet_ponter_())) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); - } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { - MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); - } else if (CLICK_FAIL(handle.get_latest(read_op, is_committed, read_seq))) { - if (OB_SNAPSHOT_DISCARDED != ret) { - MDS_LOG_GET(WARN, "failed to get mds data"); - } else { - MDS_LOG_GET(TRACE, "failed to get mds data"); - } - } - if (CLICK_FAIL(ret)) { - if (OB_ENTRY_NOT_EXIST == ret || OB_SNAPSHOT_DISCARDED == ret) { - auto func = [&read_op, &is_committed](const T& data) -> int { - is_committed = true;// FIXME: here need more judge after support dump uncommitted node - return read_op(data); - }; - if (CLICK_FAIL(get_mds_data_from_tablet(func))) { - MDS_LOG_GET(WARN, "failed to get latest data from tablet"); + do { + mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { + if (OB_ENTRY_NOT_EXIST != ret) { + MDS_LOG_GET(WARN, "failed to get_mds_table"); + } else { + MDS_LOG_GET(TRACE, "failed to get_mds_table"); + } + } else if (!handle.is_valid()) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); + } else if (CLICK_FAIL(handle.get_latest(read_op, is_committed, read_seq))) { + if (OB_SNAPSHOT_DISCARDED != ret) { + MDS_LOG_GET(WARN, "failed to get mds data"); + } else { + MDS_LOG_GET(TRACE, "failed to get mds data"); } } - } - if (OB_SUCC(ret)) { - if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { - MDS_LOG_GET(WARN, "failed to double check ls online"); - } else { - MDS_LOG_GET(TRACE, "success to get_latest"); + if (CLICK_FAIL(ret)) { + if (OB_ENTRY_NOT_EXIST == ret || OB_SNAPSHOT_DISCARDED == ret) { + auto func = [&read_op, &is_committed](const T& data) -> int { + is_committed = true;// FIXME: here need more judge after support dump uncommitted node + return read_op(data); + }; + if (CLICK_FAIL(get_mds_data_from_tablet(func))) { + MDS_LOG_GET(WARN, "failed to get latest data from tablet"); + } + } } - } + if (OB_SUCC(ret)) { + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch(is_online))) { + if (!is_online) { + ret = OB_LS_OFFLINE; + } + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get_latest"); + } + } + } while (ret == OB_VERSION_NOT_MATCH && is_online); return ret; #undef PRINT_WRAPPER } @@ -507,51 +516,56 @@ int ObITabletMdsInterface::get_snapshot(OP &&read_op, #define PRINT_WRAPPER KR(ret), K(*this), K(snapshot), K(read_seq), K(typeid(OP).name()) MDS_TG(10_ms); int ret = OB_SUCCESS; - mds::MdsTableHandle handle; - ObLSSwitchChecker ls_switch_checker; bool is_online = false; - if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { - if (OB_ENTRY_NOT_EXIST != ret) { - MDS_LOG_GET(WARN, "failed to get_mds_table"); - } else { - MDS_LOG_GET(TRACE, "failed to get_mds_table"); + do { + mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { + if (OB_ENTRY_NOT_EXIST != ret) { + MDS_LOG_GET(WARN, "failed to get_mds_table"); + } else { + MDS_LOG_GET(TRACE, "failed to get_mds_table"); + } + } else if (!handle.is_valid()) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); + } else if (CLICK_FAIL(handle.get_snapshot(read_op, snapshot, read_seq, timeout_us))) { + if (OB_SNAPSHOT_DISCARDED != ret) { + MDS_LOG_GET(WARN, "failed to get mds data"); + } else { + MDS_LOG_GET(TRACE, "failed to get mds data"); + } } - } else if (!handle.is_valid()) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "mds cannot be NULL"); - } else if (OB_ISNULL(get_tablet_ponter_())) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); - } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { - MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); - } else if (CLICK_FAIL(handle.get_snapshot(read_op, snapshot, read_seq, timeout_us))) { - if (OB_SNAPSHOT_DISCARDED != ret) { - MDS_LOG_GET(WARN, "failed to get mds data"); - } else { - MDS_LOG_GET(TRACE, "failed to get mds data"); - } - } - if (CLICK_FAIL(ret)) { - if (OB_ENTRY_NOT_EXIST == ret || OB_SNAPSHOT_DISCARDED == ret) { - auto func = [&read_op](const T& data) -> int { - return read_op(data); - }; - if (CLICK_FAIL(get_mds_data_from_tablet(func))) { - if (OB_EMPTY_RESULT == ret) { - // read nothing from tablet, maybe this is not an error - } else { - MDS_LOG_GET(WARN, "failed to get snapshot data from tablet"); + if (CLICK_FAIL(ret)) { + if (OB_ENTRY_NOT_EXIST == ret || OB_SNAPSHOT_DISCARDED == ret) { + auto func = [&read_op](const T& data) -> int { + return read_op(data); + }; + if (CLICK_FAIL(get_mds_data_from_tablet(func))) { + if (OB_EMPTY_RESULT == ret) { + // read nothing from tablet, maybe this is not an error + } else { + MDS_LOG_GET(WARN, "failed to get snapshot data from tablet"); + } } } } - } - if (OB_SUCC(ret)) { - if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { - MDS_LOG_GET(WARN, "failed to double check ls online"); - } else { - MDS_LOG_GET(TRACE, "success to get_snapshot"); + if (OB_SUCC(ret)) { + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch(is_online))) { + if (!is_online) { + ret = OB_LS_OFFLINE; + } + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get_snapshot"); + } } - } + } while (ret == OB_VERSION_NOT_MATCH && is_online); return ret; #undef PRINT_WRAPPER } @@ -566,51 +580,56 @@ int ObITabletMdsInterface::get_snapshot(const Key &key, #define PRINT_WRAPPER KR(ret), K(*this), K(key), K(snapshot), K(read_seq), K(typeid(OP).name()) MDS_TG(10_ms); int ret = OB_SUCCESS; - mds::MdsTableHandle handle; - ObLSSwitchChecker ls_switch_checker; bool is_online = false; - if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { - if (OB_ENTRY_NOT_EXIST != ret) { - MDS_LOG_GET(WARN, "failed to get_mds_table"); - } else { - MDS_LOG_GET(TRACE, "failed to get_mds_table"); + do { + mds::MdsTableHandle handle; + ObLSSwitchChecker ls_switch_checker; + if (CLICK_FAIL(get_mds_table_handle_(handle, false))) { + if (OB_ENTRY_NOT_EXIST != ret) { + MDS_LOG_GET(WARN, "failed to get_mds_table"); + } else { + MDS_LOG_GET(TRACE, "failed to get_mds_table"); + } + } else if (!handle.is_valid()) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "mds cannot be NULL"); + } else if (OB_ISNULL(get_tablet_ponter_())) { + ret = OB_ERR_UNEXPECTED; + MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); + } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { + MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); + } else if (CLICK() && OB_SUCCESS != (ret = handle.get_snapshot(key, read_op, snapshot, read_seq, timeout_us))) { + if (OB_SNAPSHOT_DISCARDED != ret) { + MDS_LOG_GET(WARN, "failed to get mds data"); + } else { + MDS_LOG_GET(TRACE, "failed to get mds data"); + } } - } else if (!handle.is_valid()) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "mds cannot be NULL"); - } else if (OB_ISNULL(get_tablet_ponter_())) { - ret = OB_ERR_UNEXPECTED; - MDS_LOG_GET(WARN, "tablet pointer is null", K(ret), KPC(this)); - } else if (MDS_FAIL(ls_switch_checker.check_ls_switch_state(get_tablet_ponter_()->get_ls(), is_online))) { - MDS_LOG_GET(WARN, "check ls online state failed", K(ret), KPC(this)); - } else if (CLICK() && OB_SUCCESS != (ret = handle.get_snapshot(key, read_op, snapshot, read_seq, timeout_us))) { - if (OB_SNAPSHOT_DISCARDED != ret) { - MDS_LOG_GET(WARN, "failed to get mds data"); - } else { - MDS_LOG_GET(TRACE, "failed to get mds data"); - } - } - if (CLICK_FAIL(ret)) { - if (OB_ENTRY_NOT_EXIST == ret || OB_SNAPSHOT_DISCARDED == ret) { - auto func = [&read_op](const Value& data) -> int { - return read_op(data); - }; - if (CLICK_FAIL(get_mds_data_from_tablet(func))) { - if (OB_EMPTY_RESULT == ret) { - // read nothing from tablet, maybe this is not an error - } else { - MDS_LOG_GET(WARN, "failed to get snapshot data from tablet"); + if (CLICK_FAIL(ret)) { + if (OB_ENTRY_NOT_EXIST == ret || OB_SNAPSHOT_DISCARDED == ret) { + auto func = [&read_op](const Value& data) -> int { + return read_op(data); + }; + if (CLICK_FAIL(get_mds_data_from_tablet(func))) { + if (OB_EMPTY_RESULT == ret) { + // read nothing from tablet, maybe this is not an error + } else { + MDS_LOG_GET(WARN, "failed to get snapshot data from tablet"); + } } } } - } - if (OB_SUCC(ret)) { - if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch())) { - MDS_LOG_GET(WARN, "failed to double check ls online"); - } else { - MDS_LOG_GET(TRACE, "success to get_snapshot"); + if (OB_SUCC(ret)) { + if (is_online && MDS_FAIL(ls_switch_checker.double_check_epoch(is_online))) { + if (!is_online) { + ret = OB_LS_OFFLINE; + } + MDS_LOG_GET(WARN, "failed to double check ls online"); + } else { + MDS_LOG_GET(TRACE, "success to get_snapshot"); + } } - } + } while (ret == OB_VERSION_NOT_MATCH && is_online); return ret; #undef PRINT_WRAPPER }