From 56af8cfa43d5d33ee9cfefa53a5d38eadbf2293d Mon Sep 17 00:00:00 2001 From: Handora Date: Mon, 26 Aug 2024 06:39:16 +0000 Subject: [PATCH] fix create memtable during replay corner case --- .../test_callbacks_with_reverse_order.cpp | 2 +- src/storage/ls/ob_ls_tablet_service.cpp | 2 +- src/storage/ob_i_tablet_memtable.cpp | 20 +--- src/storage/ob_i_tablet_memtable.h | 7 -- src/storage/ob_storage_table_guard.cpp | 91 ++++++++++++++----- src/storage/ob_storage_table_guard.h | 15 +-- src/storage/ob_value_row_iterator.cpp | 2 +- src/storage/tablet/ob_tablet.cpp | 46 ++++++---- src/storage/tablet/ob_tablet_memtable_mgr.cpp | 36 ++++---- src/storage/tx/ob_tx_replay_executor.cpp | 2 +- 10 files changed, 127 insertions(+), 96 deletions(-) diff --git a/mittest/simple_server/test_callbacks_with_reverse_order.cpp b/mittest/simple_server/test_callbacks_with_reverse_order.cpp index d7309353b..8babff814 100644 --- a/mittest/simple_server/test_callbacks_with_reverse_order.cpp +++ b/mittest/simple_server/test_callbacks_with_reverse_order.cpp @@ -109,7 +109,7 @@ int ObLSTabletService::insert_tablet_rows( return ret; } -int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table) +int ObStorageTableGuard::refresh_and_protect_memtable_for_write(ObRelativeTable &relative_table) { int ret = OB_SUCCESS; ObTabletTableIterator &iter = relative_table.tablet_iter_; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index ec7da2599..2612b1c53 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -5148,7 +5148,7 @@ int ObLSTabletService::check_row_locked_by_myself( K(relative_table), K(store_ctx), K(rowkey)); } else { ObStorageTableGuard guard(tablet, store_ctx, true); - if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret), K(tablet_handle)); } else if (OB_FAIL(tablet->check_row_locked_by_myself(relative_table, store_ctx, rowkey, locked))) { LOG_WARN("fail to check row locked, ", K(ret), K(rowkey)); diff --git a/src/storage/ob_i_tablet_memtable.cpp b/src/storage/ob_i_tablet_memtable.cpp index 9e55f582e..acfa06fc3 100644 --- a/src/storage/ob_i_tablet_memtable.cpp +++ b/src/storage/ob_i_tablet_memtable.cpp @@ -95,10 +95,9 @@ int ObITabletMemtable::resolve_left_boundary_for_active_memtable_() { int ret = OB_SUCCESS; storage::ObTabletMemtableMgr *memtable_mgr = get_memtable_mgr(); - const SCN new_start_scn = MAX(get_end_scn(), get_migration_clog_checkpoint_scn()); if (OB_NOT_NULL(memtable_mgr)) { - if (OB_FAIL(memtable_mgr->resolve_left_boundary_for_active_memtable(this, new_start_scn))) { + if (OB_FAIL(memtable_mgr->resolve_left_boundary_for_active_memtable(this, get_end_scn()))) { TRANS_LOG(WARN, "fail to resolve left boundary for active memtable", K(ret), K(ls_id_), KPC(this)); } } @@ -142,23 +141,6 @@ int ObITabletMemtable::set_freezer(ObFreezer *handler) return ret; } -int ObITabletMemtable::set_migration_clog_checkpoint_scn(const SCN &clog_checkpoint_scn) -{ - int ret = OB_SUCCESS; - - if (OB_UNLIKELY(!is_inited())) { - ret = OB_NOT_INIT; - TRANS_LOG(WARN, "not inited", K(ret)); - } else if (clog_checkpoint_scn <= ObScnRange::MIN_SCN) { - ret = OB_SCN_OUT_OF_BOUND; - TRANS_LOG(WARN, "invalid clog_checkpoint_ts", K(ret)); - } else { - (void)migration_clog_checkpoint_scn_.atomic_store(clog_checkpoint_scn); - } - - return ret; -} - int ObITabletMemtable::set_rec_scn(const SCN rec_scn) { int ret = OB_SUCCESS; diff --git a/src/storage/ob_i_tablet_memtable.h b/src/storage/ob_i_tablet_memtable.h index 06c4a7f80..07c89a2c8 100644 --- a/src/storage/ob_i_tablet_memtable.h +++ b/src/storage/ob_i_tablet_memtable.h @@ -209,14 +209,12 @@ public: unsubmitted_cnt_(0), logging_blocked_start_time_(0), write_ref_cnt_(0), - migration_clog_checkpoint_scn_(), freeze_state_(TabletMemtableFreezeState::INVALID), memtable_mgr_handle_() { max_end_scn_.set_min(); rec_scn_.set_max(); freeze_scn_.set_max(); - migration_clog_checkpoint_scn_.set_min(); } void reset() @@ -241,7 +239,6 @@ public: max_end_scn_.set_min(); rec_scn_.set_max(); freeze_scn_.set_max(); - migration_clog_checkpoint_scn_.set_min(); freezer_ = nullptr; memtable_mgr_handle_.reset(); mt_stat_.reset(); @@ -257,7 +254,6 @@ public: int dec_unsubmitted_cnt(); int set_freezer(ObFreezer *handler); int set_rec_scn(const share::SCN rec_scn); - int set_migration_clog_checkpoint_scn(const share::SCN &clog_checkpoint_scn); int resolve_left_boundary(share::SCN end_scn) { return set_start_scn(end_scn); } int resolve_right_boundary(); int replay_schema_version_change_log(const int64_t schema_version); @@ -374,7 +370,6 @@ public: const ObMtStat &get_mt_stat() const { return mt_stat_; } share::SCN get_max_end_scn() const { return max_end_scn_.atomic_get(); } share::SCN get_rec_scn() { return rec_scn_.atomic_get(); } - share::SCN get_migration_clog_checkpoint_scn() { return migration_clog_checkpoint_scn_.atomic_get(); } ObTabletMemtableMgr *get_memtable_mgr(); // *************** getter ***************** @@ -399,7 +394,6 @@ public: K(max_end_scn_), K(rec_scn_), K(freeze_scn_), - K(migration_clog_checkpoint_scn_), KP(freezer_), K(memtable_mgr_handle_), K(mt_stat_.frozen_time_), @@ -477,7 +471,6 @@ private: int64_t unsubmitted_cnt_; int64_t logging_blocked_start_time_; // record the start time of logging blocked int64_t write_ref_cnt_ CACHE_ALIGNED; - share::SCN migration_clog_checkpoint_scn_; TabletMemtableFreezeState freeze_state_; ObMemtableMgrHandle memtable_mgr_handle_; }; diff --git a/src/storage/ob_storage_table_guard.cpp b/src/storage/ob_storage_table_guard.cpp index dd739f63d..20ea22139 100644 --- a/src/storage/ob_storage_table_guard.cpp +++ b/src/storage/ob_storage_table_guard.cpp @@ -37,8 +37,7 @@ ObStorageTableGuard::ObStorageTableGuard( ObStoreCtx &store_ctx, const bool need_control_mem, const bool for_replay, - const SCN replay_scn, - const bool for_multi_source_data) + const SCN replay_scn) : tablet_(tablet), store_ctx_(store_ctx), need_control_mem_(need_control_mem), @@ -46,8 +45,7 @@ ObStorageTableGuard::ObStorageTableGuard( retry_count_(0), last_ts_(0), for_replay_(for_replay), - replay_scn_(replay_scn), - for_multi_source_data_(for_multi_source_data) + replay_scn_(replay_scn) { init_ts_ = ObClockGenerator::getClock(); share::memstore_throttled_alloc() = 0; @@ -105,7 +103,7 @@ void ObStorageTableGuard::throttle_if_needed_() } } -int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table) +int ObStorageTableGuard::refresh_and_protect_memtable_for_write(ObRelativeTable &relative_table) { int ret = OB_SUCCESS; ObTabletTableIterator &iter = relative_table.tablet_iter_; @@ -142,7 +140,7 @@ int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_tab return ret; } -int ObStorageTableGuard::refresh_and_protect_memtable() +int ObStorageTableGuard::refresh_and_protect_memtable_for_replay() { const int64_t DEFAULT_REFRESH_WARN_INTERVAL = 10LL * 1000LL; // 10 ms const int64_t FIND_DIRECT_LOAD_MT_WARN_INTERVAL = 10LL * 1000LL * 1000LL; // 10 seconds @@ -163,7 +161,7 @@ int ObStorageTableGuard::refresh_and_protect_memtable() if (OB_FAIL(tablet_->get_boundary_memtable_from_memtable_mgr(handle))) { // if there is no memtable, create a new one if (OB_ENTRY_NOT_EXIST == ret) { - ret = create_data_memtable_(ls_id, tablet_id, need_retry); + ret = create_data_memtable_for_replay_(ls_id, tablet_id, need_retry); } else { // OB_ENTRY_NOT_EXIST != ret LOG_WARN("fail to get boundary memtable", K(ret), K(ls_id), K(tablet_id)); } @@ -172,7 +170,7 @@ int ObStorageTableGuard::refresh_and_protect_memtable() } else if (tablet_memtable->is_direct_load_memtable()) { // set warn interval to 1 second because freeze direct load memtable is an async task warn_interval = FIND_DIRECT_LOAD_MT_WARN_INTERVAL; - ret = create_data_memtable_(ls_id, tablet_id, need_retry); + ret = create_data_memtable_for_replay_(ls_id, tablet_id, need_retry); } else if (OB_FAIL(check_freeze_to_inc_write_ref(static_cast(tablet_memtable), need_retry))) { if (OB_EAGAIN == ret) { } else if (OB_MINOR_FREEZE_NOT_ALLOW != ret) { @@ -201,9 +199,9 @@ int ObStorageTableGuard::refresh_and_protect_memtable() return ret; } -int ObStorageTableGuard::create_data_memtable_(const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - bool &need_retry) +int ObStorageTableGuard::create_data_memtable_for_replay_(const share::ObLSID &ls_id, + const common::ObTabletID &tablet_id, + bool &need_retry) { int ret = OB_SUCCESS; LOG_DEBUG("there is no boundary memtable", K(ret), K(ls_id), K(tablet_id)); @@ -229,13 +227,43 @@ int ObStorageTableGuard::create_data_memtable_(const share::ObLSID &ls_id, tablet_id, 0 /* schema version */, false /* for_direct_load */, for_replay_, clog_checkpoint_scn))) { LOG_WARN("fail to create a boundary memtable", K(ret), K(ls_id), K(tablet_id)); } - } else { - // replay_log_scn_ <= clog_checkpoint_scn. no need to create a boundary memtable - need_retry = false; + // In situation that replay_log_scn_ <= clog_checkpoint_scn, we have no need + // to create the memtable. While we need double check to decide whether + // another thread has created the memtable that we need replay. And if it + // does, we must replay on the memtable. + } else if (OB_FAIL(double_check_get_memtable_for_replay_(replay_scn_, need_retry))) { + LOG_WARN("fail to double check replay memtable", K(ret), K(ls_id), K(tablet_id), + K(replay_scn_), K(clog_checkpoint_scn)); } return ret; } +int ObStorageTableGuard::double_check_get_memtable_for_replay_(const share::SCN replay_scn, + bool &need_retry) +{ + int ret = OB_SUCCESS; + ObTableHandleV2 handle; + ObProtectedMemtableMgrHandle *protected_handle = NULL; + + if (OB_FAIL(tablet_->get_protected_memtable_mgr_handle(protected_handle))) { + LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(tablet_)); + } else if (OB_FAIL(protected_handle->get_memtable_for_replay(replay_scn_, handle))) { + if (OB_NO_NEED_UPDATE == ret) { + // no need to replay the log + need_retry = false; + ret = OB_SUCCESS; + } else if (OB_ENTRY_NOT_EXIST == ret) { + // memtable_mgr not exist, it means nothing need replay + need_retry = false; + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(replay_scn)); + } + } + + return ret; +} + void ObStorageTableGuard::reset() { if (NULL != memtable_) { @@ -302,7 +330,7 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo } else if (memtable->is_active_memtable()) { // the most recent memtable is active // no need to create a new memtable - if (for_replay_ || for_multi_source_data_) { + if (for_replay_) { // filter memtables for replay or multi_source_data according to scn ObTableHandleV2 handle; if (OB_FAIL(tablet_->get_protected_memtable_mgr_handle(protected_handle))) { @@ -334,7 +362,6 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo if (0 == write_ref) { SCN clog_checkpoint_scn; bool need_create_memtable = true; - SCN migration_clog_checkpoint_scn; ObTabletHandle tmp_handle; ObLSHandle ls_handle; if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { @@ -343,15 +370,13 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, invalid ls handle", K(ret), K(need_retry), K(ls_handle), K(ls_id), K(tablet_id)); } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet(tablet_id, - tmp_handle, 0, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { + tmp_handle, + 0, + ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("fail to get tablet", K(ret), K(ls_id), K(tablet_id)); } else if (FALSE_IT(clog_checkpoint_scn = tmp_handle.get_obj()->get_tablet_meta().clog_checkpoint_scn_)) { - } else if (FALSE_IT(migration_clog_checkpoint_scn = memtable->get_migration_clog_checkpoint_scn())) { - } else if (for_replay_ && !migration_clog_checkpoint_scn.is_min()) { - memtable->resolve_right_boundary(); - if (replay_scn_ <= clog_checkpoint_scn) { - need_create_memtable = false; - } + } else if (for_replay_ && replay_scn_ <= clog_checkpoint_scn) { + need_create_memtable = false; } // create a new memtable if no write in the old memtable @@ -368,6 +393,26 @@ int ObStorageTableGuard::check_freeze_to_inc_write_ref(ObMemtable *memtable, boo LOG_WARN("fail to create new memtable for freeze", K(ret), K(need_retry), K(ls_id), K(tablet_id)); } } + } else if (for_replay_) { + ObTableHandleV2 handle; + if (OB_FAIL(tablet_->get_protected_memtable_mgr_handle(protected_handle))) { + LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(tablet_)); + } else if (OB_FAIL(protected_handle->get_memtable_for_replay(replay_scn_, handle))) { + if (OB_NO_NEED_UPDATE == ret) { + // no need to replay the log + need_retry = false; + ret = OB_SUCCESS; + } else { + LOG_WARN("fail to get memtable for replay", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + } + } else if (OB_FAIL(handle.get_data_memtable(memtable))) { + LOG_WARN("fail to get memtable from ObTableHandle", K(ret), K(need_retry), K(ls_id), K(tablet_id)); + } else { + if (memtable != old_memtable) { + is_tablet_freeze = memtable->get_is_tablet_freeze(); + } + double_check_inc_write_ref(old_freeze_flag, is_tablet_freeze, memtable, need_retry); + } } } } diff --git a/src/storage/ob_storage_table_guard.h b/src/storage/ob_storage_table_guard.h index 957bc4b75..ee64f133d 100644 --- a/src/storage/ob_storage_table_guard.h +++ b/src/storage/ob_storage_table_guard.h @@ -49,21 +49,21 @@ public: ObStoreCtx &store_ctx, const bool need_control_mem, const bool for_replay = false, - const share::SCN replay_scn = share::SCN(), - const bool for_multi_source_data = false); + const share::SCN replay_scn = share::SCN()); ~ObStorageTableGuard(); ObStorageTableGuard(const ObStorageTableGuard&) = delete; ObStorageTableGuard &operator=(const ObStorageTableGuard&) = delete; public: - int refresh_and_protect_table(ObRelativeTable &relative_table); - int refresh_and_protect_memtable(); + // refresh and get/create the memtable for write + int refresh_and_protect_memtable_for_write(ObRelativeTable &relative_table); + // refresh and get/create the memtable for replay + int refresh_and_protect_memtable_for_replay(); int get_memtable_for_replay(ObIMemtable *&memtable); TO_STRING_KV(KP(tablet_), K(need_control_mem_), K(for_replay_), - K(for_multi_source_data_), K(replay_scn_), KP(memtable_), K(retry_count_), @@ -78,10 +78,12 @@ private: memtable::ObMemtable *memtable, bool &bool_ret); int check_freeze_to_inc_write_ref(memtable::ObMemtable *table, bool &bool_ret); - int create_data_memtable_(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &no_need_create); + int create_data_memtable_for_replay_(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id, bool &no_need_create); bool need_to_refresh_table(ObTableStoreIterator &iter); void check_if_need_log_(bool &need_log, bool &need_log_error); void throttle_if_needed_(); + int double_check_get_memtable_for_replay_(const share::SCN replay_scn, + bool &need_retry); private: static const int64_t LOG_INTERVAL_US = 10 * 1000 * 1000; // 10s @@ -99,7 +101,6 @@ private: int64_t init_ts_; bool for_replay_; share::SCN replay_scn_; - bool for_multi_source_data_; }; } // namespace storage } // namespace oceanbase diff --git a/src/storage/ob_value_row_iterator.cpp b/src/storage/ob_value_row_iterator.cpp index aa99478ee..124f2d377 100644 --- a/src/storage/ob_value_row_iterator.cpp +++ b/src/storage/ob_value_row_iterator.cpp @@ -261,7 +261,7 @@ int ObSingleRowGetter::open(const ObDatumRowkey &rowkey, bool use_fuse_row_cache } else { { ObStorageTableGuard guard(tablet_, *store_ctx_, false); - if (OB_FAIL(guard.refresh_and_protect_table(*relative_table_))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(*relative_table_))) { STORAGE_LOG(WARN, "fail to protect table", K(ret)); } } diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index e7d885ce0..7fbcbf133 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -3656,7 +3656,7 @@ int ObTablet::lock_row( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret), "tablet_id", tablet_meta_.tablet_id_); } if (OB_SUCC(ret)) { @@ -3696,7 +3696,7 @@ int ObTablet::lock_row( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } else { ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_STORE_ROW_LOCK_CHECKER)); @@ -4127,7 +4127,7 @@ int ObTablet::update_row( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); @@ -4182,7 +4182,7 @@ int ObTablet::insert_rows( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("Failed to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("Failed to prepare write memtable", K(ret), K(relative_table)); @@ -4231,7 +4231,7 @@ int ObTablet::insert_row_without_rowkey_check( } else if (OB_UNLIKELY(relative_table.get_tablet_id() != tablet_meta_.tablet_id_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tablet id doesn't match", K(ret), K(relative_table.get_tablet_id()), K(tablet_meta_.tablet_id_)); - } else if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + } else if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); @@ -4393,7 +4393,7 @@ int ObTablet::rowkey_exists( } else { { ObStorageTableGuard guard(this, store_ctx, false); - if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } } @@ -4445,7 +4445,7 @@ int ObTablet::rowkeys_exists( } else { { ObStorageTableGuard guard(this, store_ctx, false); - if (OB_FAIL(guard.refresh_and_protect_table(relative_table))) { + if (OB_FAIL(guard.refresh_and_protect_memtable_for_write(relative_table))) { LOG_WARN("fail to protect table", K(ret)); } } @@ -4683,7 +4683,11 @@ int ObTablet::create_memtable( ObTimeGuard time_guard("ObTablet::create_memtable", 10 * 1000); common::SpinWLockGuard guard(memtables_lock_); time_guard.click("lock"); - const SCN new_clog_checkpoint_scn = clog_checkpoint_scn.is_min() ? tablet_meta_.clog_checkpoint_scn_ : clog_checkpoint_scn; + // we use the parameter clog_checkpoint_scn to double check whether the + // clog_checkpoint_scn has been changed during memtable replay check. + // So we complement the input_clog_checkpoint_scn for other scenario. + const SCN input_clog_checkpoint_scn = clog_checkpoint_scn.is_min() ? + tablet_meta_.clog_checkpoint_scn_ : clog_checkpoint_scn; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -4692,11 +4696,14 @@ int ObTablet::create_memtable( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid schema version", K(ret), K(schema_version)); } else if (FALSE_IT(time_guard.click("prepare_memtables"))) { - } else if (OB_FAIL(inner_create_memtable(new_clog_checkpoint_scn, schema_version, for_direct_load, for_replay))) { + } else if (OB_FAIL(inner_create_memtable(input_clog_checkpoint_scn, + schema_version, + for_direct_load, + for_replay))) { if (OB_ENTRY_EXIST == ret) { ret = OB_SUCCESS; } else if (OB_MINOR_FREEZE_NOT_ALLOW != ret) { - LOG_WARN("failed to create memtable", K(ret), K(clog_checkpoint_scn), + LOG_WARN("failed to create memtable", K(ret), K(input_clog_checkpoint_scn), K(schema_version), K(for_replay)); } } else { @@ -4729,14 +4736,14 @@ int ObTablet::create_memtable( STORAGE_LOG(DEBUG, "Tablet finish create memtable", K(schema_version), - K(clog_checkpoint_scn), + K(input_clog_checkpoint_scn), K(for_replay), K(lbt())); return ret; } int ObTablet::inner_create_memtable( - const SCN clog_checkpoint_scn, + const SCN input_clog_checkpoint_scn, const int64_t schema_version, const bool for_direct_load, const bool for_replay) @@ -4744,12 +4751,12 @@ int ObTablet::inner_create_memtable( int ret = OB_SUCCESS; const share::ObLSID &ls_id = tablet_meta_.ls_id_; const common::ObTabletID &tablet_id = tablet_meta_.tablet_id_; - const SCN new_clog_checkpoint_scn = tablet_meta_.clog_checkpoint_scn_; + const SCN latest_clog_checkpoint_scn = tablet_meta_.clog_checkpoint_scn_; ObProtectedMemtableMgrHandle *protected_handle = NULL; - if (OB_UNLIKELY(!clog_checkpoint_scn.is_valid_and_not_min()) || OB_UNLIKELY(schema_version < 0)) { + if (OB_UNLIKELY(!input_clog_checkpoint_scn.is_valid_and_not_min()) || OB_UNLIKELY(schema_version < 0)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid args", K(ret), K(clog_checkpoint_scn), K(schema_version)); + LOG_WARN("invalid args", K(ret), K(input_clog_checkpoint_scn), K(schema_version)); } else if (OB_UNLIKELY(MAX_MEMSTORE_CNT == memtable_count_)) { ret = OB_MINOR_FREEZE_NOT_ALLOW; if (TC_REACH_TIME_INTERVAL(1_s)) { @@ -4760,16 +4767,17 @@ int ObTablet::inner_create_memtable( LOG_WARN("failed to get_protected_memtable_mgr_handle", K(ret), KPC(this)); } else if (OB_FAIL(protected_handle->create_memtable(tablet_meta_, CreateMemtableArg(schema_version, - clog_checkpoint_scn, - new_clog_checkpoint_scn, + input_clog_checkpoint_scn, + latest_clog_checkpoint_scn, for_replay, for_direct_load)))) { if (OB_ENTRY_EXIST != ret && OB_MINOR_FREEZE_NOT_ALLOW != ret) { LOG_WARN("failed to create memtable", K(ret), K(ls_id), K(tablet_id), KPC(this)); } } else { - LOG_INFO("succeeded to create memtable for tablet", K(ret), K(ls_id), K(tablet_id), - K(clog_checkpoint_scn), K(schema_version), K(for_replay)); + LOG_INFO("succeeded to create memtable for tablet", K(ret), K(tablet_id), + K(ls_id), K(schema_version), K(for_replay), K(for_direct_load), + K(input_clog_checkpoint_scn), K(latest_clog_checkpoint_scn)); } return ret; diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.cpp b/src/storage/tablet/ob_tablet_memtable_mgr.cpp index 6041e6d3e..1fd1dafd2 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.cpp +++ b/src/storage/tablet/ob_tablet_memtable_mgr.cpp @@ -189,12 +189,13 @@ int ObTabletMemtableMgr::try_resolve_boundary_on_create_memtable_for_leader_( } else if (can_resolve) { SCN new_start_scn; last_frozen_tablet_memtable->resolve_right_boundary(); - last_frozen_tablet_memtable->set_resolved_active_memtable_left_boundary(); if (new_tablet_memtable != last_frozen_tablet_memtable) { new_start_scn = MAX(last_frozen_tablet_memtable->get_end_scn(), - last_frozen_tablet_memtable->get_migration_clog_checkpoint_scn()); + new_tablet_memtable->get_start_scn()); if (OB_FAIL(new_tablet_memtable->resolve_left_boundary(new_start_scn))) { TRANS_LOG(ERROR, "resolve left boundary failed", KR(ret), K(new_start_scn), KPC(new_tablet_memtable)); + } else { + last_frozen_tablet_memtable->set_resolved_active_memtable_left_boundary(); } } TRANS_LOG(INFO, "[resolve_right_boundary] in create_memtable on leader", K(ret), @@ -271,9 +272,10 @@ int ObTabletMemtableMgr::check_boundary_memtable_(const uint32_t logstream_freez } else if (tablet_memtable->is_direct_load_memtable()) { if (tablet_memtable->get_end_scn().is_max()) { PAUSE(); - // if end_scn of direct load memtable has not decided, return OB_ENTRY_EXIST and Tablet will reset it to - // OB_SUCCESS. Then refresh_and_protect_table(refresh_and_protect_memtable) in StorageTableGuard will retry create - // memtable + // if end_scn of direct load memtable has not decided, return + // OB_ENTRY_EXIST and Tablet will reset it to OB_SUCCESS. Then + // refresh_and_protect_memtable_for_write(refresh_and_protect_memtable_for_replay) + // in StorageTableGuard will retry create memtable ret = OB_ENTRY_EXIST; } } else if (tablet_memtable->is_data_memtable()) { @@ -389,19 +391,19 @@ int ObTabletMemtableMgr::resolve_data_memtable_boundary_(ObITabletMemtable *froz { int ret = OB_SUCCESS; if (arg.for_replay_) { - SCN new_memtable_start_scn; frozen_tablet_memtable->resolve_right_boundary(); if (new_tablet_memtable != frozen_tablet_memtable) { - new_memtable_start_scn = - MAX(frozen_tablet_memtable->get_end_scn(), frozen_tablet_memtable->get_migration_clog_checkpoint_scn()); - if (OB_FAIL(new_tablet_memtable->resolve_left_boundary(new_memtable_start_scn))) { - TRANS_LOG(WARN, "resolve left boundary fail", K(ret), K(new_tablet_memtable)); + if (OB_FAIL(new_tablet_memtable->resolve_left_boundary( + MAX(frozen_tablet_memtable->get_end_scn(), + new_tablet_memtable->get_start_scn())))) { + TRANS_LOG(WARN, "resolve left boundary fail", K(ret), K(arg), + KPC(new_tablet_memtable), KPC(frozen_tablet_memtable)); } } - TRANS_LOG(INFO, "[resolve_right_boundary] in create_memtable on replay", KPC(frozen_tablet_memtable)); - } - // for leader, decide the right boundary of frozen memtable - else if (OB_FAIL(try_resolve_boundary_on_create_memtable_for_leader_(frozen_tablet_memtable, new_tablet_memtable))) { + TRANS_LOG(INFO, "[resolve_right_boundary] in create_memtable on replay", + KPC(frozen_tablet_memtable), KPC(new_tablet_memtable), K(arg)); + } else if (OB_FAIL(try_resolve_boundary_on_create_memtable_for_leader_( + frozen_tablet_memtable, new_tablet_memtable))) { TRANS_LOG(WARN, "try resolve boundary fail", K(ret)); } return ret; @@ -420,7 +422,7 @@ int ObTabletMemtableMgr::resolve_direct_load_memtable_boundary_(ObITabletMemtabl STORAGE_LOG(ERROR, "frozen direct load memtable must have a valid end_scn", KPC(frozen_tablet_memtable)); } else if (active_tablet_memtable != frozen_tablet_memtable) { new_memtable_start_scn = - MAX(frozen_tablet_memtable->get_end_scn(), frozen_tablet_memtable->get_migration_clog_checkpoint_scn()); + MAX(frozen_tablet_memtable->get_end_scn(), active_tablet_memtable->get_start_scn()); if (OB_FAIL(active_tablet_memtable->resolve_left_boundary(new_memtable_start_scn))) { STORAGE_LOG(ERROR, "fail to resolve left boundary", KPC(active_tablet_memtable)); } else { @@ -622,8 +624,8 @@ int ObTabletMemtableMgr::resolve_left_boundary_for_active_memtable(ObITabletMemt } else if (OB_FAIL(handle.get_tablet_memtable(active_tablet_memtable))) { LOG_WARN("fail to get active memtable", K(ret)); } else { - // set the start_scn of the new memtable - int tmp_ret = active_tablet_memtable->resolve_left_boundary(start_scn); + int tmp_ret = active_tablet_memtable->resolve_left_boundary( + MAX(start_scn, active_tablet_memtable->get_start_scn())); if (OB_SUCCESS != tmp_ret) { TRANS_LOG(ERROR, "resolve left boundary failed", K(start_scn), KPC(active_tablet_memtable), KPC(tablet_memtable)); } diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index bdf79986c..91f0b0ede 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -783,7 +783,7 @@ int ObTxReplayExecutor::prepare_memtable_replay_(ObStorageTableGuard &w_guard, ObIMemtable *&mem_ptr) { int ret = OB_SUCCESS; - if (OB_FAIL(w_guard.refresh_and_protect_memtable())) { + if (OB_FAIL(w_guard.refresh_and_protect_memtable_for_replay())) { TRANS_LOG(WARN, "[Replay Tx] refresh and protect memtable error", K(ret)); } else if (OB_FAIL(w_guard.get_memtable_for_replay(mem_ptr))) { // OB_NO_NEED_UPDATE => don't need to replay