From a4bf57407ce7328b4da3da0cf34c93f795c28be9 Mon Sep 17 00:00:00 2001 From: KyrielightWei Date: Wed, 31 May 2023 11:47:29 +0000 Subject: [PATCH] serialize all change status in DupTabletLogBody Co-authored-by: Minionyh --- .../multi_replica/test_ob_dup_table_basic.cpp | 9 + .../memtable/ob_redo_log_generator.cpp | 8 +- src/storage/tx/ob_dup_table_base.h | 12 +- src/storage/tx/ob_dup_table_tablets.cpp | 733 +++++------------- src/storage/tx/ob_dup_table_tablets.h | 318 +++----- src/storage/tx/ob_dup_table_util.cpp | 2 +- src/storage/tx/ob_trans_define.cpp | 2 - src/storage/tx/ob_trans_define.h | 1 - src/storage/tx/ob_trans_part_ctx.cpp | 176 ++--- src/storage/tx/ob_trans_part_ctx.h | 6 +- src/storage/tx/ob_tx_log.cpp | 2 +- src/storage/tx/ob_tx_replay_executor.cpp | 2 - 12 files changed, 423 insertions(+), 848 deletions(-) diff --git a/mittest/multi_replica/test_ob_dup_table_basic.cpp b/mittest/multi_replica/test_ob_dup_table_basic.cpp index 4d9f10184..88441443c 100644 --- a/mittest/multi_replica/test_ob_dup_table_basic.cpp +++ b/mittest/multi_replica/test_ob_dup_table_basic.cpp @@ -368,6 +368,7 @@ TEST_F(GET_ZONE_TEST_CLASS_NAME(2), dup_table_trx_read_committed) std::string tmp_event_val; ASSERT_EQ(OB_SUCCESS, wait_event_finish("INSERT_TRX_COMMIT", tmp_event_val, 30 * 1000 * 1000)); check_dup_table_insert_readable(this, 1, true /*expected_follower_read*/); + ASSERT_EQ(OB_SUCCESS, finish_event("ZONE2_READ_INSERT_COMMITTED", "")); } TEST_F(GET_ZONE_TEST_CLASS_NAME(3), dup_table_trx_read_committed) @@ -377,11 +378,19 @@ TEST_F(GET_ZONE_TEST_CLASS_NAME(3), dup_table_trx_read_committed) ASSERT_EQ(OB_SUCCESS, wait_event_finish("INSERT_TRX_COMMIT", tmp_event_val, 30 * 1000 * 1000, 100)); check_dup_table_insert_readable(this, 1, true /*expected_follower_read*/); + ASSERT_EQ(OB_SUCCESS, finish_event("ZONE3_READ_INSERT_COMMITTED", "")); } TEST_F(GET_ZONE_TEST_CLASS_NAME(1), remove_dup_table) { int ret = OB_SUCCESS; + + std::string tmp_event_val; + ASSERT_EQ(OB_SUCCESS, + wait_event_finish("ZONE2_READ_INSERT_COMMITTED", tmp_event_val, 30 * 1000 * 1000)); + ASSERT_EQ(OB_SUCCESS, + wait_event_finish("ZONE3_READ_INSERT_COMMITTED", tmp_event_val, 30 * 1000 * 1000)); + WRITE_SQL_BY_CONN(static_test_conn_, "drop table Persons"); GET_LS(static_basic_arg_.tenant_id_, static_basic_arg_.ls_id_num_, ls_handle); diff --git a/src/storage/memtable/ob_redo_log_generator.cpp b/src/storage/memtable/ob_redo_log_generator.cpp index 73f963ca7..b5d336d40 100644 --- a/src/storage/memtable/ob_redo_log_generator.cpp +++ b/src/storage/memtable/ob_redo_log_generator.cpp @@ -378,10 +378,10 @@ bool ObRedoLogGenerator::check_dup_tablet_(const ObITransCallback *callback_ptr) if (MutatorType::MUTATOR_ROW == callback_ptr->get_mutator_type()) { const ObMvccRowCallback *row_iter = static_cast(callback_ptr); const ObTabletID &target_tablet = row_iter->get_tablet_id(); - if (OB_TMP_FAIL(mem_ctx_->get_trans_ctx()->merge_tablet_modify_record_(target_tablet))) { - TRANS_LOG_RET(WARN, tmp_ret, "merge tablet modify record failed", K(tmp_ret), - K(target_tablet), KPC(row_iter)); - } + // if (OB_TMP_FAIL(mem_ctx_->get_trans_ctx()->merge_tablet_modify_record_(target_tablet))) { + // TRANS_LOG_RET(WARN, tmp_ret, "merge tablet modify record failed", K(tmp_ret), + // K(target_tablet), KPC(row_iter)); + // } // check dup table } diff --git a/src/storage/tx/ob_dup_table_base.h b/src/storage/tx/ob_dup_table_base.h index 6dd52e57e..cea03c35c 100644 --- a/src/storage/tx/ob_dup_table_base.h +++ b/src/storage/tx/ob_dup_table_base.h @@ -563,7 +563,7 @@ private: * Dup_Table Tablets *******************************************************/ -class DupTabletCommonHeader +class DupTabletSetCommonHeader { OB_UNIS_VERSION(1); @@ -600,14 +600,14 @@ public: K(tablet_set_type_), K(sp_op_type_)); - DupTabletCommonHeader(const uint64_t id) : unique_id_(id) + DupTabletSetCommonHeader(const uint64_t id) : unique_id_(id) { // set_free(); // set_invalid_sp_op_type(); reuse(); } - DupTabletCommonHeader() { reset(); } - ~DupTabletCommonHeader() { reset(); } + DupTabletSetCommonHeader() { reset(); } + ~DupTabletSetCommonHeader() { reset(); } bool is_valid() const { return unique_id_is_valid() && tablet_set_type_is_valid(); @@ -669,7 +669,7 @@ public: } // bool contain_special_op(uint64_t special_op) const { return get_special_op_() == special_op; } bool no_specail_op() const { return INVALID_SPECIAL_OP == sp_op_type_; } - void copy_tablet_set_type(const DupTabletCommonHeader &src_common_header) + void copy_tablet_set_type(const DupTabletSetCommonHeader &src_common_header) { set_unique_id_(src_common_header.get_unique_id()); set_special_op_(src_common_header.get_special_op()); @@ -712,7 +712,7 @@ private: int64_t sp_op_type_; }; -typedef common::ObSEArray DupTabletSetIDArray; +typedef common::ObSEArray DupTabletSetIDArray; /******************************************************* * Dup_Table Checkpoint diff --git a/src/storage/tx/ob_dup_table_tablets.cpp b/src/storage/tx/ob_dup_table_tablets.cpp index f8504c2ab..3ef56a6d2 100644 --- a/src/storage/tx/ob_dup_table_tablets.cpp +++ b/src/storage/tx/ob_dup_table_tablets.cpp @@ -34,46 +34,19 @@ const int64_t ObLSDupTabletsMgr::RESERVED_FREE_SET_COUNT = 64; const int64_t ObLSDupTabletsMgr::MAX_FREE_SET_COUNT = 1000; const int64_t ObLSDupTabletsMgr::MAX_CONFIRMING_TABLET_COUNT = 20000; -OB_SERIALIZE_MEMBER(DupTabletCommonHeader, unique_id_, tablet_set_type_, sp_op_type_); -OB_SERIALIZE_MEMBER(DupTabletChangeLogTail, readable_version_, has_confirmed_); -OB_SERIALIZE_MEMBER(DupTabletSpecialOpArg, op_objects_); - -OB_SERIALIZE_MEMBER(DupTabletCommonLogBody, tablet_id_map_); -OB_SERIALIZE_MEMBER_INHERIT(DupTabletChangeLogBody, DupTabletCommonLogBody, change_tail_); -OB_SERIALIZE_MEMBER_INHERIT(DupTabletSpecialOpLogBody, DupTabletChangeLogBody, sp_op_arg_); +OB_SERIALIZE_MEMBER(DupTabletSetCommonHeader, unique_id_, tablet_set_type_, sp_op_type_); +OB_SERIALIZE_MEMBER(DupTabletSetChangeStatus, + flag_, + tablet_change_scn_, + need_confirm_scn_, + readable_version_); +OB_SERIALIZE_MEMBER(DupTabletSetAttribute, common_header_, change_status_); +OB_SERIALIZE_MEMBER(DupTabletLogBody, hash_map_); //********************************************************************** //****** Hash Callback //********************************************************************** -int TabletsSerCallBack::operator()( - const common::hash::HashMapPair &hash_pair) -{ - return hash_pair.first.serialize(buf_, buf_len_, pos_); -} - -int TabletsDeSerCallBack::operator()(DupTabletChangeMap &dup_tablet_map) -{ - int ret = OB_SUCCESS; - ObTabletID tablet_id; - DupTabletInfo tmp_info; - tmp_info.update_dup_schema_ts_ = deser_time_; - - if (OB_FAIL(tablet_id.deserialize(buf_, buf_len_, pos_))) { - DUP_TABLE_LOG(WARN, "deserialize tablet id failed", K(ret)); - } else if (OB_FAIL(dup_tablet_map.set_refactored(tablet_id, tmp_info, 1))) { - DUP_TABLE_LOG(WARN, "insert tablet failed", K(ret), K(tablet_id), K(tmp_info)); - } - - return ret; -} - -int64_t TabletsGetSizeCallBack::operator()( - const common::hash::HashMapPair &hash_pair) -{ - return hash_pair.first.get_serialize_size(); -} - bool ObLSDupTabletsMgr::GcDiscardedDupTabletHandler::operator()( common::hash::HashMapPair &hash_pair) { @@ -180,27 +153,51 @@ int ObLSDupTabletsMgr::CollectTabletsHandler::operator()( int DupTabletChangeMap::create(int64_t bucket_num) { int ret = OB_SUCCESS; - if (!common_header_.is_valid()) { + if (!dup_set_attr_.common_header_.is_valid()) { ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "invalid unique_id", K(ret), K(common_header_)); + DUP_TABLE_LOG(WARN, "invalid unique_id", K(ret), K(dup_set_attr_)); } else if (OB_FAIL(DupTabletIdMap::create(bucket_num, "DupTabletHash"))) { - DUP_TABLE_LOG(WARN, "create dup tablet id map failed", K(ret), K(common_header_), - K(bucket_num)); + DUP_TABLE_LOG(WARN, "create dup tablet id map failed", K(ret), K(dup_set_attr_), K(bucket_num)); } return ret; } +int DupTabletChangeMap::deserialize_set_attribute(const char *buf, + int64_t data_len, + int64_t pos, + DupTabletSetAttribute &deser_attr) +{ + int ret = OB_SUCCESS; + + if (OB_SUCC(ret)) { + if (OB_FAIL(deser_attr.deserialize(buf, data_len, pos))) { + + DUP_TABLE_LOG(WARN, "deserialize dup tablet attribute failed", K(ret), K(pos), K(deser_attr)); + } + } + return ret; +} + int DupTabletChangeMap::serialize(char *buf, const int64_t buf_len, int64_t &pos) const { int ret = OB_SUCCESS; int64_t tmp_pos = pos; - TabletsSerCallBack ser_cb(buf, buf_len, tmp_pos); - if (OB_FAIL(hash_for_each_serialize(*this, ser_cb))) { - DUP_TABLE_LOG(WARN, "serialize dup tablet hash map faild", K(ret)); - } else { - tmp_pos = ser_cb.get_pos(); + if (OB_SUCC(ret)) { + if (OB_FAIL(dup_set_attr_.serialize(buf, buf_len, tmp_pos))) { + DUP_TABLE_LOG(WARN, "serialize dup tablet attribute failed", K(ret), K(tmp_pos), + K(dup_set_attr_)); + } + } + + if (OB_SUCC(ret)) { + TabletsSerCallBack ser_cb(buf, buf_len, tmp_pos); + if (OB_FAIL(hash_for_each_serialize(*this, ser_cb))) { + DUP_TABLE_LOG(WARN, "serialize dup tablet hash map faild", K(ret)); + } else { + tmp_pos = ser_cb.get_pos(); + } } if (OB_SUCC(ret)) { @@ -215,13 +212,24 @@ int DupTabletChangeMap::deserialize(const char *buf, const int64_t data_len, int int64_t deser_time = ObTimeUtility::fast_current_time(); int64_t tmp_pos = pos; - TabletsDeSerCallBack deser_cb(buf, data_len, tmp_pos, deser_time); - if (OB_FAIL(this->clear())) { - DUP_TABLE_LOG(WARN, "clear dup tablet hash map faild", K(ret)); - } else if (OB_FAIL(hash_for_each_deserialize(*this, deser_cb))) { - DUP_TABLE_LOG(WARN, "deserialize dup tablet hash map faild", K(ret)); - } else { - tmp_pos = deser_cb.get_pos(); + + if (OB_SUCC(ret)) { + if (OB_FAIL(dup_set_attr_.deserialize(buf, data_len, tmp_pos))) { + + DUP_TABLE_LOG(WARN, "deserialize dup tablet attribute failed", K(ret), K(tmp_pos), + K(dup_set_attr_)); + } + } + + if (OB_SUCC(ret)) { + TabletsDeSerCallBack deser_cb(buf, data_len, tmp_pos, deser_time); + if (OB_FAIL(this->clear())) { + DUP_TABLE_LOG(WARN, "clear dup tablet hash map faild", K(ret)); + } else if (OB_FAIL(hash_for_each_deserialize(*this, deser_cb))) { + DUP_TABLE_LOG(WARN, "deserialize dup tablet hash map faild", K(ret)); + } else { + tmp_pos = deser_cb.get_pos(); + } } if (OB_SUCC(ret)) { @@ -237,151 +245,60 @@ int64_t DupTabletChangeMap::get_serialize_size() const TabletsGetSizeCallBack get_size_cb; serialize_size += hash_for_each_serialize_size(*this, get_size_cb); + serialize_size += dup_set_attr_.get_serialize_size(); return serialize_size; } -bool DupTabletChangeLogTail::is_valid() const { return readable_version_.is_valid(); } - -int DupTabletLog::serialize(char *buf, const int64_t buf_len, int64_t &pos) const +int TabletsSerCallBack::operator()( + const common::hash::HashMapPair &hash_pair) { - int ret = OB_SUCCESS; - int64_t tmp_pos = pos; - - if (OB_FAIL(common_header_.serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize dup tablet set header faild", K(ret), K(buf_len), K(tmp_pos), - K(pos)); - - } else if (!common_header_.is_valid()) { - ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "invalid common header", K(ret), K(common_header_), K(buf_len), K(tmp_pos), - K(pos)); - } else if (common_header_.is_readable_set()) { - if (OB_FAIL(hash_map_->serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize readable hash map failed", K(ret), KPC(this), K(buf_len), - K(tmp_pos), K(pos)); - } - } else { - if (!change_tail_.is_valid()) { - ret = OB_INVALID_ARGUMENT; - DUP_TABLE_LOG(WARN, "invalid change header", K(ret), KPC(this)); - } else if (OB_FAIL(hash_map_->serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize new/old hash map failed", K(ret), KPC(this), K(buf_len), - K(tmp_pos), K(pos)); - } else if (OB_FAIL(change_tail_.serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize change header failed", K(ret), KPC(this), K(buf_len), - K(tmp_pos), K(pos)); - } - } - - if (OB_SUCC(ret)) { - pos = tmp_pos; - } - return ret; + return hash_pair.first.serialize(buf_, buf_len_, pos_); } -int DupTabletLog::deserialize_common_header(const char *buf, const int64_t data_len, int64_t &pos) +int TabletsDeSerCallBack::operator()(DupTabletChangeMap &dup_tablet_map) { int ret = OB_SUCCESS; + ObTabletID tablet_id; + DupTabletInfo tmp_info; + tmp_info.update_dup_schema_ts_ = deser_time_; - int64_t tmp_pos = pos; - - if (OB_FAIL(common_header_.deserialize(buf, data_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "deserialize dup tablet set header faild", K(ret), K(data_len), K(tmp_pos), - K(pos)); - - } else if (!common_header_.is_valid()) { - - ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "invalid common header", K(ret), K(common_header_), K(data_len), K(tmp_pos), - K(pos)); - } - - if (OB_SUCC(ret)) { - pos = tmp_pos; - } - return ret; -} - -int DupTabletLog::deserialize_content(const char *buf, const int64_t data_len, int64_t &pos) -{ - int ret = OB_SUCCESS; - - int64_t tmp_pos = pos; - - if (!common_header_.is_valid() || OB_ISNULL(hash_map_)) { - ret = OB_INVALID_ARGUMENT; - DUP_TABLE_LOG(WARN, "invalid arguments", K(ret), KP(hash_map_), K(common_header_)); - } else if (hash_map_->get_common_header().get_unique_id() != common_header_.get_unique_id()) { - ret = OB_INVALID_ARGUMENT; - DUP_TABLE_LOG(WARN, "invalid hash map", K(ret), K(hash_map_->get_common_header()), - K(common_header_)); - } else { - - if (common_header_.is_readable_set()) { - if (OB_FAIL(hash_map_->deserialize(buf, data_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize readable hash map failed", K(ret), KPC(this), K(data_len), - K(tmp_pos), K(pos)); - } - } else { - if (OB_FAIL(hash_map_->deserialize(buf, data_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize new/old hash map failed", K(ret), KPC(this), K(data_len), - K(tmp_pos), K(pos)); - } else if (OB_FAIL(change_tail_.deserialize(buf, data_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize change header failed", K(ret), KPC(this), K(data_len), - K(tmp_pos), K(pos)); - } else if (!change_tail_.is_valid()) { - ret = OB_INVALID_ARGUMENT; - DUP_TABLE_LOG(WARN, "invalid change header", K(ret), KPC(this)); - } else { - DUP_TABLE_LOG(DEBUG, "deser tablet end", K(ret), K(change_tail_)); - } - } - } - - if (OB_SUCC(ret)) { - pos = tmp_pos; - } - return ret; -} - -int64_t DupTabletLog::get_serialize_size() -{ - int64_t max_size = 0; - - if (OB_NOT_NULL(hash_map_)) { - max_size += common_header_.get_serialize_size(); - max_size += hash_map_->get_serialize_size(); - if (!common_header_.is_readable_set()) { - max_size += change_tail_.get_serialize_size(); - } - } else { - DUP_TABLE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected error"); - } - - return max_size; -} - -int DupTabletLog::set_hash_map_ptr(DupTabletChangeMap *hash_map_ptr, DupTabletSpecialOpArg *arg_ptr) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(hash_map_ptr)) { - ret = OB_INVALID_ARGUMENT; - DUP_TABLE_LOG(WARN, "invalid hash map", K(ret), KP(hash_map_ptr)); - } else if (hash_map_ptr->get_common_header().get_unique_id() != common_header_.get_unique_id()) { - ret = OB_INVALID_ARGUMENT; - DUP_TABLE_LOG(WARN, "unexpected hash map", K(ret), K(hash_map_ptr->get_common_header()), - K(common_header_)); - } else { - hash_map_ = hash_map_ptr; + if (OB_FAIL(tablet_id.deserialize(buf_, buf_len_, pos_))) { + DUP_TABLE_LOG(WARN, "deserialize tablet id failed", K(ret)); + } else if (OB_FAIL(dup_tablet_map.set_refactored(tablet_id, tmp_info, 1))) { + DUP_TABLE_LOG(WARN, "insert tablet failed", K(ret), K(tablet_id), K(tmp_info)); } return ret; } -const DupTabletCommonHeader &DupTabletLog::get_common_header() { return common_header_; } +int64_t TabletsGetSizeCallBack::operator()( + const common::hash::HashMapPair &hash_pair) +{ + return hash_pair.first.get_serialize_size(); +} -const DupTabletChangeLogTail &DupTabletLog::get_change_tail() { return change_tail_; } +int DupTabletLogBody::deserialize_dup_attribute(const char *buf, + const int64_t data_len, + int64_t pos, + DupTabletSetAttribute &deser_attr) +{ + int ret = OB_SUCCESS; + + int64_t version = 0; + int64_t body_size = 0; + + if (OB_FAIL(serialization::decode(buf, data_len, pos, version))) { + DUP_TABLE_LOG(WARN, "deserialize UNIS_VERSION error", K(ret), K(deser_attr), K(pos), + K(version)); + } else if (OB_FAIL(serialization::decode(buf, data_len, pos, body_size))) { + DUP_TABLE_LOG(WARN, "deserialize body_size error", K(ret), K(deser_attr), K(pos), K(body_size)); + } else if (OB_FAIL( + DupTabletChangeMap::deserialize_set_attribute(buf, data_len, pos, deser_attr))) { + DUP_TABLE_LOG(WARN, "deserialize dup set attribute", K(ret), K(deser_attr)); + } + return ret; +} //********************************************************************** //****** ObLSDupTabletsMgr @@ -396,8 +313,8 @@ int ObLSDupTabletsMgr::init(ObDupTableLSHandler *dup_ls_handle) ret = OB_INIT_TWICE; } else if (OB_FAIL(init_free_tablet_pool_())) { DUP_TABLE_LOG(WARN, "init tablet change set failed", K(ret)); - } else if (OB_FAIL(op_arg_map_.create(8, "DupSpecOp"))) { - DUP_TABLE_LOG(WARN, "create spec op failed", K(ret)); + // } else if (OB_FAIL(op_arg_map_.create(8, "DupSpecOp"))) { + // DUP_TABLE_LOG(WARN, "create spec op failed", K(ret)); } else { ATOMIC_STORE(&is_stopped_, false); ls_id_ = dup_ls_handle->get_ls_id(); @@ -621,7 +538,7 @@ int ObLSDupTabletsMgr::find_dup_tablet_in_set(const common::ObTabletID &tablet_i if (OB_SUCC(ret) && !is_dup_table) { DupTabletChangeMap *old_tablet_set = nullptr; - DupTabletCommonHeader target_common_header; + DupTabletSetCommonHeader target_common_header; target_common_header.set_old(); target_common_header.set_invalid_unique_id(); @@ -669,7 +586,7 @@ int ObLSDupTabletsMgr::gc_dup_tablets(const int64_t gc_ts, const int64_t max_tas // run gc now if (OB_FAIL(ERRSIM_DUP_TABLE_GC_RIGHT_NOW)) { ret = OB_SUCCESS; - last_gc_succ_time_ = gc_ts - GC_DUP_TABLETS_TIME_INTERVAL; + last_gc_succ_time_ = gc_ts - GC_DUP_TABLETS_TIME_INTERVAL; DUP_TABLE_LOG(WARN, "use errsim to invoke gc", KR(ret), K(last_gc_succ_time_), K(gc_ts), K(max_task_interval)); } @@ -699,7 +616,7 @@ int ObLSDupTabletsMgr::gc_dup_tablets(const int64_t gc_ts, const int64_t max_tas * Gc readable tablet set * */ DupTabletChangeMap *old_tablet_set = nullptr; - DupTabletCommonHeader old_tablet_common_header; + DupTabletSetCommonHeader old_tablet_common_header; old_tablet_common_header.set_old(); old_tablet_common_header.set_invalid_unique_id(); if (OB_FAIL(ret)) { @@ -733,7 +650,7 @@ int ObLSDupTabletsMgr::gc_dup_tablets(const int64_t gc_ts, const int64_t max_tas * Gc new tablet set * */ DupTabletChangeMap *changing_new_set = nullptr; - DupTabletCommonHeader new_tablet_common_header; + DupTabletSetCommonHeader new_tablet_common_header; new_tablet_common_header.set_new(); new_tablet_common_header.set_invalid_unique_id(); if (OB_FAIL(ret)) { @@ -871,7 +788,7 @@ int ObLSDupTabletsMgr::prepare_serialize(int64_t &max_ser_size, if (OB_SUCC(ret)) { DupTabletChangeMap *old_tablet_set = nullptr; - DupTabletCommonHeader old_tablets_header; + DupTabletSetCommonHeader old_tablets_header; old_tablets_header.set_old(); old_tablets_header.set_invalid_unique_id(); if (OB_FAIL(get_target_tablet_set_(old_tablets_header, old_tablet_set))) { @@ -935,57 +852,19 @@ int ObLSDupTabletsMgr::serialize_tablet_log(const DupTabletSetIDArray &unique_id } else { for (int i = 0; i < unique_id_array.count() && OB_SUCC(ret); i++) { DupTabletChangeMap *tablet_set_ptr = nullptr; - const DupTabletCommonHeader &seralize_common_header = unique_id_array.at(i); + const DupTabletSetCommonHeader &seralize_common_header = unique_id_array.at(i); if (OB_FAIL(get_target_tablet_set_(seralize_common_header, tablet_set_ptr))) { DUP_TABLE_LOG(WARN, "get target tablet set failed", K(ret), K(i), KPC(tablet_set_ptr)); - } else if (OB_FAIL(seralize_common_header.serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize common header failed", K(ret), K(seralize_common_header)); - } else if (seralize_common_header.is_readable_set()) { - // DupTabletLog readable_log(tablet_set_ptr); - // if (OB_FAIL(readable_log.serialize(buf, buf_len, tmp_pos))) { - // DUP_TABLE_LOG(WARN, "serialize readable log failed", K(ret)); - // } - DupTabletCommonLogBody readable_log(*tablet_set_ptr); - if (OB_FAIL(readable_log.serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize readable tablet log failed", K(ret), KPC(tablet_set_ptr)); - } - } else if (seralize_common_header.is_new_set() || seralize_common_header.is_old_set()) { - if (seralize_common_header.no_specail_op()) { - DupTabletChangeLogBody change_log(*tablet_set_ptr); - if (OB_FAIL(change_log.serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize new/old tablet log failed", K(ret), KPC(tablet_set_ptr)); - } + } else if (OB_ISNULL(tablet_set_ptr)) { + ret = OB_ERR_UNEXPECTED; + DUP_TABLE_LOG(WARN, "tablet set ptr is null", K(ret), KPC(this), K(seralize_common_header), + K(i)); - } else { - DupTabletSpecialOpArg *sp_op_arg = nullptr; - uint64_t uid = seralize_common_header.get_unique_id(); - if (OB_ISNULL(sp_op_arg = op_arg_map_.get(uid))) { - ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "get special op arg failed", K(ret), KPC(sp_op_arg)); - } else { - DupTabletSpecialOpLogBody sp_op_log(*tablet_set_ptr, *sp_op_arg); - if (OB_FAIL(sp_op_log.serialize(buf, buf_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "serialize special op log failed", K(ret), KPC(tablet_set_ptr), - KPC(sp_op_arg)); - } - } + } else { + DupTabletLogBody log_body(*tablet_set_ptr); + if (OB_FAIL(log_body.serialize(buf, buf_len, tmp_pos))) { + DUP_TABLE_LOG(WARN, "serialize common header failed", K(ret), K(seralize_common_header)); } - // DupTabletChangeLogTail log_tail(tablet_set_ptr->get_change_status()->readable_version_, - // tablet_set_ptr->get_change_status()->has_confirmed()); - // DupTabletSpecialOpArg sp_op_arg; - // if (!tablet_set_ptr->get_common_header().no_specail_op()) { - // if (OB_FAIL(op_arg_map_.get_refactored( - // tablet_set_ptr->get_common_header().get_unique_id(), sp_op_arg))) { - // DUP_TABLE_LOG(WARN, "get special op failed", K(ret), KPC(tablet_set_ptr), - // K(sp_op_arg)); - // } - // } - // - // DupTabletLog change_log(log_tail, tablet_set_ptr, &sp_op_arg); - // if (OB_FAIL(ret)) { - // } else if (OB_FAIL(change_log.serialize(buf, buf_len, tmp_pos))) { - // DUP_TABLE_LOG(WARN, "serialize tablet change log failed", K(ret)); - // } } } @@ -1015,146 +894,81 @@ int ObLSDupTabletsMgr::deserialize_tablet_log(DupTabletSetIDArray &unique_id_arr DUP_TABLE_LOG(WARN, "invalid argument", K(ret), KP(buf), K(data_len), K(pos)); } else { // DupTabletLog tablet_log; - DupTabletCommonHeader deser_common_header; + DupTabletSetAttribute deser_dup_attr; while (OB_SUCC(ret) && tmp_pos < data_len) { - deser_common_header.reset(); + deser_dup_attr.reset(); DupTabletChangeMap *tablet_set_ptr = nullptr; bool construct_from_free = false; - share::SCN readable_version; - bool deser_has_confirmed = false; + // share::SCN readable_version; + // bool deser_has_confirmed = false; /* * 1. deserialize tablet set common header * 2. find a target tablet set by common header * */ - if (OB_FAIL(deser_common_header.deserialize(buf, data_len, tmp_pos))) { + if (OB_FAIL(DupTabletLogBody::deserialize_dup_attribute(buf, data_len, tmp_pos, + deser_dup_attr))) { DUP_TABLE_LOG(WARN, "deserialize common header failed", K(ret), K(tmp_pos), K(data_len)); - } else if (OB_FAIL(get_target_tablet_set_(deser_common_header, tablet_set_ptr, + } else if (OB_FAIL(get_target_tablet_set_(deser_dup_attr.common_header_, tablet_set_ptr, true /*construct_target_set*/))) { - DUP_TABLE_LOG(WARN, "get target tablet set failed", K(ret), K(deser_common_header), + DUP_TABLE_LOG(WARN, "get target tablet set failed", K(ret), K(deser_dup_attr), KPC(tablet_set_ptr)); - } else if (tablet_set_ptr->get_common_header().is_free()) { - tablet_set_ptr->get_common_header().copy_tablet_set_type(deser_common_header); - construct_from_free = true; } + if (OB_FAIL(ret)) { // do nothing - } else if (OB_FAIL(unique_id_array.push_back(deser_common_header))) { + } else if (OB_FAIL(unique_id_array.push_back(deser_dup_attr.common_header_))) { DUP_TABLE_LOG(WARN, "push back unique_id into logging array failed", K(ret), - K(deser_common_header)); - } else if (deser_common_header.is_free()) { - /* - * free a empty readable tablet set - * */ - if (!tablet_set_ptr->get_common_header().is_readable_set()) { - ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "can not free a tablet_set in changing", K(ret)); - } else if (OB_FAIL(return_tablet_set(tablet_set_ptr))) { - DUP_TABLE_LOG(WARN, "free a readable set because of compact", K(ret), - KPC(tablet_set_ptr)); - } - DUP_TABLE_LOG(INFO, "deserialize a free tablet set", K(ret), K(tablet_set_ptr)); - } else if (deser_common_header.is_readable_set()) { - /* - * deserialize readable tablet set - * */ - if (construct_from_free) { - if (false == readable_tablets_list_.add_last(tablet_set_ptr)) { - if (OB_FAIL(return_tablet_set(tablet_set_ptr))) { - DUP_TABLE_LOG(WARN, "return tablet set failed", K(ret), KPC(tablet_set_ptr)); - } - // rewrite ret code + K(deser_dup_attr)); + } else { + DupTabletSetCommonHeader origin_common_header = tablet_set_ptr->get_common_header(); + DupTabletLogBody dup_log_body(*tablet_set_ptr); + if (OB_FAIL(dup_log_body.deserialize(buf, data_len, tmp_pos))) { + DUP_TABLE_LOG(WARN, "deserialize dup log body failed", K(ret), K(dup_log_body)); + } else if (deser_dup_attr.common_header_.is_free()) { + /* + * free a empty readable tablet set + * */ + if (!origin_common_header.is_readable_set()) { ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "push back into readable_tablets_list_ failed", K(ret), + DUP_TABLE_LOG(WARN, "can not free a tablet_set in changing", K(ret)); + } else if (OB_FAIL(return_tablet_set(tablet_set_ptr))) { + DUP_TABLE_LOG(WARN, "free a readable set because of compact", K(ret), KPC(tablet_set_ptr)); } - } - if (OB_SUCC(ret)) { - DupTabletCommonLogBody readable_log_body(*tablet_set_ptr); - if (OB_FAIL(readable_log_body.deserialize(buf, data_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "deserialize dup tablet readable log failed", K(ret)); - } - } - } else { - // DUP_TABLE_LOG(INFO, "deser a change set", K(ret), K(tablet_log.get_common_header()), - // KPC(tablet_set_ptr)); - DupTabletSpecialOpArg tmp_op_arg; - if (construct_from_free && deser_common_header.is_new_set() - && (false == need_confirm_new_queue_.add_last(tablet_set_ptr))) { - ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "push back into need_confirm_new_queue_ failed", K(ret), + DUP_TABLE_LOG(INFO, "deserialize a free tablet set", K(ret), K(origin_common_header), KPC(tablet_set_ptr)); - return_tablet_set(tablet_set_ptr); - } else if (deser_common_header.is_old_set() && removing_old_set_ != tablet_set_ptr) { - ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "unexpected old tablets set ptr", K(ret), KPC(tablet_set_ptr), - KPC(removing_old_set_)); + } else if (deser_dup_attr.common_header_.is_readable_set()) { + /* + * deserialize readable tablet set + * */ + if (origin_common_header.is_free()) { + if (false == readable_tablets_list_.add_last(tablet_set_ptr)) { + if (OB_FAIL(return_tablet_set(tablet_set_ptr))) { + DUP_TABLE_LOG(WARN, "return tablet set failed", K(ret), KPC(tablet_set_ptr)); + } + // rewrite ret code + ret = OB_ERR_UNEXPECTED; + DUP_TABLE_LOG(WARN, "push back into readable_tablets_list_ failed", K(ret), + K(origin_common_header), KPC(tablet_set_ptr)); + } + } } else { - if (deser_common_header.no_specail_op()) { - /* - * deserialize new/old tablet set without special op - * */ - DupTabletChangeLogBody change_log_body(*tablet_set_ptr); - if (OB_FAIL(change_log_body.deserialize(buf, data_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "deserialize new/old tablet log failed", K(ret), - K(change_log_body), KPC(tablet_set_ptr)); - } else { - readable_version = change_log_body.get_change_tail().readable_version_; - deser_has_confirmed = change_log_body.get_change_tail().has_confirmed_; - } - } else { - /* - * deserialize special op arg - * */ - DupTabletSpecialOpArg tmp_op_arg; - DupTabletSpecialOpLogBody sp_op_log_body(*tablet_set_ptr, tmp_op_arg); - if (OB_FAIL(sp_op_log_body.deserialize(buf, data_len, tmp_pos))) { - DUP_TABLE_LOG(WARN, "deserialize new/old tablet log failed", K(ret), - K(sp_op_log_body), KPC(tablet_set_ptr)); - } else if (OB_FAIL(op_arg_map_.set_refactored(deser_common_header.get_unique_id(), - tmp_op_arg, 1))) { - DUP_TABLE_LOG(WARN, "insert into op_arg_map_ failed", K(ret), K(deser_common_header), - K(tmp_op_arg)); - } else { - readable_version = sp_op_log_body.get_change_tail().readable_version_; - deser_has_confirmed = sp_op_log_body.get_change_tail().has_confirmed_; - } + // DUP_TABLE_LOG(INFO, "deser a change set", K(ret), K(tablet_log.get_common_header()), + // KPC(tablet_set_ptr)); + if (origin_common_header.is_free() && deser_dup_attr.common_header_.is_new_set() + && (false == need_confirm_new_queue_.add_last(tablet_set_ptr))) { + ret = OB_ERR_UNEXPECTED; + DUP_TABLE_LOG(WARN, "push back into need_confirm_new_queue_ failed", K(ret), + KPC(tablet_set_ptr)); + return_tablet_set(tablet_set_ptr); + } else if (deser_dup_attr.common_header_.is_old_set() + && removing_old_set_ != tablet_set_ptr) { + ret = OB_ERR_UNEXPECTED; + DUP_TABLE_LOG(WARN, "unexpected old tablets set ptr", K(ret), KPC(tablet_set_ptr), + KPC(removing_old_set_)); } } - - /* - * set tablet set state as tablet_log_submitted - * */ - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(tablet_set_ptr->get_change_status()->prepare_serialize())) { - DUP_TABLE_LOG(WARN, "prepare serialize failed", K(ret), KPC(tablet_set_ptr)); - // } else if (tablet_set_ptr->get_change_status()->is_change_logging() - // && OB_FAIL(tablet_set_ptr->get_change_status()->tablet_change_log_submitted( - // share::SCN::min_scn(), true /*submit_result*/))) { - // DUP_TABLE_LOG(WARN, "set tablet change log submitted failed", K(ret), - // KPC(tablet_set_ptr)); - // } else if (tablet_set_ptr->get_change_status()->is_change_logging() - // && OB_FAIL(tablet_set_ptr->get_change_status()->prepare_confirm( - // share::SCN::min_scn(), true /*sync _result*/))) { - // DUP_TABLE_LOG(WARN, "prepare confirm tablet_set failed", K(ret), - // KPC(tablet_set_ptr)); - // - /* Step 2 : as try_to_confirm_tablets*/ - } else if (OB_FAIL(tablet_set_ptr->get_change_status()->push_need_confirm_scn( - readable_version))) { - DUP_TABLE_LOG(WARN, "set need_confirm_scn_ failed", K(ret), K(readable_version), - KPC(tablet_set_ptr)); - } else if (OB_FAIL( - tablet_set_ptr->get_change_status()->push_readable_scn(readable_version))) { - DUP_TABLE_LOG(WARN, "set readable version failed", K(ret), K(readable_version), - KPC(tablet_set_ptr)); - } else if (tablet_set_ptr->get_change_status()->is_confirming() - && OB_FAIL(tablet_set_ptr->get_change_status()->try_set_confirmed( - deser_has_confirmed))) { - DUP_TABLE_LOG(WARN, "replay confirmed flag failed", K(ret), K(deser_has_confirmed), - KPC(tablet_set_ptr)); - } } // DUP_TABLE_LOG(INFO, "deser tablet log for one set", K(ret), // K(tablet_log.get_common_header()), @@ -1182,7 +996,7 @@ int ObLSDupTabletsMgr::tablet_log_submitted(const bool submit_result, UNUSED(for_replay); for (int i = 0; OB_SUCC(ret) && i < unique_id_array.count(); i++) { - const DupTabletCommonHeader logging_common_header = unique_id_array[i]; + const DupTabletSetCommonHeader logging_common_header = unique_id_array[i]; DupTabletChangeMap *logging_tablet_set = nullptr; if (!logging_common_header.is_valid()) { ret = OB_INVALID_ARGUMENT; @@ -1216,7 +1030,7 @@ int ObLSDupTabletsMgr::tablet_log_synced(const bool sync_result, SpinWLockGuard guard(dup_tablets_lock_); for (int i = 0; OB_SUCC(ret) && i < unique_id_array.count(); i++) { - const DupTabletCommonHeader logging_common_header = unique_id_array[i]; + const DupTabletSetCommonHeader logging_common_header = unique_id_array[i]; DupTabletChangeMap *logging_tablet_set = nullptr; if (!logging_common_header.is_valid()) { @@ -1301,25 +1115,8 @@ int ObLSDupTabletsMgr::cal_single_set_max_ser_size_(DupTabletChangeMap *hash_map ret = OB_INVALID_ARGUMENT; DUP_TABLE_LOG(WARN, "invalid hash map", K(ret)); } else { - int64_t tmp_ser_size = hash_map->get_serialize_size(); - if (hash_map->get_common_header().is_readable_set()) { - DupTabletCommonLogBody common_log_body(*hash_map); - tmp_ser_size += common_log_body.get_serialize_size(); - } else if (hash_map->get_common_header().no_specail_op()) { - DupTabletChangeLogBody change_log_body(*hash_map); - tmp_ser_size += change_log_body.get_serialize_size(); - } else { - uint64_t uid = hash_map->get_common_header().get_unique_id(); - DupTabletSpecialOpArg *op_arg = op_arg_map_.get(uid); - DupTabletSpecialOpLogBody sp_log_body(*hash_map, *op_arg); - - if (OB_ISNULL(op_arg)) { - ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(ERROR, "invalid special op arg", K(ret), KPC(op_arg), KPC(hash_map)); - } else { - tmp_ser_size += sp_log_body.get_serialize_size(); - } - } + DupTabletLogBody log_body(*hash_map); + int64_t tmp_ser_size = log_body.get_serialize_size(); if (OB_FAIL(ret)) { // do nothing @@ -1540,33 +1337,6 @@ int ObLSDupTabletsMgr::try_to_confirm_tablets( return ret; } -// bool ObLSDupTabletsMgr::need_log_tablets() -// { -// bool need_log = false; -// if (!need_confirm_new_queue_.is_empty()) { -// DLIST_FOREACH_NORET(node, need_confirm_new_queue_) -// { -// if (node->get_change_status()->need_log()) { -// need_log = true; -// break; -// } -// } -// } -// if (false == need_log) { -// if (!readable_tablets_list_.is_empty()) { -// need_log = true; -// } else if (removing_old_set_->get_change_status()->need_log()) { -// need_log = true; -// } else if (OB_NOT_NULL(changing_new_set_)) { -// if (changing_new_set_->get_change_status()->need_log()) { -// need_log = true; -// } -// } -// } -// DUP_TABLE_LOG(DEBUG, "need log tablet", K(need_log)); -// return need_log; -// } - void ObLSDupTabletsMgr::print_tablet_diag_info_log(bool is_master) { SpinRLockGuard guard(dup_tablets_lock_); @@ -1646,44 +1416,7 @@ void ObLSDupTabletsMgr::print_tablet_diag_info_log(bool is_master) } } } - // } else { - // // for (int i = 0; i < MAX_NEW_DUP_TABLET_SET_CNT && OB_SUCC(ret); i++) { - // DLIST_FOREACH(need_confirm_set, need_confirm_new_queue_) { - // if (need_confirm_set->size() > 0) { - // - // if (OB_FAIL(::oceanbase::common::databuff_printf( - // tablet_diag_info_log_buf_, TABLET_PRINT_BUF_LEN, tablet_diag_pos, - // "\n%s[%sNew Dup Tablet Set - Replay] unique_id = %lu, tablet_count = %lu, " - // "flag " - // "= %s, change_scn = %s, readable_version = %s, trx_ref_ = %lu", - // DupTableDiagStd::DUP_DIAG_INDENT_SPACE, - // DupTableDiagStd::DUP_DIAG_COMMON_PREFIX, - // need_confirm_set->get_common_header()->unique_id_, - // new_tablets_array_[i].tablet_change_map_.size(), - // get_dup_tablet_flag_str(new_tablets_array_[i].change_status_.flag_), - // to_cstring(new_tablets_array_[i].change_status_.tablet_change_scn_), - // to_cstring(new_tablets_array_[i].change_status_.readable_version_), - // new_tablets_array_[i].change_status_.trx_ref_))) { - // - // _DUP_TABLE_LOG(WARN, - // "%sprint replay new tablet list header failed, ret=%d, ls_id=%lu", - // DupTableDiagStd::DUP_DIAG_COMMON_PREFIX, ret, ls_id.id()); - // } else { - // DiagInfoGenerator diag_info_gen( - // tablet_diag_info_log_buf_, TABLET_PRINT_BUF_LEN, tablet_diag_pos, - // new_tablets_array_[i].tablet_change_map_.get_common_header().unique_id_); - // if (OB_FAIL(hash_for_each_update(new_tablets_array_[i].tablet_change_map_, - // diag_info_gen))) { - // _DUP_TABLE_LOG(WARN, "%sprint replay new tablet list failed, ret=%d, ls_id=%lu", - // DupTableDiagStd::DUP_DIAG_COMMON_PREFIX, ret, ls_id.id()); - // } else { - // tablet_diag_pos = diag_info_gen.get_buf_pos(); - // } - // } - // } - // } - // } - // + // old tablet print if (OB_SUCC(ret) && OB_NOT_NULL(removing_old_set_) && removing_old_set_->size() > 0) { @@ -1764,7 +1497,7 @@ int ObLSDupTabletsMgr::lose_dup_tablet_(const common::ObTabletID &tablet_id) * */ DupTabletInfo tmp_info; - DupTabletCommonHeader changing_new_header; + DupTabletSetCommonHeader changing_new_header; changing_new_header.set_invalid_unique_id(); changing_new_header.set_new(); DupTabletChangeMap *changing_new_map = nullptr; @@ -1786,7 +1519,7 @@ int ObLSDupTabletsMgr::lose_dup_tablet_(const common::ObTabletID &tablet_id) ret = OB_SUCCESS; } - DupTabletCommonHeader old_set_header; + DupTabletSetCommonHeader old_set_header; old_set_header.set_invalid_unique_id(); old_set_header.set_old(); DupTabletChangeMap *old_tablet_set = nullptr; @@ -1871,7 +1604,7 @@ int ObLSDupTabletsMgr::discover_dup_tablet_(const common::ObTabletID &tablet_id, } } - DupTabletCommonHeader old_set_header; + DupTabletSetCommonHeader old_set_header; old_set_header.set_invalid_unique_id(); old_set_header.set_old(); DupTabletChangeMap *old_tablets_ptr = nullptr; @@ -1899,7 +1632,7 @@ int ObLSDupTabletsMgr::discover_dup_tablet_(const common::ObTabletID &tablet_id, if (OB_HASH_NOT_EXIST == ret) { // search a temporary new tablets set and insert it tmp_status.update_dup_schema_ts_ = update_ts; - DupTabletCommonHeader changing_new_set_header; + DupTabletSetCommonHeader changing_new_set_header; changing_new_set_header.set_invalid_unique_id(); changing_new_set_header.set_new(); DupTabletChangeMap *changing_new_map = nullptr; @@ -2025,7 +1758,7 @@ int ObLSDupTabletsMgr::get_free_tablet_set(DupTabletChangeMap *&free_set, const return ret; } -int ObLSDupTabletsMgr::get_target_tablet_set_(const DupTabletCommonHeader &target_common_header, +int ObLSDupTabletsMgr::get_target_tablet_set_(const DupTabletSetCommonHeader &target_common_header, DupTabletChangeMap *&target_set, const bool construct_target_set, const bool need_changing_new_set) @@ -2036,17 +1769,7 @@ int ObLSDupTabletsMgr::get_target_tablet_set_(const DupTabletCommonHeader &targe if (target_set != nullptr) { ret = OB_INVALID_ARGUMENT; DUP_TABLE_LOG(WARN, "invalid argument", K(ret), K(target_common_header), KPC(target_set)); - // } else if (need_changing_new_set) { - // if (OB_ISNULL(changing_new_set_)) { - // if (OB_FAIL(get_free_tablet_set(changing_new_set_))) { - // DUP_TABLE_LOG(WARN, "get free tablet set failed", K(ret)); - // } - // } - // - // if (OB_SUCC(ret)) { - // target_set = changing_new_set_ - // } - } else if (unique_id == DupTabletCommonHeader::INVALID_UNIQUE_ID) { + } else if (unique_id == DupTabletSetCommonHeader::INVALID_UNIQUE_ID) { if (target_common_header.is_free()) { if (OB_FAIL(get_free_tablet_set(target_set, unique_id))) { DUP_TABLE_LOG(WARN, "get free tablet set failed", K(ret)); @@ -2166,12 +1889,6 @@ int ObLSDupTabletsMgr::return_tablet_set(DupTabletChangeMap *need_free_set) need_free_set->reuse(); need_free_set->get_common_header().set_old(); } else { - if (!need_free_set->get_common_header().no_specail_op()) { - if (OB_FAIL( - op_arg_map_.erase_refactored(need_free_set->get_common_header().get_unique_id()))) { - DUP_TABLE_LOG(WARN, "remove from op_arg_map failed", K(ret), KPC(need_free_set)); - } - } if (OB_FAIL(ret)) { } else { need_free_set->reuse(); @@ -2321,13 +2038,12 @@ int ObLSDupTabletsMgr::construct_empty_block_confirm_task_(const int64_t trx_ref int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - DupTabletCommonHeader empty_new_common_header; + DupTabletSetCommonHeader empty_new_common_header; empty_new_common_header.set_invalid_unique_id(); empty_new_common_header.set_free(); - uint64_t block_confirm_uid = DupTabletCommonHeader::INVALID_UNIQUE_ID; + uint64_t block_confirm_uid = DupTabletSetCommonHeader::INVALID_UNIQUE_ID; DupTabletChangeMap *block_confirm_task = nullptr; - DupTabletSpecialOpArg tmp_op; if (trx_ref < 0) { ret = OB_INVALID_ARGUMENT; @@ -2346,9 +2062,6 @@ int ObLSDupTabletsMgr::construct_empty_block_confirm_task_(const int64_t trx_ref } else if (false == need_confirm_new_queue_.add_last(block_confirm_task)) { DUP_TABLE_LOG(WARN, "insert into need_confirm_new_queue_ failed", K(ret), KPC(block_confirm_task), K(need_confirm_new_queue_.get_size())); - } else if (OB_FAIL(op_arg_map_.set_refactored(block_confirm_uid, tmp_op))) { - DUP_TABLE_LOG(WARN, "insert into special op map failed", K(ret), KPC(block_confirm_task), - K(tmp_op)); } } @@ -2363,10 +2076,9 @@ int ObLSDupTabletsMgr::construct_empty_block_confirm_task_(const int64_t trx_ref DUP_TABLE_LOG(WARN, "return block_confirm_task failed", K(tmp_ret), KPC(block_confirm_task)); } } - // if task is null, it would not insert into op_arg_map, do nothing } else { DUP_TABLE_LOG(INFO, "construct empty block confirming set task successfully", K(ret), - KPC(block_confirm_task), K(tmp_op)); + KPC(block_confirm_task)); } return ret; @@ -2384,13 +2096,13 @@ int ObLSDupTabletsMgr::construct_clean_confirming_set_task_() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - DupTabletCommonHeader clean_confirming_common_header; + DupTabletSetCommonHeader clean_confirming_common_header; clean_confirming_common_header.set_invalid_unique_id(); clean_confirming_common_header.set_free(); - uint64_t clean_confirming_uid = DupTabletCommonHeader::INVALID_UNIQUE_ID; + uint64_t clean_confirming_uid = DupTabletSetCommonHeader::INVALID_UNIQUE_ID; DupTabletChangeMap *clean_confirming_task = nullptr; - DupTabletSpecialOpArg tmp_op; + // DupTabletSpecialOpArg tmp_op; if (OB_FAIL(get_target_tablet_set_(clean_confirming_common_header, clean_confirming_task))) { DUP_TABLE_LOG(WARN, "get free_set as empty_new_set", K(ret), KPC(clean_confirming_task), @@ -2410,20 +2122,10 @@ int ObLSDupTabletsMgr::construct_clean_confirming_set_task_() { if (need_confirm_ptr == clean_confirming_task) { // do nothing - // } else if - // (OB_FAIL(tmp_op.op_objects_.push_back(need_confirm_ptr->get_common_header()))) { - // DUP_TABLE_LOG(WARN, "push back into special op arg failed", K(ret), - // KPC(need_confirm_ptr)); } else { need_confirm_ptr->get_change_status()->set_confirm_invalid(); } } - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(op_arg_map_.set_refactored(clean_confirming_uid, tmp_op))) { - DUP_TABLE_LOG(WARN, "insert into special op map failed", K(ret), KPC(clean_confirming_task), - K(tmp_op)); - } } } @@ -2439,10 +2141,9 @@ int ObLSDupTabletsMgr::construct_clean_confirming_set_task_() KPC(clean_confirming_task)); } } - // if task is null, it would not insert into op_arg_map, do nothing } else { DUP_TABLE_LOG(INFO, "construct clean data confirming set task successfully", K(ret), - KPC(clean_confirming_task), K(tmp_op)); + KPC(clean_confirming_task)); } return ret; @@ -2453,13 +2154,13 @@ int ObLSDupTabletsMgr::construct_clean_all_readable_set_task_() int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - DupTabletCommonHeader clean_readable_common_header; + DupTabletSetCommonHeader clean_readable_common_header; clean_readable_common_header.set_invalid_unique_id(); clean_readable_common_header.set_free(); - uint64_t clean_readbale_uid = DupTabletCommonHeader::INVALID_UNIQUE_ID; + uint64_t clean_readable_uid = DupTabletSetCommonHeader::INVALID_UNIQUE_ID; DupTabletChangeMap *clean_readable_task = nullptr; - DupTabletSpecialOpArg tmp_op; + // DupTabletSpecialOpArg tmp_op; if (OB_FAIL(get_target_tablet_set_(clean_readable_common_header, clean_readable_task))) { DUP_TABLE_LOG(WARN, "get free_set as empty_new_set", K(ret), KPC(clean_readable_task), @@ -2467,7 +2168,7 @@ int ObLSDupTabletsMgr::construct_clean_all_readable_set_task_() } else { clean_readable_task->get_common_header().set_new(); clean_readable_task->get_common_header().set_op_of_clean_all_readable_set(); - clean_readbale_uid = clean_readable_task->get_common_header().get_unique_id(); + clean_readable_uid = clean_readable_task->get_common_header().get_unique_id(); // set empty tablet_set as a normal tablet_set which has submit log failed if (OB_FAIL(clean_readable_task->get_change_status()->prepare_serialize())) { DUP_TABLE_LOG(WARN, "prepare serialize for empty_new_set failed", K(ret)); @@ -2475,19 +2176,6 @@ int ObLSDupTabletsMgr::construct_clean_all_readable_set_task_() DUP_TABLE_LOG(WARN, "insert into need_confirm_new_queue_ failed", K(ret), KPC(clean_readable_task), K(need_confirm_new_queue_.get_size())); } else { - // DLIST_FOREACH(readable_ptr, readable_tablets_list_) - // { - // if (OB_FAIL(tmp_op.op_objects_.push_back(readable_ptr->get_common_header()))) { - // DUP_TABLE_LOG(WARN, "push back into special op arg failed", K(ret), - // KPC(clean_readable_task)); - // } - // } - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(op_arg_map_.set_refactored(clean_readbale_uid, tmp_op))) { - DUP_TABLE_LOG(WARN, "insert into special op map failed", K(ret), KPC(clean_readable_task), - K(tmp_op)); - } } } @@ -2503,10 +2191,9 @@ int ObLSDupTabletsMgr::construct_clean_all_readable_set_task_() DUP_TABLE_LOG(WARN, "return clean_readable_task failed", K(ret), KPC(clean_readable_task)); } } - // if task is null, it would not insert into op_arg_map, do nothing } else { DUP_TABLE_LOG(INFO, "construct clean all readable task successfully", K(ret), - KPC(clean_readable_task), K(tmp_op)); + KPC(clean_readable_task)); } return ret; } @@ -2521,38 +2208,19 @@ int ObLSDupTabletsMgr::try_exec_special_op_(DupTabletChangeMap *op_tablet_set, ret = OB_INVALID_ARGUMENT; DUP_TABLE_LOG(WARN, "invalid argument", K(ret), KPC(op_tablet_set)); } else if (op_tablet_set->get_common_header().no_specail_op()) { - // filter no sp op tablet set - } else if (!op_tablet_set->get_change_status()->tablet_change_scn_.is_valid() || - (op_tablet_set->get_change_status()->tablet_change_scn_.is_valid() && - min_reserve_tablet_scn > op_tablet_set->get_change_status()->tablet_change_scn_)) { + // filter no sp op tablet set + } else if (!op_tablet_set->get_change_status()->tablet_change_scn_.is_valid() + || (op_tablet_set->get_change_status()->tablet_change_scn_.is_valid() + && min_reserve_tablet_scn + > op_tablet_set->get_change_status()->tablet_change_scn_)) { // filter those sp op set with invalid change scn or not equal to min reserve scn // do nothing - DUP_TABLE_LOG(INFO, "not need exec sp op", K(ret), K(min_reserve_tablet_scn), KPC(op_tablet_set)); + DUP_TABLE_LOG(INFO, "not need exec sp op", K(ret), K(min_reserve_tablet_scn), + KPC(op_tablet_set)); } else if (op_tablet_set->get_common_header().need_clean_all_readable_set()) { if (OB_FAIL(clean_readable_tablets_(min_reserve_tablet_scn))) { DUP_TABLE_LOG(WARN, "clean readable tablets failed", K(ret), K(min_reserve_tablet_scn)); } - // if (OB_SUCC(ret) && !readable_tablets_list_.is_empty()) { - // DLIST_FOREACH_REMOVESAFE(readable_set_ptr, readable_tablets_list_) - // { - // if (nullptr == readable_tablets_list_.remove(readable_set_ptr)) { - // ret = OB_ERR_UNEXPECTED; - // DUP_TABLE_LOG(WARN, "remove readable_set failed", K(ret), KPC(readable_set_ptr)); - // } else if (OB_FAIL(return_tablet_set(readable_set_ptr))) { - // DUP_TABLE_LOG(WARN, "free readable_set failed", K(ret), KPC(readable_set_ptr), - // K(readable_tablets_list_.get_size()), K(free_set_pool_.get_size())); - // } - // } - // } - } else if (op_tablet_set->get_common_header().need_clean_data_confirming_set()) { - // DupTabletSpecialOpArg tmp_arg; - // DUP_TABLE_LOG(WARN, "try clean unreadable tablets", K(ret), - // K(op_tablet_set->get_common_header())); - // if (OB_FAIL(op_arg_map_.get_refactored(op_tablet_set->get_common_header().get_unique_id(), - // tmp_arg))) { - // DUP_TABLE_LOG(WARN, "get sp op arg failed", K(ret), K(op_tablet_set->get_common_header())); - // // @param1 type clean_all - // } else if (OB_FAIL(clean_durable_confirming_tablets_(min_reserve_tablet_scn))) { DUP_TABLE_LOG(WARN, "clean unreadable tablets failed", K(ret), K(min_reserve_tablet_scn)); } @@ -2563,6 +2231,7 @@ int ObLSDupTabletsMgr::try_exec_special_op_(DupTabletChangeMap *op_tablet_set, return ret; } + // tablet set virtual table interface // all tablets virtual table interface int ObLSDupTabletsMgr::get_tablets_stat(ObDupLSTabletsStatIterator &collect_iter, @@ -2656,7 +2325,7 @@ int ObLSDupTabletsMgr::get_tablets_stat(ObDupLSTabletsStatIterator &collect_iter } int ObLSDupTabletsMgr::get_tablet_set_stat(ObDupLSTabletSetStatIterator &collect_iter, - const share::ObLSID &ls_id) + const share::ObLSID &ls_id) { int ret = OB_SUCCESS; // iter changing new @@ -2708,8 +2377,7 @@ int ObLSDupTabletsMgr::get_tablet_set_stat(ObDupLSTabletSetStatIterator &collect } } else { ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "iter need confirm failed", K(ret), KPC(this), - KP(need_confirm_set)); + DUP_TABLE_LOG(WARN, "iter need confirm failed", K(ret), KPC(this), KP(need_confirm_set)); } } } @@ -2733,8 +2401,7 @@ int ObLSDupTabletsMgr::get_tablet_set_stat(ObDupLSTabletSetStatIterator &collect } } else { ret = OB_ERR_UNEXPECTED; - DUP_TABLE_LOG(WARN, "iter readable failed", K(ret), KPC(this), - KP(readable_set)); + DUP_TABLE_LOG(WARN, "iter readable failed", K(ret), KPC(this), KP(readable_set)); } } } diff --git a/src/storage/tx/ob_dup_table_tablets.h b/src/storage/tx/ob_dup_table_tablets.h index c3a3f2023..70b75e694 100644 --- a/src/storage/tx/ob_dup_table_tablets.h +++ b/src/storage/tx/ob_dup_table_tablets.h @@ -1,4 +1,4 @@ -//Copyrigh (c) 2021 OceanBase +// Copyrigh (c) 2021 OceanBase // OceanBase is licensed under Mulan PubL v2. // You can use this software according to the terms and conditions of the Mulan PubL v2. // You may obtain a copy of Mulan PubL v2 at: @@ -11,14 +11,14 @@ #ifndef OCEANBASE_DUP_TABLE_TABLETS_H #define OCEANBASE_DUP_TABLE_TABLETS_H -#include "lib/list/ob_dlist.h" -#include "lib/queue/ob_fixed_queue.h" #include "common/ob_tablet_id.h" -#include "lib/lock/ob_spin_rwlock.h" #include "lib/hash/ob_hashmap.h" #include "lib/hash/ob_hashset.h" -#include "ob_dup_table_ts_sync.h" +#include "lib/list/ob_dlist.h" +#include "lib/lock/ob_spin_rwlock.h" +#include "lib/queue/ob_fixed_queue.h" #include "ob_dup_table_base.h" +#include "ob_dup_table_ts_sync.h" namespace oceanbase { @@ -35,6 +35,21 @@ class ObDupTableLSHandler; //****** ObLSDupTabletsMgr //********************************************************************** +struct DupTabletInfo +{ + int64_t update_dup_schema_ts_; + + void reset() { update_dup_schema_ts_ = 0; } + + DupTabletInfo() { reset(); } + + TO_STRING_KV(K(update_dup_schema_ts_)); +}; + +typedef common::hash:: + ObHashMap + DupTabletIdMap; + enum class DupTabletSetChangeFlag { UNKNOWN = -1, @@ -81,14 +96,11 @@ static const char *get_dup_tablet_flag_str(const DupTabletSetChangeFlag &flag) return flag_str; } -struct DupTabletSetChangeStatus +class DupTabletSetChangeStatus { - DupTabletSetChangeFlag flag_; - share::SCN tablet_change_scn_; - share::SCN need_confirm_scn_; - share::SCN readable_version_; - int64_t trx_ref_; + OB_UNIS_VERSION(1); +public: void init() { reset(); @@ -105,6 +117,12 @@ struct DupTabletSetChangeStatus } DupTabletSetChangeStatus() { reset(); } + ~DupTabletSetChangeStatus() { reset(); } + + const share::SCN &get_readable_version() { return readable_version_; } + const share::SCN &get_tablet_change_scn() { return tablet_change_scn_; } + const share::SCN &get_need_confirm_scn() { return need_confirm_scn_; } + const DupTabletSetChangeFlag &get_tablet_set_change_falg() { return flag_; } bool is_valid() const { return flag_ != DupTabletSetChangeFlag::UNKNOWN; } bool need_log() const @@ -131,7 +149,7 @@ struct DupTabletSetChangeStatus bool can_be_confirmed_anytime() const { return (trx_ref_ == 0 && readable_version_ >= need_confirm_scn_ - && flag_ == DupTabletSetChangeFlag::CONFIRMING) + && flag_ == DupTabletSetChangeFlag::CONFIRMING) || flag_ == DupTabletSetChangeFlag::CONFIRMED; } bool has_confirmed() const { return DupTabletSetChangeFlag::CONFIRMED == flag_; } @@ -236,60 +254,64 @@ struct DupTabletSetChangeStatus K(need_confirm_scn_), K(readable_version_), K(trx_ref_)); -}; -struct DupTabletInfo -{ - int64_t update_dup_schema_ts_; - - void reset() { update_dup_schema_ts_ = 0; } - - DupTabletInfo() { reset(); } - - TO_STRING_KV(K(update_dup_schema_ts_)); -}; - -typedef common::hash:: - ObHashMap - DupTabletIdMap; - -// class DupTabletHashMap : public DupTabletIdMap -// { -// public: -// NEED_SERIALIZE_AND_DESERIALIZE; -// TO_STRING_KV(K(common_header_), K(size())); -// const DupTabletCommonHeader &get_common_header() { return common_header_; } -// -// int create(int64_t unique_id, int64_t bucket_num, const lib::ObLabel &bucket_label); -// void destroy() -// { -// DupTabletIdMap::destroy(); -// common_header_.reset(); -// } -// -// private: -// DupTabletCommonHeader common_header_; -// }; - -class DupTabletChangeMap : public common::ObDLinkBase, public DupTabletIdMap -{ public: - NEED_SERIALIZE_AND_DESERIALIZE; - DupTabletChangeMap(const uint64_t set_id) : change_status_(), common_header_(set_id) { reuse(); } + DupTabletSetChangeFlag flag_; + share::SCN tablet_change_scn_; + share::SCN need_confirm_scn_; + share::SCN readable_version_; + int64_t trx_ref_; +}; + +struct DupTabletSetAttribute +{ + DupTabletSetCommonHeader common_header_; + DupTabletSetChangeStatus change_status_; void reuse() { + change_status_.init(); change_status_.set_temporary(); // common_header_.clean_sp_op(); // common_header_.set_free(); common_header_.reuse(); + } + + void reset() + { + common_header_.reset(); + change_status_.reset(); + } + DupTabletSetAttribute(const uint64_t uid) : common_header_(uid), change_status_() { reuse(); } + DupTabletSetAttribute() { reset(); } + + TO_STRING_KV(K(common_header_), K(change_status_)); + + OB_UNIS_VERSION(1); +}; + +class DupTabletChangeMap : public common::ObDLinkBase, public DupTabletIdMap +{ +public: + NEED_SERIALIZE_AND_DESERIALIZE; + static int deserialize_set_attribute(const char *buf, + int64_t data_len, + int64_t pos, + DupTabletSetAttribute &deser_attr); + + DupTabletChangeMap(const uint64_t set_id) : dup_set_attr_(set_id) { reuse(); } + + void reuse() + { + dup_set_attr_.reuse(); DupTabletIdMap::clear(); } void destroy() { - reset(); + dup_set_attr_.reset(); + common::ObDLinkBase::reset(); DupTabletIdMap::destroy(); } @@ -298,28 +320,23 @@ public: DupTabletSetChangeStatus *get_change_status() { DupTabletSetChangeStatus *change_status_ptr = nullptr; - if (common_header_.is_readable_set()) { + if (dup_set_attr_.common_header_.is_readable_set()) { change_status_ptr = nullptr; } else { - change_status_ptr = &change_status_; + change_status_ptr = &dup_set_attr_.change_status_; } return change_status_ptr; } - DupTabletCommonHeader &get_common_header() { return common_header_; } + DupTabletSetCommonHeader &get_common_header() { return dup_set_attr_.common_header_; } bool need_reserve(const share::SCN &scn) const { - return change_status_.need_reserve(scn); + return dup_set_attr_.change_status_.need_reserve(scn); } - TO_STRING_KV(K(change_status_), - K(common_header_), - K(DupTabletIdMap::size()), - K(DupTabletIdMap::created())); + TO_STRING_KV(K(dup_set_attr_), K(DupTabletIdMap::size()), K(DupTabletIdMap::created())); private: - DupTabletSetChangeStatus change_status_; - DupTabletCommonHeader common_header_; - // DupTabletIdMap tablet_id_map_; + DupTabletSetAttribute dup_set_attr_; }; class TabletsSerCallBack : public IHashSerCallBack @@ -333,10 +350,7 @@ public: class TabletsDeSerCallBack : public IHashDeSerCallBack { public: - TabletsDeSerCallBack(const char *buf, - int64_t buf_len, - int64_t pos, - int64_t deser_time) + TabletsDeSerCallBack(const char *buf, int64_t buf_len, int64_t pos, int64_t deser_time) : IHashDeSerCallBack(buf, buf_len, pos), deser_time_(deser_time) {} int operator()(DupTabletChangeMap &dup_tablet_map); @@ -351,141 +365,27 @@ public: int64_t operator()(const common::hash::HashMapPair &hash_pair); }; - -class DupTabletChangeLogTail +class DupTabletLogBody { OB_UNIS_VERSION(1); public: - DupTabletChangeLogTail() {} - DupTabletChangeLogTail(const share::SCN &readable_version, bool confirm_all) - : readable_version_(readable_version), has_confirmed_(confirm_all) - {} - bool is_valid() const; - void reset() - { - readable_version_.set_min(); - has_confirmed_ = false; - } - - share::SCN readable_version_; - bool has_confirmed_; - - TO_STRING_KV(K(readable_version_), K(has_confirmed_)); -}; - -typedef ObSEArray OpObjectArr; -class DupTabletSpecialOpArg -{ - OB_UNIS_VERSION(1); + static int deserialize_dup_attribute(const char *buf, + const int64_t data_len, + int64_t pos, + DupTabletSetAttribute &deser_attr); public: - DupTabletSpecialOpArg() {} + DupTabletLogBody(DupTabletChangeMap &hash_map) : hash_map_(hash_map) {} - void reset() { op_objects_.reset(); } - bool is_valid() { return !op_objects_.empty(); } + void reset() {} - TO_STRING_KV(K(op_objects_)); -public: - OpObjectArr op_objects_; -}; + const DupTabletChangeMap &get_cur_map_ref() { return hash_map_; } -typedef common::hash::ObHashMap - SpecialOpArgMap; - -class DupTabletCommonLogBody -{ - OB_UNIS_VERSION(1); - -public: - DupTabletCommonLogBody(DupTabletChangeMap &hash_map) : tablet_id_map_(hash_map) {} - - TO_STRING_KV(K(tablet_id_map_)); - -protected: - DupTabletChangeMap &tablet_id_map_; -}; - -class DupTabletChangeLogBody : public DupTabletCommonLogBody -{ - OB_UNIS_VERSION(1); - -public: - DupTabletChangeLogBody(DupTabletChangeMap &hash_map) : DupTabletCommonLogBody(hash_map) - { - change_tail_.readable_version_ = - DupTabletCommonLogBody::tablet_id_map_.get_change_status()->readable_version_; - change_tail_.has_confirmed_ = tablet_id_map_.get_change_status()->has_confirmed(); - } - - const DupTabletChangeLogTail &get_change_tail() { return change_tail_; } - - INHERIT_TO_STRING_KV("common_log_body", DupTabletCommonLogBody, K(change_tail_)); + TO_STRING_KV(K(hash_map_)); private: - DupTabletChangeLogTail change_tail_; -}; - -class DupTabletSpecialOpLogBody : public DupTabletChangeLogBody -{ - OB_UNIS_VERSION(1); - -public: - DupTabletSpecialOpLogBody(DupTabletChangeMap &hash_map, DupTabletSpecialOpArg &op_arg) - : DupTabletChangeLogBody(hash_map), sp_op_arg_(op_arg) - {} - - INHERIT_TO_STRING_KV("change_log_body", DupTabletChangeLogBody, K(sp_op_arg_)); - -private: - DupTabletSpecialOpArg &sp_op_arg_; -}; - -class DupTabletLog -{ -public: - int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; - int deserialize_common_header(const char *buf, const int64_t data_len, int64_t &pos); - int deserialize_content(const char *buf, const int64_t data_len, int64_t &pos); - - int64_t get_serialize_size(); - -public: - DupTabletLog(DupTabletChangeMap *hash_map) : hash_map_(hash_map) - { - common_header_ = hash_map_->get_common_header(); - } - - DupTabletLog(DupTabletChangeLogTail change_tail, - DupTabletChangeMap *hash_map, - DupTabletSpecialOpArg *sp_op_arg = nullptr) - : hash_map_(hash_map), change_tail_(change_tail), special_op_arg_(sp_op_arg) - { - common_header_ = hash_map_->get_common_header(); - }; - - DupTabletLog() { reset(); }; - - void reset() - { - common_header_.reset(); - hash_map_ = nullptr; - change_tail_.reset(); - special_op_arg_ = nullptr; - } - - int set_hash_map_ptr(DupTabletChangeMap *hash_map_ptr, - DupTabletSpecialOpArg *special_op_arg_ = nullptr); - const DupTabletCommonHeader &get_common_header(); - const DupTabletChangeLogTail &get_change_tail(); - - TO_STRING_KV(K(common_header_), K(change_tail_), K(hash_map_)); - -private: - DupTabletCommonHeader common_header_; - DupTabletChangeMap *hash_map_; - DupTabletChangeLogTail change_tail_; - DupTabletSpecialOpArg *special_op_arg_; + DupTabletChangeMap &hash_map_; }; // *********************************************************************************************** @@ -570,7 +470,8 @@ private: // *********************************************************************************************** // * If move a tablet from old to readable without confirm // * Problem: -// * 1. Leader (A) tablet1(readable->old); submit lease log(log_ts = n); tablet1(old->new->readable); switch_to_follower +// * 1. Leader (A) tablet1(readable->old); submit lease log(log_ts = n); +// tablet1(old->new->readable); switch_to_follower // * Follower (B) replay log n=>tablet1(readable->old); switch_to_leader // * 2. Follower(A) tablet1(readable),replay_ts = n // * Leaser(B) tablet1(old); confirm A replay_ts > n ; tablet1(old->delete) @@ -591,6 +492,7 @@ public: bool is_master() { return ATOMIC_LOAD(&is_master_); } const static int64_t MAX_CONFIRMING_TABLET_COUNT; + public: int check_readable(const common::ObTabletID &tablet_id, bool &readable, @@ -646,10 +548,8 @@ public: // K(need_confirm_new_queue_.get_size()), // K(old_tablets_), // K(readable_tablets_)); - int get_tablets_stat(ObDupLSTabletsStatIterator &collect_iter, - const share::ObLSID &ls_id); - int get_tablet_set_stat(ObDupLSTabletSetStatIterator &collect_iter, - const share::ObLSID &ls_id); + int get_tablets_stat(ObDupLSTabletsStatIterator &collect_iter, const share::ObLSID &ls_id); + int get_tablet_set_stat(ObDupLSTabletSetStatIterator &collect_iter, const share::ObLSID &ls_id); private: class GcDiscardedDupTabletHandler @@ -657,7 +557,7 @@ private: public: GcDiscardedDupTabletHandler(int64_t update_ts, int64_t gc_time_interval, - const DupTabletCommonHeader &common_header, + const DupTabletSetCommonHeader &common_header, DupTabletChangeMap &old_tablets) : gc_ts_(update_ts), gc_time_interval_(gc_time_interval), gc_tablet_cnt_(0), ret_(OB_SUCCESS), src_common_header_(common_header), old_tablets_(old_tablets) @@ -671,7 +571,7 @@ private: int64_t gc_time_interval_; int64_t gc_tablet_cnt_; int ret_; - DupTabletCommonHeader src_common_header_; + DupTabletSetCommonHeader src_common_header_; DupTabletChangeMap &old_tablets_; }; @@ -726,9 +626,9 @@ private: const TabletSetAttr attr, // const int64_t tablet_gc_window, ObDupLSTabletsStatIterator &collect_iter) - : collect_ts_(collect_ts),ls_id_(ls_id), tenant_id_(tenant_id), addr_(addr), + : collect_ts_(collect_ts), ls_id_(ls_id), tenant_id_(tenant_id), addr_(addr), is_master_(is_master), tablet_set_id_(tablet_set_id), attr_(attr), - //tablet_gc_window_(tablet_gc_window) + // tablet_gc_window_(tablet_gc_window) collect_iter_(collect_iter) {} int operator()(const common::hash::HashMapPair &hash_pair); @@ -759,7 +659,7 @@ private: int get_free_tablet_set(DupTabletChangeMap *&free_set, const uint64_t target_id = 0); // If get a free tablet set, need set tablet set type and push into queue - int get_target_tablet_set_(const DupTabletCommonHeader &target_common_header, + int get_target_tablet_set_(const DupTabletSetCommonHeader &target_common_header, DupTabletChangeMap *&target_set, const bool construct_target_set = false, const bool need_changing_new_set = false); @@ -767,8 +667,8 @@ private: int check_and_recycle_empty_readable_set(DupTabletChangeMap *need_free_set, bool &need_remove); int return_tablet_set(DupTabletChangeMap *need_free_set); - int clean_readable_tablets_(const share::SCN & min_reserve_tablet_scn); - int clean_durable_confirming_tablets_(const share::SCN & min_reserve_tablet_scn); + int clean_readable_tablets_(const share::SCN &min_reserve_tablet_scn); + int clean_durable_confirming_tablets_(const share::SCN &min_reserve_tablet_scn); int clean_unlog_tablets_(); int construct_empty_block_confirm_task_(const int64_t trx_ref); int search_special_op_(uint64_t special_op_type); @@ -790,7 +690,7 @@ private: // static int64_t GC_DUP_TABLETS_TIME_INTERVAL; // 5 min static int64_t GC_DUP_TABLETS_FAILED_TIMEOUT; // 25 min - const static int64_t GC_TIMEOUT; // 1s + const static int64_t GC_TIMEOUT; // 1s const static int64_t RESERVED_FREE_SET_COUNT; const static int64_t MAX_FREE_SET_COUNT; @@ -822,8 +722,6 @@ private: common::ObDList readable_tablets_list_; DupTabletChangeMap *removing_old_set_; - SpecialOpArgMap op_arg_map_; - // gc_dup_table int64_t last_gc_succ_time_; @@ -845,6 +743,7 @@ public: ObTabletIDArray &get_array() { return array_; } const ObTabletIDArray &get_array() const { return array_; } void set_ls_id(const share::ObLSID &ls_id) { ls_id_ = ls_id; } + private: share::ObLSID ls_id_; ObTabletIDArray array_; @@ -854,16 +753,19 @@ class ObTenantDupTabletSchemaHelper { public: typedef common::hash::ObHashSet TabletIDSet; + public: - ObTenantDupTabletSchemaHelper() {} + ObTenantDupTabletSchemaHelper() {} + public: int refresh_and_get_tablet_set(TabletIDSet &tenant_dup_tablet_set); + private: - int get_all_dup_tablet_set_(TabletIDSet & tablets_set); + int get_all_dup_tablet_set_(TabletIDSet &tablets_set); + private: }; - } // namespace transaction } // namespace oceanbase diff --git a/src/storage/tx/ob_dup_table_util.cpp b/src/storage/tx/ob_dup_table_util.cpp index 03fe6e45e..ef3d35aec 100644 --- a/src/storage/tx/ob_dup_table_util.cpp +++ b/src/storage/tx/ob_dup_table_util.cpp @@ -91,7 +91,7 @@ int ObDupTabletScanTask::refresh_dup_tablet_schema_( if (OB_FAIL(ls_status_op.get_duplicate_ls_status_info(MTL_ID(), *GCTX.sql_proxy_, dup_ls_status_info))) { if (OB_ENTRY_NOT_EXIST == ret) { - DUP_TABLE_LOG(WARN, "no duplicate ls", K(dup_ls_status_info)); + DUP_TABLE_LOG(DEBUG, "no duplicate ls", K(dup_ls_status_info)); } else { DUP_TABLE_LOG(WARN, "get duplicate ls status info failed", K(ret), K(dup_ls_status_info)); } diff --git a/src/storage/tx/ob_trans_define.cpp b/src/storage/tx/ob_trans_define.cpp index 2c1605e1e..9c0e2a875 100644 --- a/src/storage/tx/ob_trans_define.cpp +++ b/src/storage/tx/ob_trans_define.cpp @@ -1018,7 +1018,6 @@ void ObTxExecInfo::reset() prepare_log_info_arr_.reset(); xid_.reset(); need_checksum_ = true; - tablet_modify_record_.reset(); is_sub2pc_ = false; } @@ -1058,7 +1057,6 @@ OB_SERIALIZE_MEMBER(ObTxExecInfo, prepare_log_info_arr_, xid_, need_checksum_, - tablet_modify_record_, is_sub2pc_); bool ObMulSourceDataNotifyArg::is_redo_submitted() const { return redo_submitted_; } diff --git a/src/storage/tx/ob_trans_define.h b/src/storage/tx/ob_trans_define.h index 688e0c544..37b4d7357 100644 --- a/src/storage/tx/ob_trans_define.h +++ b/src/storage/tx/ob_trans_define.h @@ -1670,7 +1670,6 @@ public: // for xa ObXATransID xid_; bool need_checksum_; - common::ObSEArray tablet_modify_record_; bool is_sub2pc_; }; diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 4b620f691..04b3ffbad 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -6314,8 +6314,6 @@ int ObPartTransCtx::search_unsubmitted_dup_table_redo_() if (ls_tx_ctx_mgr_->get_ls_log_adapter()->has_dup_tablet()) { if (OB_FAIL(submit_log_impl_(ObTxLogType::TX_COMMIT_INFO_LOG))) { TRANS_LOG(WARN, "submit commit info log failed", K(ret), KPC(this)); - // } else if (OB_FAIL(check_tablet_modify_record_())) { - // TRANS_LOG(WARN, "check the modify tablet failed", K(ret)); } } return ret; @@ -6438,16 +6436,20 @@ int ObPartTransCtx::check_dup_trx_with_submitting_all_redo(ObTxLogBlock &log_blo // c. return OB_EAGAIN if (ls_tx_ctx_mgr_->get_ls_log_adapter()->has_dup_tablet()) { if (!sub_state_.is_info_log_submitted() && get_downstream_state() < ObTxState::REDO_COMPLETE) { - - ret = submit_pending_log_block_(log_block, helper); - - TRANS_LOG(INFO, "submit all redo log for dup table check", K(ret), - K(exec_info_.tablet_modify_record_.count()), KPC(this)); - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_FAIL(check_tablet_modify_record_())) { - TRANS_LOG(WARN, "check the modify tablet failed", K(ret)); + if (exec_info_.redo_lsns_.count() > 0 || exec_info_.prev_record_lsn_.is_valid() + || !helper.callbacks_.is_empty()) { + set_dup_table_tx_(); } + + // ret = submit_pending_log_block_(log_block, helper); + // + // TRANS_LOG(INFO, "submit all redo log for dup table check", K(ret), + // K(exec_info_.tablet_modify_record_.count()), KPC(this)); + // if (OB_FAIL(ret)) { + // // do nothing + // } else if (OB_FAIL(check_tablet_modify_record_())) { + // TRANS_LOG(WARN, "check the modify tablet failed", K(ret)); + // } } } else { @@ -6485,65 +6487,65 @@ int ObPartTransCtx::dup_table_tx_pre_commit_() return ret; } -int ObPartTransCtx::merge_tablet_modify_record_(const common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; +// int ObPartTransCtx::merge_tablet_modify_record_(const common::ObTabletID &tablet_id) +// { +// int ret = OB_SUCCESS; +// +// if (exec_info_.tablet_modify_record_.count() >= MAX_TABLET_MODIFY_RECORD_COUNT) { +// // do nothing +// } else { +// bool is_contain = false; +// for (int i = 0; i < exec_info_.tablet_modify_record_.count(); i++) { +// if (exec_info_.tablet_modify_record_[i] == tablet_id) { +// is_contain = true; +// } +// } +// if (!is_contain && OB_FAIL(exec_info_.tablet_modify_record_.push_back(tablet_id))) { +// TRANS_LOG(WARN, "push back tablet id failed", K(ret), K(tablet_id), +// K(exec_info_.tablet_modify_record_)); +// } +// } +// +// return ret; +// } - if (exec_info_.tablet_modify_record_.count() >= MAX_TABLET_MODIFY_RECORD_COUNT) { - // do nothing - } else { - bool is_contain = false; - for (int i = 0; i < exec_info_.tablet_modify_record_.count(); i++) { - if (exec_info_.tablet_modify_record_[i] == tablet_id) { - is_contain = true; - } - } - if (!is_contain && OB_FAIL(exec_info_.tablet_modify_record_.push_back(tablet_id))) { - TRANS_LOG(WARN, "push back tablet id failed", K(ret), K(tablet_id), - K(exec_info_.tablet_modify_record_)); - } - } - - return ret; -} - -int ObPartTransCtx::check_tablet_modify_record_() -{ - int ret = OB_SUCCESS; - - if (!exec_info_.is_dup_tx_) { - bool has_dup_tablet = false; - if (!ls_tx_ctx_mgr_->get_ls_log_adapter()->has_dup_tablet()) { - has_dup_tablet = false; - TRANS_LOG(INFO, "no dup tablet in this ls", K(has_dup_tablet), K(trans_id_), K(ls_id_)); - } else if (exec_info_.tablet_modify_record_.count() >= MAX_TABLET_MODIFY_RECORD_COUNT) { - has_dup_tablet = true; - TRANS_LOG(INFO, "too much tablet, consider it as a dup trx", K(ret), K(has_dup_tablet), - K(exec_info_.tablet_modify_record_), KPC(this)); - } else { - has_dup_tablet = false; - for (int i = 0; i < exec_info_.tablet_modify_record_.count(); i++) { - - if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->check_dup_tablet_in_redo( - exec_info_.tablet_modify_record_[i], has_dup_tablet, share::SCN::min_scn(), - share::SCN::max_scn()))) { - TRANS_LOG(WARN, "check dup tablet failed", K(ret), K(trans_id_), K(ls_id_), - K(exec_info_.tablet_modify_record_[i])); - } else if (has_dup_tablet) { - TRANS_LOG(INFO, "modify a dup tablet, consider it as a dup trx", K(ret), - K(has_dup_tablet), K(exec_info_.tablet_modify_record_[i]), - K(exec_info_.tablet_modify_record_.count()), KPC(this)); - break; - } - } - } - if (has_dup_tablet) { - set_dup_table_tx_(); - } - } - - return ret; -} +// int ObPartTransCtx::check_tablet_modify_record_() +// { +// int ret = OB_SUCCESS; +// +// if (!exec_info_.is_dup_tx_) { +// bool has_dup_tablet = false; +// if (!ls_tx_ctx_mgr_->get_ls_log_adapter()->has_dup_tablet()) { +// has_dup_tablet = false; +// TRANS_LOG(INFO, "no dup tablet in this ls", K(has_dup_tablet), K(trans_id_), K(ls_id_)); +// } else if (exec_info_.tablet_modify_record_.count() >= MAX_TABLET_MODIFY_RECORD_COUNT) { +// has_dup_tablet = true; +// TRANS_LOG(INFO, "too much tablet, consider it as a dup trx", K(ret), K(has_dup_tablet), +// K(exec_info_.tablet_modify_record_), KPC(this)); +// } else { +// has_dup_tablet = false; +// for (int i = 0; i < exec_info_.tablet_modify_record_.count(); i++) { +// +// if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->check_dup_tablet_in_redo( +// exec_info_.tablet_modify_record_[i], has_dup_tablet, share::SCN::min_scn(), +// share::SCN::max_scn()))) { +// TRANS_LOG(WARN, "check dup tablet failed", K(ret), K(trans_id_), K(ls_id_), +// K(exec_info_.tablet_modify_record_[i])); +// } else if (has_dup_tablet) { +// TRANS_LOG(INFO, "modify a dup tablet, consider it as a dup trx", K(ret), +// K(has_dup_tablet), K(exec_info_.tablet_modify_record_[i]), +// K(exec_info_.tablet_modify_record_.count()), KPC(this)); +// break; +// } +// } +// } +// if (has_dup_tablet) { +// set_dup_table_tx_(); +// } +// } +// +// return ret; +// } int ObPartTransCtx::clear_dup_table_redo_sync_result_() { @@ -6599,24 +6601,24 @@ int ObPartTransCtx::retry_dup_trx_before_prepare(const share::SCN &before_prepar return ret; } -int ObPartTransCtx::merge_tablet_modify_record(const common::ObTabletID &tablet_id) -{ - int ret = OB_SUCCESS; - - CtxLockGuard guard(lock_); - - if (is_exiting_) { - // ret = OB_TRANS_CTX_NOT_EXIST; - TRANS_LOG(WARN, "merge tablet modify record into a exiting part_ctx", K(ret), K(tablet_id), - KPC(this)); - } else if (!is_follower_()) { - ret = OB_NOT_FOLLOWER; - TRANS_LOG(WARN, "can not invoke on leader", K(ret), K(tablet_id), KPC(this)); - } else if (OB_FAIL(merge_tablet_modify_record_(tablet_id))) { - TRANS_LOG(WARN, "merge tablet modify record failed", K(ret)); - } - return ret; -} +// int ObPartTransCtx::merge_tablet_modify_record(const common::ObTabletID &tablet_id) +// { +// int ret = OB_SUCCESS; +// +// CtxLockGuard guard(lock_); +// +// if (is_exiting_) { +// // ret = OB_TRANS_CTX_NOT_EXIST; +// TRANS_LOG(WARN, "merge tablet modify record into a exiting part_ctx", K(ret), K(tablet_id), +// KPC(this)); +// } else if (!is_follower_()) { +// ret = OB_NOT_FOLLOWER; +// TRANS_LOG(WARN, "can not invoke on leader", K(ret), K(tablet_id), KPC(this)); +// } else if (OB_FAIL(merge_tablet_modify_record_(tablet_id))) { +// TRANS_LOG(WARN, "merge tablet modify record failed", K(ret)); +// } +// return ret; +// } int ObPartTransCtx::sub_prepare(const ObLSArray &parts, const MonotonicTs &commit_time, diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_trans_part_ctx.h index d378b6df1..074a5806f 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_trans_part_ctx.h @@ -294,7 +294,7 @@ public: // newly added for 4.0 int retry_dup_trx_before_prepare(const share::SCN &before_prepare_version); - int merge_tablet_modify_record(const common::ObTabletID &tablet_id); + // int merge_tablet_modify_record(const common::ObTabletID &tablet_id); int set_scheduler(const common::ObAddr &scheduler); const common::ObAddr &get_scheduler() const; int on_success(ObTxLogCb *log_cb); @@ -475,8 +475,8 @@ private: memtable::ObRedoLogSubmitHelper &helper); bool is_dup_table_redo_sync_completed_(); int dup_table_tx_pre_commit_(); - int merge_tablet_modify_record_(const common::ObTabletID &tablet_id); - int check_tablet_modify_record_(); + // int merge_tablet_modify_record_(const common::ObTabletID &tablet_id); + // int check_tablet_modify_record_(); void set_dup_table_tx_() { exec_info_.is_dup_tx_ = true; diff --git a/src/storage/tx/ob_tx_log.cpp b/src/storage/tx/ob_tx_log.cpp index a39496f2b..89daaac75 100644 --- a/src/storage/tx/ob_tx_log.cpp +++ b/src/storage/tx/ob_tx_log.cpp @@ -1368,7 +1368,7 @@ int ObTxLogBlock::update_next_log_pos_() if (OB_FAIL(serialization::decode(replay_buf_, len_, tmp_pos, version))) { TRANS_LOG(WARN, "deserialize UNIS_VERSION error", K(ret), K(*this), K(tmp_pos), K(version)); } else if (OB_FAIL(serialization::decode(replay_buf_, len_, tmp_pos, body_size))) { - TRANS_LOG(WARN, "deserialize body_size error", K(ret), K(*this), K(tmp_pos), K(version)); + TRANS_LOG(WARN, "deserialize body_size error", K(ret), K(*this), K(tmp_pos), K(body_size)); } else if (tmp_pos + body_size > len_) { ret = OB_SIZE_OVERFLOW; TRANS_LOG(WARN, "has not enough space for deserializing tx_log_body", K(body_size), diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index aef9d95b3..7605d1718 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -557,8 +557,6 @@ int ObTxReplayExecutor::replay_redo_in_memtable_(ObTxRedoLog &redo) } } else if (FALSE_IT(row_head = mmi_ptr_->get_row_head())) { // do nothing - } else if (OB_NOT_NULL(ctx_) && OB_FAIL(ctx_->merge_tablet_modify_record(row_head.tablet_id_))) { - TRANS_LOG(WARN, "record tablet_id in redo failed", K(ret), K(row_head)); } else if (OB_FAIL(replay_one_row_in_memtable_(row_head, mmi_ptr_))) { if (OB_MINOR_FREEZE_NOT_ALLOW == ret) { if (TC_REACH_TIME_INTERVAL(1000 * 1000)) {