From f00f696510f88bfe59a7ac4ef0e6b885761424ea Mon Sep 17 00:00:00 2001 From: obdev Date: Wed, 18 Sep 2024 08:17:06 +0000 Subject: [PATCH] add column for tablet_replica_checksum and tablet_compaction_history --- deps/oblib/src/lib/ob_define.h | 1 + ..._all_virtual_tablet_compaction_history.cpp | 30 ++ ...ob_all_virtual_tablet_compaction_history.h | 7 +- .../freeze/ob_checksum_validator.cpp | 3 +- .../ob_compaction_locality_cache.cpp | 347 +----------------- .../compaction/ob_compaction_locality_cache.h | 51 --- .../ob_inner_table_schema.11101_11150.cpp | 75 ++++ .../ob_inner_table_schema.12251_12300.cpp | 19 + .../ob_inner_table_schema.15201_15250.cpp | 75 ++++ .../ob_inner_table_schema.15451_15500.cpp | 240 ++++++------ .../ob_inner_table_schema.21201_21250.cpp | 4 +- .../ob_inner_table_schema.28101_28150.cpp | 4 +- .../ob_inner_table_schema.351_400.cpp | 19 + .../inner_table/ob_inner_table_schema_def.py | 38 +- .../ob_tablet_replica_checksum_operator.cpp | 88 +++-- .../ob_tablet_replica_checksum_operator.h | 7 +- src/share/tablet/ob_tablet_info.cpp | 18 + src/share/tablet/ob_tablet_info.h | 20 + .../blocksstable/ob_major_checksum_info.cpp | 2 +- src/storage/column_store/ob_co_merge_ctx.cpp | 8 +- src/storage/column_store/ob_co_merge_ctx.h | 2 +- .../ob_column_oriented_sstable.cpp | 27 +- .../column_store/ob_column_oriented_sstable.h | 6 +- .../compaction/ob_basic_tablet_merge_ctx.cpp | 16 +- .../compaction/ob_medium_compaction_func.cpp | 36 +- .../compaction/ob_medium_compaction_func.h | 5 +- .../compaction/ob_sstable_merge_history.cpp | 12 + .../compaction/ob_sstable_merge_history.h | 12 +- .../compaction/ob_tenant_medium_checker.cpp | 12 +- .../compaction/ob_tenant_medium_checker.h | 3 +- .../high_availability/ob_storage_ha_utils.cpp | 30 +- src/storage/tablet/ob_tablet.cpp | 8 +- src/storage/tablet/ob_tablet.h | 2 +- .../r/mysql/desc_sys_views_in_mysql.result | 10 + .../r/mysql/desc_sys_views_in_sys.result | 10 + .../mysql/desc_virtual_table_in_mysql.result | 5 + .../r/mysql/desc_virtual_table_in_sys.result | 6 + unittest/share/compaction/CMakeLists.txt | 3 +- .../compaction/test_checksum_operator.cpp | 84 +++++ 39 files changed, 695 insertions(+), 650 deletions(-) create mode 100644 unittest/share/compaction/test_checksum_operator.cpp diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 12774fd2d..f0fbd2a2d 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -1499,6 +1499,7 @@ const int64_t OB_MERGE_COMMENT_INNER_STR_LENGTH = 800; const int64_t OB_CKM_ERROR_INFO_STR_LENGTH = 1024; const int64_t OB_COMPACTION_STATUS_STR_LENGTH = 256; const int64_t OB_STORAGE_PATH_STR_LENGTH = 256; +const int64_t OB_MERGE_REASON_STR_LENGTH = 32; // for erasure code const int64_t OB_MAX_EC_STRIPE_COUNT = 32; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.cpp b/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.cpp index 21dcf0acc..9bd446ab5 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.cpp @@ -245,6 +245,36 @@ int ObAllVirtualTabletCompactionHistory::process_curr_tenant(ObNewRow *&row) } cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); break; + case IS_FULL_MERGE: + cells[i].set_bool(static_info.is_full_merge_); + break; + case IO_COST_TIME_PERCENTAGE: + cells[i].set_int(running_info.io_percentage_); + break; + case MERGE_REASON: + if (ObAdaptiveMergePolicy::is_valid_merge_reason(static_info.merge_reason_)) { + cells[i].set_varchar(ObAdaptiveMergePolicy::merge_reason_to_str(static_info.merge_reason_)); + } else { + cells[i].set_varchar(""); + } + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + case BASE_MAJOR_STATUS: + if (is_valid_co_major_sstable_status(static_info.base_major_status_)) { + cells[i].set_varchar(co_major_sstable_status_to_str(static_info.base_major_status_)); + } else { + cells[i].set_varchar(""); + } + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; + case CO_MERGE_TYPE: + if (ObCOMajorMergePolicy::is_valid_major_merge_type(static_info.co_major_merge_type_)) { + cells[i].set_varchar(ObCOMajorMergePolicy::co_major_merge_type_to_str(static_info.co_major_merge_type_)); + } else { + cells[i].set_varchar(""); + } + cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + break; default: ret = OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "invalid column id", K(ret), K(col_id)); diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.h b/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.h index cfc8190b9..dcc52cef0 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_compaction_history.h @@ -59,7 +59,12 @@ public: END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, - EXEC_MODE + EXEC_MODE, + IS_FULL_MERGE, + IO_COST_TIME_PERCENTAGE, + MERGE_REASON, + BASE_MAJOR_STATUS, + CO_MERGE_TYPE }; ObAllVirtualTabletCompactionHistory(); virtual ~ObAllVirtualTabletCompactionHistory(); diff --git a/src/rootserver/freeze/ob_checksum_validator.cpp b/src/rootserver/freeze/ob_checksum_validator.cpp index 6e08fb084..295d5d468 100755 --- a/src/rootserver/freeze/ob_checksum_validator.cpp +++ b/src/rootserver/freeze/ob_checksum_validator.cpp @@ -427,8 +427,7 @@ int ObChecksumValidator::verify_tablet_replica_checksum() if (OB_UNLIKELY(replica_ckm_items_.empty())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(replica_ckm_items_)); - } else if (OB_FAIL(ObMediumCompactionScheduleFunc::check_replica_checksum_items( - replica_ckm_items_, ls_locality_cache_.get_cs_replica_cache(), false /*is_medium_checker*/))) { + } else if (OB_FAIL(ObMediumCompactionScheduleFunc::check_replica_checksum_items(replica_ckm_items_, false /*is_medium_checker*/))) { LOG_WARN("failed to verify tablet replica checksum", K(ret)); } return ret; diff --git a/src/share/compaction/ob_compaction_locality_cache.cpp b/src/share/compaction/ob_compaction_locality_cache.cpp index 8b56d4589..2a30a5a03 100644 --- a/src/share/compaction/ob_compaction_locality_cache.cpp +++ b/src/share/compaction/ob_compaction_locality_cache.cpp @@ -23,347 +23,12 @@ namespace oceanbase namespace share { -/****************************** ObLSReplicaUniItem ******************************/ -ObLSReplicaUniItem::ObLSReplicaUniItem() - : ls_id_(), - server_() -{} - -ObLSReplicaUniItem::ObLSReplicaUniItem(const ObLSID &ls_id, const common::ObAddr &server) - : ls_id_(ls_id), - server_(server) -{} - -ObLSReplicaUniItem::~ObLSReplicaUniItem() -{ - reset(); -} - -void ObLSReplicaUniItem::reset() { - ls_id_.reset(); - server_.reset(); -} - -uint64_t ObLSReplicaUniItem::hash() const -{ - uint64_t hash_val = 0; - hash_val += ls_id_.hash(); - hash_val += server_.hash(); - return hash_val; -} - -int ObLSReplicaUniItem::hash(uint64_t &hash_val) const -{ - hash_val = hash(); - return OB_SUCCESS; -} - -bool ObLSReplicaUniItem::is_valid() const -{ - return ls_id_.is_valid() && server_.is_valid(); -} - -bool ObLSReplicaUniItem::operator == (const ObLSReplicaUniItem &other) const -{ - bool bret = true; - if (this == &other) { - } else if (ls_id_ != other.ls_id_ || server_ != other.server_) { - bret = false; - } - return bret; -} - -bool ObLSReplicaUniItem::operator != (const ObLSReplicaUniItem &other) const -{ - return !(*this == other); -} - -/****************************** ObLSColumnReplicaCache ******************************/ -ObLSColumnReplicaCache::ObLSColumnReplicaCache() - : is_inited_(false), - ls_id_set_(), - ls_replica_set_(), - ls_infos_() -{} - -ObLSColumnReplicaCache::~ObLSColumnReplicaCache() -{ - destroy(); -} - -// init -int ObLSColumnReplicaCache::init() -{ - int ret = OB_SUCCESS; - if (IS_INIT) { - ret = OB_INIT_TWICE; - LOG_WARN("init twice", K(ret)); - } else if (OB_FAIL(ls_id_set_.create(BUCKET_NUM_OF_LS_ID_SET, ObMemAttr(MTL_ID(), "LSIDsForCkm")))) { - LOG_WARN("fail to create ls id set", K(ret)); - } else if (OB_FAIL(ls_replica_set_.create(BUCKET_NUM_OF_LS_REPLICA_SET, ObMemAttr(MTL_ID(), "LSReplTypes")))) { - LOG_WARN("fail to create ls replica type map", K(ret)); - } else { - is_inited_ = true; - } - return ret; -} - -void ObLSColumnReplicaCache::destroy() -{ - int ret = OB_SUCCESS; // only for log - if (ls_replica_set_.created() && OB_FAIL(ls_replica_set_.destroy())) { - LOG_WARN("fail to destroy ls replica set", K(ret)); - } - if (ls_id_set_.created() && OB_FAIL(ls_id_set_.destroy())) { - LOG_WARN("fail to destroy ls replica set", K(ret)); - } - ls_infos_.reset(); - is_inited_ = false; -} - -void ObLSColumnReplicaCache::reuse() -{ - ls_id_set_.reuse(); - ls_replica_set_.reuse(); - ls_infos_.reuse(); -} - -int ObLSColumnReplicaCache::check_contains_ls(const ObLSID &ls_id, bool &contained) const -{ - int ret = OB_SUCCESS; - contained = false; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(ls_id_set_.exist_refactored(ls_id))) { - if (OB_HASH_EXIST == ret || OB_HASH_NOT_EXIST == ret) { - contained = (OB_HASH_EXIST == ret); - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to check contains ls", K(ret), K(ls_id), KPC(this)); - } - } - return ret; -} - -int ObLSColumnReplicaCache::mark_ls_finished(const ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(ls_id_set_.set_refactored(ls_id))) { - if (OB_HASH_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to mark ls finish", K(ret), K(ls_id), KPC(this)); - } - } - return ret; -} - -int ObLSColumnReplicaCache::add_cs_replica(const ObLSReplicaUniItem &ls_item) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(ls_replica_set_.set_refactored(ls_item))) { - LOG_WARN("fail to add col replica", K(ret), K(ls_item), KPC(this)); - } - return ret; -} - -int ObLSColumnReplicaCache::assign(const ObLSColumnReplicaCache &other) -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(other.deep_fetch(ls_id_set_, ls_replica_set_, ls_infos_))) { - LOG_WARN("fail to assign", K(ret), K(other)); - } - return ret; -} - -int ObLSColumnReplicaCache::deep_fetch( - hash::ObHashSet &target_ls_id_set, - hash::ObHashSet &target_ls_replica_set, - common::ObIArray &target_ls_infos) const -{ - int ret = OB_SUCCESS; - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else { - target_ls_id_set.reuse(); - target_ls_replica_set.reuse(); - target_ls_infos.reuse(); - - for (hash::ObHashSet::const_iterator it = ls_id_set_.begin(); OB_SUCC(ret) && it != ls_id_set_.end(); ++it) { - if (OB_FAIL(target_ls_id_set.set_refactored(it->first))) { - LOG_WARN("fail to add ls id", K(ret)); - } - } - for (hash::ObHashSet::const_iterator it = ls_replica_set_.begin(); OB_SUCC(ret) && it != ls_replica_set_.end(); ++it) { - if (OB_FAIL(target_ls_replica_set.set_refactored(it->first))) { - LOG_WARN("fail to add ls replica", K(ret)); - } - } - for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_infos_.count(); ++idx) { - if (OB_FAIL(target_ls_infos.push_back(ls_infos_.at(idx)))) { - LOG_WARN("fail to push back ls info", K(ret), K(idx), K_(ls_infos)); - } - } - } - return ret; -} - - -int ObLSColumnReplicaCache::update(const ObLSID &ls_id) -{ - int ret = OB_SUCCESS; - const int64_t cluster_id = GCONF.cluster_id; - bool is_contained = false; - ObLSInfo ls_info; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(check_contains_ls(ls_id, is_contained))) { - LOG_WARN("fail to check exist for ls", K(ls_id), KPC(this)); - } else if (is_contained) { - // do nothing - } else if (OB_ISNULL(GCTX.lst_operator_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("lst operator is null", K(ret)); - } else if (OB_FAIL(GCTX.lst_operator_->get(cluster_id, MTL_ID(), ls_id, ObLSTable::DEFAULT_MODE, ls_info))) { - LOG_WARN("fail to get ls info", K(ret), K(ls_id), K(cluster_id), K(MTL_ID())); - } else if (OB_FAIL(update_with_ls_info(ls_info))) { - LOG_WARN("failed to try add cs replica info", K(ret), K(ls_id), K(ls_info)); - } - return ret; -} - -int ObLSColumnReplicaCache::update_with_ls_info(const ObLSInfo &ls_info) -{ - int ret = OB_SUCCESS; - const ObLSID &ls_id = ls_info.get_ls_id(); - bool replica_has_leader = false; - - for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_info.get_replicas().count(); ++idx) { - const ObLSReplica &replica = ls_info.get_replicas().at(idx); - if (ObRole::LEADER == replica.get_role()) { - replica_has_leader = true; - } - - if (!replica.is_column_replica()) { - // do nothing - } else if (OB_FAIL(add_cs_replica(ObLSReplicaUniItem(ls_id, replica.get_server())))) { - LOG_WARN("failed to add cs replica", K(ret), K(ls_id), K(replica), K(ls_info)); - } else { - LOG_INFO("success to add cs replica", K(ls_id), K(replica), K(ls_info)); - } - } - - if (OB_FAIL(ret)) { - } else if (!replica_has_leader) { - ret = OB_LEADER_NOT_EXIST; - LOG_WARN("ls has no leader, ls column replica cache cannot read", K(ret), K(ls_id)); - } else if (OB_FAIL(mark_ls_finished(ls_id))) { - LOG_WARN("fail to make ls finished", K(ret)); - } else if (OB_FAIL(ls_infos_.push_back(ls_info))) { - LOG_WARN("failed to add ls info", K(ret), K(ls_info)); - } else { - LOG_INFO("success to update with ls info", KPC(this)); // debug log, remove later - } - return ret; -} - -int ObLSColumnReplicaCache::check_can_skip( - const ObLSReplicaUniItem &ls_item, - bool &can_skip) const -{ - int ret = OB_SUCCESS; - can_skip = false; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret), KPC(this)); - } else if (OB_UNLIKELY(!ls_item.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_item)); - } else { - const ObLSInfo *ls_info = nullptr; - for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_infos_.count(); ++idx) { - if (ls_item.ls_id_ == ls_infos_.at(idx).get_ls_id()) { - ls_info = &ls_infos_.at(idx); - break; - } - } - if (OB_ISNULL(ls_info)) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("ls info not found", K(ret), K(ls_item), K(ls_infos_)); - } else { - const ObLSInfo::ReplicaArray &replicas = ls_info->get_replicas(); - for (int64_t idx = 0; OB_SUCC(ret) && idx < replicas.count(); ++idx) { - const ObLSReplica &curr_replica = replicas.at(idx); - ObMember learner; - - if (ObRole::LEADER != curr_replica.get_role()) { - continue; // follower replica, do nothing - } else if (curr_replica.server_is_in_member_list(curr_replica.get_member_list(), ls_item.server_)) { - // item is in member list, need to check - } else if (OB_FAIL(curr_replica.get_learner_list().get_learner_by_addr(ls_item.server_, learner))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - can_skip = true; // item not in member list && not in learner list, skip to check - } else { - LOG_WARN("faile to get learner", K(ret), K(ls_item), KPC(this)); - } - } else { - // both R replica and CS replica need to check - } - break; // found leader replica, stop checking another replica - } - } - } - return ret; -} - -int ObLSColumnReplicaCache::check_is_cs_replica(const ObLSReplicaUniItem &ls_item, bool &is_cs_replica) const -{ - int ret = OB_SUCCESS; - is_cs_replica = false; - bool is_contained = false; - - if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("not inited", K(ret)); - } else if (OB_FAIL(check_contains_ls(ls_item.ls_id_, is_contained))) { - LOG_WARN("fail to check exist for ls", K(ls_item), KPC(this)); - } else if (OB_UNLIKELY(!is_contained)) { - ret = OB_LEADER_NOT_EXIST; - LOG_WARN("ls has no leader, ls column replica cache cannot read", K(ret)); - } else if (OB_FAIL(ls_replica_set_.exist_refactored(ls_item))) { - if (OB_HASH_EXIST == ret || OB_HASH_NOT_EXIST == ret) { - is_cs_replica = (OB_HASH_EXIST == ret); - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to check contains ls", K(ret), K(ls_item), KPC(this)); - } - } - LOG_TRACE("[CS-Replica] check current ls is cs replica", K(ret), K(ls_item), K(is_cs_replica), KPC(this)); - return ret; -} - /****************************** ObCompactionLocalityCache ******************************/ ObCompactionLocalityCache::ObCompactionLocalityCache() : is_inited_(false), tenant_id_(OB_INVALID_TENANT_ID), merge_info_mgr_(nullptr), - ls_infos_map_(), - ls_cs_replica_cache_() + ls_infos_map_() {} ObCompactionLocalityCache::~ObCompactionLocalityCache() @@ -382,8 +47,6 @@ int ObCompactionLocalityCache::init(const uint64_t tenant_id, rootserver::ObMajo LOG_WARN("invalid argument", K(ret), K(tenant_id)); } else if (OB_FAIL(ls_infos_map_.create(OB_MAX_LS_NUM_PER_TENANT_PER_SERVER, "CaLsInfoMap", "CaLsInfoNode", tenant_id))) { LOG_WARN("fail to create ls info map", K(ret)); - } else if (OB_FAIL(ls_cs_replica_cache_.init())) { - LOG_WARN("fail to init col replica cache", K(ret)); } if (OB_FAIL(ret)) { destroy(); @@ -435,13 +98,11 @@ int ObCompactionLocalityCache::inner_refresh_ls_locality() if (OB_FAIL(ret)) { } else if (OB_UNLIKELY(zone_list.empty())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("zone list is empty, skip get ls locality", K(ret), K_(tenant_id)); + LOG_INFO("zone list is empty, skip get ls locality", K(ret), K_(tenant_id)); MTL(compaction::ObDiagnoseTabletMgr *)->add_diagnose_tablet(UNKNOW_LS_ID, UNKNOW_TABLET_ID, ObDiagnoseTabletType::TYPE_MEDIUM_MERGE); } else { // 1. clear ls_infos cached in memory ls_infos_map_.reuse(); - ls_cs_replica_cache_.reuse(); // 2. load ls_infos from __all_ls_meta_table ObArray ls_infos; ls_infos.set_attr(ObMemAttr(tenant_id_, "RefLSInfos")); @@ -575,10 +236,8 @@ int ObCompactionLocalityCache::refresh_by_zone( } if (FAILEDx(ls_infos_map_.set_refactored(ls_id, tmp_ls_info, 1/*overwrite*/))) { LOG_WARN("fail to set refactored", KR(ret), K(ls_id), K(tmp_ls_info)); - } else if (OB_FAIL(ls_cs_replica_cache_.update_with_ls_info(tmp_ls_info))) { - LOG_WARN("failed to update with ls info", K(ret), K(tmp_ls_info)); } else { - FLOG_INFO("success to refresh cached ls_info", K(ret), K(tmp_ls_info), K(zone_list), K_(ls_cs_replica_cache)); + FLOG_INFO("success to refresh cached ls_info", K(ret), K(tmp_ls_info), K(zone_list)); } } return ret; diff --git a/src/share/compaction/ob_compaction_locality_cache.h b/src/share/compaction/ob_compaction_locality_cache.h index 48bfa8316..1e3ac0f76 100644 --- a/src/share/compaction/ob_compaction_locality_cache.h +++ b/src/share/compaction/ob_compaction_locality_cache.h @@ -29,55 +29,6 @@ class ObMySQLResult; namespace share { -struct ObLSReplicaUniItem -{ - ObLSReplicaUniItem(); - ObLSReplicaUniItem(const ObLSID &ls_id, const common::ObAddr &server); - ~ObLSReplicaUniItem(); - void reset(); - uint64_t hash() const; - int hash(uint64_t &hash_val) const; - bool is_valid() const; - bool operator == (const ObLSReplicaUniItem &other) const; - bool operator != (const ObLSReplicaUniItem &other) const; - TO_STRING_KV(K_(ls_id), K_(server)); - - share::ObLSID ls_id_; - common::ObAddr server_; -}; - -class ObLSColumnReplicaCache -{ -public: - ObLSColumnReplicaCache(); - ~ObLSColumnReplicaCache(); - int init(); - void destroy(); - void reuse(); - int assign(const ObLSColumnReplicaCache &other); - int deep_fetch( - hash::ObHashSet &target_ls_id_set, - hash::ObHashSet &target_ls_replica_set, - common::ObIArray &target_ls_infos) const; - int update(const ObLSID &ls_id); - int update_with_ls_info(const ObLSInfo &ls_info); - int check_is_cs_replica(const ObLSReplicaUniItem &ls_item, bool &is_cs_replica) const; - int check_can_skip(const ObLSReplicaUniItem &ls_item, bool &can_skip) const; - TO_STRING_KV(K_(is_inited), K_(ls_id_set), K_(ls_replica_set), K_(ls_infos)); -private: - int check_contains_ls(const ObLSID &ls_id, bool &contained) const; - int mark_ls_finished(const ObLSID &ls_id); - int add_cs_replica(const ObLSReplicaUniItem &ls_item); -private: - const static int64_t BUCKET_NUM_OF_LS_ID_SET = 15; - const static int64_t BUCKET_NUM_OF_LS_REPLICA_SET = 31; -private: - bool is_inited_; - hash::ObHashSet ls_id_set_; // record looped ls id - hash::ObHashSet ls_replica_set_; // cs-prelica ls - common::ObSEArray ls_infos_; // used for check member list and learner list -}; - class ObCompactionLocalityCache { public: @@ -88,7 +39,6 @@ public: bool empty() const { return ls_infos_map_.empty(); } int refresh_ls_locality(const bool force_refresh); int get_ls_info(const share::ObLSID &ls_id, share::ObLSInfo &ls_info); - const share::ObLSColumnReplicaCache& get_cs_replica_cache() const { return ls_cs_replica_cache_; } TO_STRING_KV(K_(is_inited), K_(tenant_id)); private: const int64_t CHECK_LS_LOCALITY_INTERVAL = 5 * 60 * 1000 * 1000L; // 5 mins @@ -111,7 +61,6 @@ private: uint64_t tenant_id_; rootserver::ObMajorMergeInfoManager *merge_info_mgr_; common::hash::ObHashMap ls_infos_map_; - share::ObLSColumnReplicaCache ls_cs_replica_cache_; }; struct ObMemberListInfo diff --git a/src/share/inner_table/ob_inner_table_schema.11101_11150.cpp b/src/share/inner_table/ob_inner_table_schema.11101_11150.cpp index 231ae8940..9e601ed17 100644 --- a/src/share/inner_table/ob_inner_table_schema.11101_11150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.11101_11150.cpp @@ -2772,6 +2772,81 @@ int ObInnerTableSchema::all_virtual_tablet_compaction_history_schema(ObTableSche false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("is_full_merge", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObTinyIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + 1, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("io_cost_time_percentage", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("merge_reason", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MERGE_REASON_STR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("base_major_status", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MERGE_TYPE_STR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("co_merge_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_INVALID, //column_collation_type + OB_MERGE_TYPE_STR_LENGTH, //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp index d62d7d62b..def2bce78 100644 --- a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp @@ -2267,6 +2267,25 @@ int ObInnerTableSchema::all_virtual_tablet_replica_checksum_schema(ObTableSchema true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj data_checksum_type_default; + data_checksum_type_default.set_int(0); + ADD_COLUMN_SCHEMA_T("data_checksum_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + data_checksum_type_default, + data_checksum_type_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp index d82049283..2caecd233 100644 --- a/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15201_15250.cpp @@ -4523,6 +4523,81 @@ int ObInnerTableSchema::all_virtual_tablet_compaction_history_ora_schema(ObTable false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("IS_FULL_MERGE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("IO_COST_TIME_PERCENTAGE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("MERGE_REASON", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MERGE_REASON_STR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("BASE_MAJOR_STATUS", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MERGE_TYPE_STR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CO_MERGE_TYPE", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + OB_MERGE_TYPE_STR_LENGTH, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp b/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp index ac44aba6b..243d98855 100644 --- a/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp +++ b/src/share/inner_table/ob_inner_table_schema.15451_15500.cpp @@ -2463,126 +2463,6 @@ int ObInnerTableSchema::all_virtual_vector_index_info_ora_schema(ObTableSchema & false, //is_nullable false); //is_autoincrement } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("FILE_PTR", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 20, //column_length - 2, //column_precision - -1, //column_scale - false, //is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("FILE_LABEL", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObVarcharType, //column_type - CS_TYPE_UTF8MB4_BIN, //column_collation_type - 16, //column_length - 2, //column_precision - -1, //column_scale - true, //is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("META_TREE_EPOCH", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false, //is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("META_TREE_LEVELS", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false, //is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("META_BYTES", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false, //is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("CACHED_META_PAGE_NUM", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false, //is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("WRITE_BACK_META_PAGE_NUM", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false, //is_nullable - false); //is_autoincrement - } - - if (OB_SUCC(ret)) { - ADD_COLUMN_SCHEMA("PAGE_FLUSH_CNT", //column_name - ++column_id, //column_id - 0, //rowkey_id - 0, //index_id - 0, //part_key_pos - ObNumberType, //column_type - CS_TYPE_INVALID, //column_collation_type - 38, //column_length - 38, //column_precision - 0, //column_scale - false, //is_nullable - false); //is_autoincrement - } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); @@ -3173,6 +3053,126 @@ int ObInnerTableSchema::all_virtual_temp_file_ora_schema(ObTableSchema &table_sc false, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FILE_PTR", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 20, //column_length + 2, //column_precision + -1, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("FILE_LABEL", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObVarcharType, //column_type + CS_TYPE_UTF8MB4_BIN, //column_collation_type + 16, //column_length + 2, //column_precision + -1, //column_scale + true, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("META_TREE_EPOCH", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("META_TREE_LEVELS", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("META_BYTES", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("CACHED_META_PAGE_NUM", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("WRITE_BACK_META_PAGE_NUM", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } + + if (OB_SUCC(ret)) { + ADD_COLUMN_SCHEMA("PAGE_FLUSH_CNT", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObNumberType, //column_type + CS_TYPE_INVALID, //column_collation_type + 38, //column_length + 38, //column_precision + 0, //column_scale + false, //is_nullable + false); //is_autoincrement + } if (OB_SUCC(ret)) { table_schema.get_part_option().set_part_num(1); table_schema.set_part_level(PARTITION_LEVEL_ONE); diff --git a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp index 6292c96f7..f7e972b3a 100644 --- a/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21201_21250.cpp @@ -1590,7 +1590,7 @@ int ObInnerTableSchema::gv_ob_tablet_compaction_history_schema(ObTableSchema &ta table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE FROM oceanbase.__all_virtual_tablet_compaction_history )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE, (CASE IS_FULL_MERGE WHEN false THEN "FALSE" ELSE "TRUE" END) AS IS_FULL_MERGE, IO_COST_TIME_PERCENTAGE, MERGE_REASON, BASE_MAJOR_STATUS, CO_MERGE_TYPE FROM oceanbase.__all_virtual_tablet_compaction_history )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1641,7 +1641,7 @@ int ObInnerTableSchema::v_ob_tablet_compaction_history_schema(ObTableSchema &tab table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE FROM oceanbase.GV$OB_TABLET_COMPACTION_HISTORY WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE, IS_FULL_MERGE, IO_COST_TIME_PERCENTAGE, MERGE_REASON, BASE_MAJOR_STATUS, CO_MERGE_TYPE FROM oceanbase.GV$OB_TABLET_COMPACTION_HISTORY WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp index 712c9a527..9c6cad534 100644 --- a/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp +++ b/src/share/inner_table/ob_inner_table_schema.28101_28150.cpp @@ -1488,7 +1488,7 @@ int ObInnerTableSchema::gv_ob_tablet_compaction_history_ora_schema(ObTableSchema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE FROM SYS.ALL_VIRTUAL_TABLET_COMPACTION_HISTORY )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE, CASE WHEN IS_FULL_MERGE = 0 THEN 'FALSE' ELSE 'TRUE' END AS IS_FULL_MERGE, IO_COST_TIME_PERCENTAGE, MERGE_REASON, BASE_MAJOR_STATUS, CO_MERGE_TYPE FROM SYS.ALL_VIRTUAL_TABLET_COMPACTION_HISTORY )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } @@ -1539,7 +1539,7 @@ int ObInnerTableSchema::v_ob_tablet_compaction_history_ora_schema(ObTableSchema table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); if (OB_SUCC(ret)) { - if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE FROM SYS.GV$OB_TABLET_COMPACTION_HISTORY WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() )__"))) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT SVR_IP, SVR_PORT, TENANT_ID, LS_ID, TABLET_ID, TYPE, COMPACTION_SCN, START_TIME, FINISH_TIME, TASK_ID, OCCUPY_SIZE, MACRO_BLOCK_COUNT, MULTIPLEXED_MACRO_BLOCK_COUNT, NEW_MICRO_COUNT_IN_NEW_MACRO, MULTIPLEXED_MICRO_COUNT_IN_NEW_MACRO, TOTAL_ROW_COUNT, INCREMENTAL_ROW_COUNT, COMPRESSION_RATIO, NEW_FLUSH_DATA_RATE, PROGRESSIVE_COMPACTION_ROUND, PROGRESSIVE_COMPACTION_NUM, PARALLEL_DEGREE, PARALLEL_INFO, PARTICIPANT_TABLE, MACRO_ID_LIST, COMMENTS, START_CG_ID, END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, EXEC_MODE, IS_FULL_MERGE, IO_COST_TIME_PERCENTAGE, MERGE_REASON, BASE_MAJOR_STATUS, CO_MERGE_TYPE FROM SYS.GV$OB_TABLET_COMPACTION_HISTORY WHERE SVR_IP=HOST_IP() AND SVR_PORT=RPC_PORT() )__"))) { LOG_ERROR("fail to set view_definition", K(ret)); } } diff --git a/src/share/inner_table/ob_inner_table_schema.351_400.cpp b/src/share/inner_table/ob_inner_table_schema.351_400.cpp index ae759b1a8..d56c538d8 100644 --- a/src/share/inner_table/ob_inner_table_schema.351_400.cpp +++ b/src/share/inner_table/ob_inner_table_schema.351_400.cpp @@ -7132,6 +7132,25 @@ int ObInnerTableSchema::all_tablet_replica_checksum_schema(ObTableSchema &table_ true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj data_checksum_type_default; + data_checksum_type_default.set_int(0); + ADD_COLUMN_SCHEMA_T("data_checksum_type", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + data_checksum_type_default, + data_checksum_type_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 152a34100..6436e7e9b 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -4272,6 +4272,7 @@ def_table_schema( ('data_checksum', 'int'), ('column_checksums', 'longtext', 'true'), ('b_column_checksums', 'varbinary:OB_MAX_VARBINARY_LENGTH', 'true'), + ('data_checksum_type', 'int', 'false', 0) ], ) @@ -10058,7 +10059,12 @@ def_table_schema( ('end_cg_id', 'int'), ('kept_snapshot', 'varchar:OB_COMPACTION_INFO_LENGTH'), ('merge_level', 'varchar:OB_MERGE_LEVEL_STR_LENGTH'), - ('exec_mode', 'varchar:OB_MERGE_TYPE_STR_LENGTH') + ('exec_mode', 'varchar:OB_MERGE_TYPE_STR_LENGTH'), + ('is_full_merge', 'bool'), + ('io_cost_time_percentage', 'int'), + ('merge_reason', 'varchar:OB_MERGE_REASON_STR_LENGTH'), + ('base_major_status', 'varchar:OB_MERGE_TYPE_STR_LENGTH'), + ('co_merge_type', 'varchar:OB_MERGE_TYPE_STR_LENGTH') ], partition_columns = ['svr_ip', 'svr_port'], vtable_route_policy = 'distributed', @@ -24962,7 +24968,14 @@ def_table_schema( END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, - EXEC_MODE + EXEC_MODE, + (CASE IS_FULL_MERGE + WHEN false THEN "FALSE" + ELSE "TRUE" END) AS IS_FULL_MERGE, + IO_COST_TIME_PERCENTAGE, + MERGE_REASON, + BASE_MAJOR_STATUS, + CO_MERGE_TYPE FROM oceanbase.__all_virtual_tablet_compaction_history """.replace("\n", " ") ) @@ -25007,7 +25020,12 @@ def_table_schema( END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, - EXEC_MODE + EXEC_MODE, + IS_FULL_MERGE, + IO_COST_TIME_PERCENTAGE, + MERGE_REASON, + BASE_MAJOR_STATUS, + CO_MERGE_TYPE FROM oceanbase.GV$OB_TABLET_COMPACTION_HISTORY WHERE SVR_IP=HOST_IP() @@ -62877,7 +62895,12 @@ def_table_schema( END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, - EXEC_MODE + EXEC_MODE, + CASE WHEN IS_FULL_MERGE = 0 THEN 'FALSE' ELSE 'TRUE' END AS IS_FULL_MERGE, + IO_COST_TIME_PERCENTAGE, + MERGE_REASON, + BASE_MAJOR_STATUS, + CO_MERGE_TYPE FROM SYS.ALL_VIRTUAL_TABLET_COMPACTION_HISTORY """.replace("\n", " ") ) @@ -62924,7 +62947,12 @@ def_table_schema( END_CG_ID, KEPT_SNAPSHOT, MERGE_LEVEL, - EXEC_MODE + EXEC_MODE, + IS_FULL_MERGE, + IO_COST_TIME_PERCENTAGE, + MERGE_REASON, + BASE_MAJOR_STATUS, + CO_MERGE_TYPE FROM SYS.GV$OB_TABLET_COMPACTION_HISTORY WHERE SVR_IP=HOST_IP() diff --git a/src/share/ob_tablet_replica_checksum_operator.cpp b/src/share/ob_tablet_replica_checksum_operator.cpp index c05d45478..81fcd2961 100644 --- a/src/share/ob_tablet_replica_checksum_operator.cpp +++ b/src/share/ob_tablet_replica_checksum_operator.cpp @@ -258,7 +258,8 @@ ObTabletReplicaChecksumItem::ObTabletReplicaChecksumItem() row_count_(0), compaction_scn_(), data_checksum_(0), - column_meta_() + column_meta_(), + data_checksum_type_(ObDataChecksumType::DATA_CHECKSUM_MAX) {} void ObTabletReplicaChecksumItem::reset() @@ -271,6 +272,7 @@ void ObTabletReplicaChecksumItem::reset() compaction_scn_.reset(); data_checksum_ = 0; column_meta_.reset(); + data_checksum_type_ = ObDataChecksumType::DATA_CHECKSUM_MAX; } bool ObTabletReplicaChecksumItem::is_key_valid() const @@ -289,7 +291,9 @@ bool ObTabletReplicaChecksumItem::is_key_valid() const bool ObTabletReplicaChecksumItem::is_valid() const { - return is_key_valid() && column_meta_.is_valid(); + return is_key_valid() + && column_meta_.is_valid() + && is_valid_data_checksum_type(data_checksum_type_); } bool ObTabletReplicaChecksumItem::is_same_tablet(const ObTabletReplicaChecksumItem &other) const @@ -301,6 +305,19 @@ bool ObTabletReplicaChecksumItem::is_same_tablet(const ObTabletReplicaChecksumIt && tablet_id_ == other.tablet_id_; } +int ObTabletReplicaChecksumItem::check_data_checksum_type(bool &is_cs_replica) const +{ + int ret = OB_SUCCESS; + is_cs_replica = false; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid checksum item", K(ret), KPC(this)); + } else if (ObDataChecksumType::DATA_CHECKSUM_COLUMN_STORE == data_checksum_type_) { + is_cs_replica = true; + } + return ret; +} + int ObTabletReplicaChecksumItem::verify_checksum(const ObTabletReplicaChecksumItem &other) const { int ret = OB_SUCCESS; @@ -364,6 +381,7 @@ int ObTabletReplicaChecksumItem::assign(const ObTabletReplicaChecksumItem &other row_count_ = other.row_count_; compaction_scn_ = other.compaction_scn_; data_checksum_ = other.data_checksum_; + data_checksum_type_ = other.data_checksum_type_; } } return ret; @@ -683,6 +701,9 @@ int ObTabletReplicaChecksumOperator::construct_tablet_replica_checksum_item_( int64_t port = OB_INVALID_INDEX; int64_t ls_id = OB_INVALID_ID; uint64_t compaction_scn_val = 0; + uint64_t min_data_version = 0; + const int64_t tenant_id = MTL_ID(); + int64_t data_checksum_type = 0; ObString column_meta_hex_str; (void)GET_COL_IGNORE_NULL(res.get_int, "tenant_id", int_tenant_id); @@ -695,6 +716,20 @@ int ObTabletReplicaChecksumOperator::construct_tablet_replica_checksum_item_( (void)GET_COL_IGNORE_NULL(res.get_int, "data_checksum", item.data_checksum_); (void)GET_COL_IGNORE_NULL(res.get_varchar, "b_column_checksums", column_meta_hex_str); + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, min_data_version))) { + LOG_WARN("get tenant data version failed", K(ret), K(tenant_id)); + } else if (min_data_version < DATA_VERSION_4_3_3_0) { + item.data_checksum_type_ = ObDataChecksumType::DATA_CHECKSUM_NORMAL; + } else { + (void)GET_COL_IGNORE_NULL(res.get_int, "data_checksum_type", data_checksum_type); + if (is_valid_data_checksum_type(data_checksum_type)) { + item.data_checksum_type_ = static_cast(data_checksum_type); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid data checksum type", KR(ret), K(data_checksum_type)); + } + } + if (FAILEDx(item.compaction_scn_.convert_for_inner_table_field(compaction_scn_val))) { LOG_WARN("fail to convert val to SCN", KR(ret), K(compaction_scn_val)); } else if (OB_FAIL(item.set_tenant_id((uint64_t)int_tenant_id))) { @@ -760,11 +795,11 @@ int ObTabletReplicaChecksumOperator::inner_batch_insert_or_update_by_sql_( } else { int64_t affected_rows = 0; ObSqlString sql; + ObDMLSqlSplicer dml_splicer; + uint64_t min_data_version = 0; - if (OB_FAIL(sql.assign_fmt("INSERT INTO %s (tenant_id, tablet_id, ls_id, svr_ip, svr_port, row_count, " - "compaction_scn, data_checksum, column_checksums, b_column_checksums, gmt_modified, gmt_create) VALUES", - OB_ALL_TABLET_REPLICA_CHECKSUM_TNAME))) { - LOG_WARN("fail to assign sql", KR(ret), K(tenant_id)); + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, min_data_version))) { + LOG_WARN("get tenant data version failed", K(ret), K(tenant_id)); } else { ObArenaAllocator allocator; char ip[OB_MAX_SERVER_ADDR_SIZE] = ""; @@ -784,24 +819,28 @@ int ObTabletReplicaChecksumOperator::inner_batch_insert_or_update_by_sql_( LOG_WARN("fail to get visible column meta str", KR(ret)); } else if (OB_FAIL(get_hex_column_meta(cur_item.column_meta_, allocator, hex_column_meta))) { LOG_WARN("fail to get hex column meta str", KR(ret)); - } else if (OB_FAIL(sql.append_fmt("('%lu', '%lu', %ld, '%s', %d, %ld, %lu, %ld, " - "'%.*s', '%.*s', now(6), now(6))%s", cur_item.tenant_id_, cur_item.tablet_id_.id(), - cur_item.ls_id_.id(), ip, port, cur_item.row_count_, - compaction_scn_val, cur_item.data_checksum_, visible_column_meta.length(), - visible_column_meta.ptr(), hex_column_meta.length(), hex_column_meta.ptr(), - ((idx == end_idx - 1) ? " " : ", ")))) { - LOG_WARN("fail to assign sql", KR(ret), K(idx), K(end_idx), K(cur_item)); + } else if (OB_FAIL(dml_splicer.add_gmt_modified()) + || OB_FAIL(dml_splicer.add_gmt_create()) + || OB_FAIL(dml_splicer.add_pk_column("tenant_id", cur_item.tenant_id_)) + || OB_FAIL(dml_splicer.add_pk_column("tablet_id", cur_item.tablet_id_.id())) + || OB_FAIL(dml_splicer.add_pk_column("svr_ip", ip)) + || OB_FAIL(dml_splicer.add_pk_column("svr_port", port)) + || OB_FAIL(dml_splicer.add_pk_column("ls_id", cur_item.ls_id_.id())) + || OB_FAIL(dml_splicer.add_column("compaction_scn", compaction_scn_val)) + || OB_FAIL(dml_splicer.add_column("row_count", cur_item.row_count_)) + || OB_FAIL(dml_splicer.add_column("data_checksum", cur_item.data_checksum_)) + || OB_FAIL(dml_splicer.add_column("column_checksums", visible_column_meta)) + || OB_FAIL(dml_splicer.add_column("b_column_checksums", hex_column_meta)) + || (min_data_version >= DATA_VERSION_4_3_3_0 && OB_FAIL(dml_splicer.add_column("data_checksum_type", cur_item.data_checksum_type_))) + ) { + LOG_WARN("fail to fill dml splicer", KR(ret), K(min_data_version), K(idx), K(end_idx), K(cur_item)); + } else if (OB_FAIL(dml_splicer.finish_row())) { + LOG_WARN("fail to finish row", KR(ret), K(idx), K(end_idx), K(cur_item)); } } - if (FAILEDx(sql.append_fmt(" ON DUPLICATE KEY UPDATE "))) { - LOG_WARN("fail to append sql string", KR(ret), K(sql)); - } else if (OB_FAIL(sql.append(" row_count = values(row_count)")) - || OB_FAIL(sql.append(", compaction_scn = values(compaction_scn)")) - || OB_FAIL(sql.append(", data_checksum = values(data_checksum)")) - || OB_FAIL(sql.append(", column_checksums = values(column_checksums)")) - || OB_FAIL(sql.append(", b_column_checksums = values(b_column_checksums)"))) { - LOG_WARN("fail to append sql string", KR(ret), K(sql)); + if (FAILEDx(dml_splicer.splice_batch_insert_update_sql(OB_ALL_TABLET_REPLICA_CHECKSUM_TNAME, sql))) { + LOG_WARN("fail to splice batch insert update sql", KR(ret), K(sql)); } } @@ -1406,10 +1445,13 @@ void ObTabletDataChecksumChecker::reset() cs_replica_ckm_item_ = nullptr; } -int ObTabletDataChecksumChecker::check_data_checksum(const ObTabletReplicaChecksumItem& curr_item, bool is_cs_replica) +int ObTabletDataChecksumChecker::check_data_checksum(const ObTabletReplicaChecksumItem& curr_item) { int ret = OB_SUCCESS; - if (is_cs_replica) { + bool is_cs_replica = false; + if (OB_FAIL(curr_item.check_data_checksum_type(is_cs_replica))) { + LOG_WARN("fail to check data checksum type", KR(ret), K(curr_item)); + } else if (is_cs_replica) { if (OB_ISNULL(cs_replica_ckm_item_)) { cs_replica_ckm_item_ = &curr_item; } else if (cs_replica_ckm_item_->compaction_scn_ != curr_item.compaction_scn_) { diff --git a/src/share/ob_tablet_replica_checksum_operator.h b/src/share/ob_tablet_replica_checksum_operator.h index 0d288fdb9..2a3c22c69 100644 --- a/src/share/ob_tablet_replica_checksum_operator.h +++ b/src/share/ob_tablet_replica_checksum_operator.h @@ -88,10 +88,11 @@ public: int assign_key(const ObTabletReplicaChecksumItem &other); int assign(const ObTabletReplicaChecksumItem &other); int set_tenant_id(const uint64_t tenant_id); + int check_data_checksum_type(bool &is_cs_replica) const; common::ObTabletID get_tablet_id() const { return tablet_id_; } TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(tablet_id), K_(server), K_(row_count), - K_(compaction_scn), K_(data_checksum), K_(column_meta)); + K_(compaction_scn), K_(data_checksum), K_(column_meta), K_(data_checksum_type)); public: uint64_t tenant_id_; @@ -102,6 +103,7 @@ public: SCN compaction_scn_; int64_t data_checksum_; ObTabletReplicaReportColumnMeta column_meta_; + ObDataChecksumType data_checksum_type_; }; typedef ObArrayWithMap ObReplicaCkmArray; // Operator for __all_tablet_replica_checksum @@ -146,7 +148,6 @@ public: static int set_column_meta_with_hex_str( const ObString &hex_str, ObTabletReplicaReportColumnMeta &column_meta); - static int get_visible_column_meta( const ObTabletReplicaReportColumnMeta &column_meta, common::ObIAllocator &allocator, @@ -349,7 +350,7 @@ public: ObTabletDataChecksumChecker(); ~ObTabletDataChecksumChecker(); void reset(); - int check_data_checksum(const ObTabletReplicaChecksumItem& curr_item, bool is_cs_replica); + int check_data_checksum(const ObTabletReplicaChecksumItem& curr_item); TO_STRING_KV(KPC_(normal_ckm_item), KPC_(cs_replica_ckm_item)); private: const ObTabletReplicaChecksumItem *normal_ckm_item_; diff --git a/src/share/tablet/ob_tablet_info.cpp b/src/share/tablet/ob_tablet_info.cpp index 46a5acfea..ec5db407e 100644 --- a/src/share/tablet/ob_tablet_info.cpp +++ b/src/share/tablet/ob_tablet_info.cpp @@ -21,6 +21,24 @@ namespace oceanbase { namespace share { + +const static char * ObDataChecksumTypeStr[] = { + "NORMAL", + "COLUMNSTORE", +}; + +const char *data_check_checksum_type_to_str(const ObDataChecksumType type) +{ + STATIC_ASSERT(static_cast(ObDataChecksumType::DATA_CHECKSUM_MAX) == ARRAYSIZEOF(ObDataChecksumTypeStr), "checksum type len is mismatch"); + const char *str = ""; + if (is_valid_data_checksum_type(type)) { + str = ObDataChecksumTypeStr[static_cast(type)]; + } else { + str = "INVALI_DATACHECKSUM_TYPE"; + } + return str; +} + ObTabletReplica::ObTabletReplica() : tenant_id_(OB_INVALID_TENANT_ID), tablet_id_(), diff --git a/src/share/tablet/ob_tablet_info.h b/src/share/tablet/ob_tablet_info.h index 3a5e69778..d62bb7a6e 100644 --- a/src/share/tablet/ob_tablet_info.h +++ b/src/share/tablet/ob_tablet_info.h @@ -24,6 +24,26 @@ namespace share { class ObTabletReplicaFilter; +enum ObDataChecksumType : uint8_t +{ + DATA_CHECKSUM_NORMAL = 0, + DATA_CHECKSUM_COLUMN_STORE = 1, // for column store replica data checksum + DATA_CHECKSUM_MAX +}; + +inline bool is_valid_data_checksum_type(const int64_t &type) +{ + return type >= ObDataChecksumType::DATA_CHECKSUM_NORMAL + && type < ObDataChecksumType::DATA_CHECKSUM_MAX; +} + +inline bool is_valid_data_checksum_type(const ObDataChecksumType &type) +{ + return is_valid_data_checksum_type(static_cast(type)); +} + +const char *data_check_checksum_type_to_str(const ObDataChecksumType type); + class ObTabletReplica { public: diff --git a/src/storage/blocksstable/ob_major_checksum_info.cpp b/src/storage/blocksstable/ob_major_checksum_info.cpp index dcff29b28..d713f4902 100644 --- a/src/storage/blocksstable/ob_major_checksum_info.cpp +++ b/src/storage/blocksstable/ob_major_checksum_info.cpp @@ -150,7 +150,7 @@ int ObMajorChecksumInfo::init_from_sstable( tmp_col_ckm_array.set_attr(ObMemAttr(MTL_ID(), "MajorCkmInfo")); if (sstable.is_co_sstable()) { const ObCOSSTableV2 &co_sstable = static_cast(sstable); - if (OB_FAIL(co_sstable.fill_column_ckm_array(storage_schema, tmp_col_ckm_array))) { + if (OB_FAIL(co_sstable.fill_column_ckm_array(storage_schema, tmp_col_ckm_array, false /*need_process_cs_replica*/))) { LOG_WARN("fail to fill column checksum array", K(ret), KPC(this), K(sstable)); } } else if (OB_FAIL(sstable.fill_column_ckm_array(tmp_col_ckm_array))) { diff --git a/src/storage/column_store/ob_co_merge_ctx.cpp b/src/storage/column_store/ob_co_merge_ctx.cpp index cc25588d9..d80882fb4 100644 --- a/src/storage/column_store/ob_co_merge_ctx.cpp +++ b/src/storage/column_store/ob_co_merge_ctx.cpp @@ -230,6 +230,10 @@ int ObCOTabletMergeCtx::prepare_schema() LOG_INFO("[CS-Replica] finish prepare schema for co merge", K(ret), "is_cs_replica", static_param_.is_cs_replica_, KPC(this)); } + + if (FAILEDx(prepare_row_store_cg_schema())) { + LOG_WARN("failed to init major sstable status", K(ret)); + } return ret; } @@ -239,8 +243,6 @@ int ObCOTabletMergeCtx::build_ctx(bool &finish_flag) // finish_flag in this function is useless, just for virtual function definition. if (OB_FAIL(ObBasicTabletMergeCtx::build_ctx(finish_flag))) { LOG_WARN("failed to build basic ctx", KR(ret), "param", get_dag_param(), KPC(this)); - } else if (OB_FAIL(init_major_sstable_status())) { - LOG_WARN("failed to init major sstable status", K(ret)); } else if (is_major_merge_type(get_merge_type())) { // meta major merge not support row col switch now if (is_build_row_store_from_rowkey_cg() && OB_FAIL(mock_row_store_table_read_info())) { @@ -755,7 +757,7 @@ int ObCOTabletMergeCtx::get_cg_schema_for_merge(const int64_t idx, const ObStora return ret; } -int ObCOTabletMergeCtx::init_major_sstable_status() +int ObCOTabletMergeCtx::prepare_row_store_cg_schema() { int ret = OB_SUCCESS; ObSSTable *sstable = static_cast(get_tables_handle().get_table(0)); diff --git a/src/storage/column_store/ob_co_merge_ctx.h b/src/storage/column_store/ob_co_merge_ctx.h index c4840f98a..71c1ec3cc 100644 --- a/src/storage/column_store/ob_co_merge_ctx.h +++ b/src/storage/column_store/ob_co_merge_ctx.h @@ -129,7 +129,7 @@ struct ObCOTabletMergeCtx : public ObBasicTabletMergeCtx const uint32_t start_cg_idx, const uint32_t end_cg_idx, const bool check_info_ready) const; - int init_major_sstable_status(); + int prepare_row_store_cg_schema(); int construct_column_param( const uint64_t column_id, const ObStorageColumnSchema *column_schema, diff --git a/src/storage/column_store/ob_column_oriented_sstable.cpp b/src/storage/column_store/ob_column_oriented_sstable.cpp index fead90500..0efd3be7a 100644 --- a/src/storage/column_store/ob_column_oriented_sstable.cpp +++ b/src/storage/column_store/ob_column_oriented_sstable.cpp @@ -215,6 +215,27 @@ int ObCOSSTableMeta::deserialize(const char *buf, const int64_t data_len, int64_ return ret; } +/************************************* ObCOMajorSSTableStatus *************************************/ +const char* ObCOMajorSSTableStatusStr[] = { + "INVALID_STATUS", + "COL_WITH_ALL", + "COL_ONLY_ALL", + "PURE_COL", + "PURE_COL_ONLY_ALL", + "COL_REPLICA_MAJOR" +}; + +const char* co_major_sstable_status_to_str(const ObCOMajorSSTableStatus& major_sstable_status) +{ + STATIC_ASSERT(static_cast(MAX_CO_MAJOR_SSTABLE_STATUS) == ARRAYSIZEOF(ObCOMajorSSTableStatusStr), "co major sstable str len is mismatch"); + const char *str = ""; + if (is_valid_co_major_sstable_status(major_sstable_status)) { + str = ObCOMajorSSTableStatusStr[major_sstable_status]; + } else { + str = "invalid_co_major_status"; + } + return str; +} /************************************* ObCOSSTableV2 *************************************/ ObCOSSTableV2::ObCOSSTableV2() @@ -899,10 +920,12 @@ int ObCOSSTableV2::multi_get( int ObCOSSTableV2::fill_column_ckm_array( const ObStorageSchema &storage_schema, - ObIArray &column_checksums) const + ObIArray &column_checksums, + bool need_process_cs_replica) const { int ret = OB_SUCCESS; - if (is_cgs_empty_co_table()) { + // for cgs emtpy co table, only report rowkey cg/all cgs column checksum. but need report all columns for cs replica + if (is_cgs_empty_co_table() && !need_process_cs_replica) { ret = ObSSTable::fill_column_ckm_array(column_checksums); } else { const common::ObIArray &column_groups = storage_schema.get_column_groups(); diff --git a/src/storage/column_store/ob_column_oriented_sstable.h b/src/storage/column_store/ob_column_oriented_sstable.h index 7ef0fd2de..15be76be5 100644 --- a/src/storage/column_store/ob_column_oriented_sstable.h +++ b/src/storage/column_store/ob_column_oriented_sstable.h @@ -141,6 +141,7 @@ inline bool is_major_sstable_match_schema(const ObCOMajorSSTableStatus& major_ss return major_sstable_status == COL_WITH_ALL || major_sstable_status == PURE_COL; } +const char* co_major_sstable_status_to_str(const ObCOMajorSSTableStatus& major_sstable_status); /* * The base part of ObCOSSTable maybe */ @@ -223,7 +224,10 @@ public: ObTableAccessContext &context, const common::ObIArray &rowkeys, ObStoreRowIterator *&row_iter) override; - int fill_column_ckm_array(const ObStorageSchema &storage_schema, ObIArray &column_checksums) const; + int fill_column_ckm_array( + const ObStorageSchema &storage_schema, + ObIArray &column_checksums, + bool need_process_cs_replica) const; INHERIT_TO_STRING_KV("ObSSTable", ObSSTable, KP(this), K_(cs_meta), K_(base_type), K_(is_cgs_empty_co), K_(valid_for_cs_reading)); private: diff --git a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp index 03fea9293..418841141 100644 --- a/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_basic_tablet_merge_ctx.cpp @@ -733,17 +733,6 @@ void ObBasicTabletMergeCtx::add_sstable_merge_info( #define ADD_COMMENT(...) \ ADD_COMPACTION_INFO_PARAM(running_info.comment_, sizeof(running_info.comment_), __VA_ARGS__) - if (get_is_full_merge()) { - ADD_COMMENT("is_full_merge", true); - } - if (ObAdaptiveMergePolicy::AdaptiveMergeReason::NONE != static_param_.merge_reason_) { - ADD_COMMENT("merge_reason", ObAdaptiveMergePolicy::merge_reason_to_str(static_param_.merge_reason_)); - } - if (is_major_merge_type(get_merge_type()) - && ObCOMajorMergePolicy::INVALID_CO_MAJOR_MERGE_TYPE != static_param_.co_major_merge_type_) { - ADD_COMMENT("major", static_param_.major_sstable_status_); - ADD_COMMENT("co", ObCOMajorMergePolicy::co_major_merge_type_to_str(static_param_.co_major_merge_type_)); - } // calc flush macro speed uint32_t exe_ts = time_guard.get_specified_cost_time(ObStorageCompactionTimeGuard::EXECUTE); if (exe_ts > 0 && block_info.new_micro_info_.get_data_micro_size() > 0) { @@ -751,7 +740,7 @@ void ObBasicTabletMergeCtx::add_sstable_merge_info( int64_t io_percentage = block_info.block_io_us_ * 100 / (float)exe_ts; ADD_COMMENT("block_io_us", block_info.block_io_us_); if (io_percentage > 0) { - ADD_COMMENT("io_percent", io_percentage); + running_info.io_percentage_ = io_percentage; } } int64_t mem_peak_mb = mem_ctx_.get_total_mem_peak() >> 20; @@ -1317,6 +1306,9 @@ int ObBasicTabletMergeCtx::init_sstable_merge_history() static_history_.is_full_merge_ = static_param_.is_full_merge_; static_history_.merge_level_ = static_param_.merge_level_; static_history_.exec_mode_ = get_exec_mode(); + static_history_.merge_reason_ = static_param_.merge_reason_; + static_history_.base_major_status_ = static_param_.major_sstable_status_; + static_history_.co_major_merge_type_ = static_param_.co_major_merge_type_; if (!static_history_.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("static info is invalid", KR(ret), K_(static_history)); diff --git a/src/storage/compaction/ob_medium_compaction_func.cpp b/src/storage/compaction/ob_medium_compaction_func.cpp index c2243e068..a2f191bcc 100644 --- a/src/storage/compaction/ob_medium_compaction_func.cpp +++ b/src/storage/compaction/ob_medium_compaction_func.cpp @@ -1267,7 +1267,6 @@ int ObMediumCompactionScheduleFunc::init_tablet_filters(share::ObTabletReplicaFi int ObMediumCompactionScheduleFunc::check_tablet_checksum( const ObReplicaCkmArray &checksum_items, - const ObLSColumnReplicaCache &ls_cs_replica_cache, const int64_t start_idx, const int64_t end_idx, const bool is_medium_checker, @@ -1287,35 +1286,16 @@ int ObMediumCompactionScheduleFunc::check_tablet_checksum( ObLSID prev_error_ls_id; for (int64_t idx = start_idx; OB_SUCC(ret) && idx < end_idx; ++idx) { const ObTabletReplicaChecksumItem &curr_item = checksum_items.at(idx); - bool is_cs_replica = false; - ObLSReplicaUniItem ls_item(curr_item.ls_id_, curr_item.server_); - const ObLSReplica *replica = nullptr; - bool can_skip = false; - - if (OB_FAIL(ls_cs_replica_cache.check_can_skip(ls_item, can_skip))) { - LOG_WARN("failed to check item can skip", K(ret), K(ls_item), K(ls_cs_replica_cache)); - } else if (can_skip) { - LOG_INFO("curr ls item should skip check", K(ls_item), K(ls_cs_replica_cache)); - continue; - } else if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { - LOG_WARN("fail to check is column replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); - } else if (OB_ISNULL(prev_item)) { + if (OB_ISNULL(prev_item)) { } else if (!curr_item.is_same_tablet(*prev_item)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("not continuous same tablet id", K(ret), K(curr_item), KPC(prev_item)); - } else if (OB_TMP_FAIL(data_checksum_checker.check_data_checksum(curr_item, is_cs_replica)) + } else if (OB_TMP_FAIL(data_checksum_checker.check_data_checksum(curr_item)) || OB_TMP_FAIL(curr_item.verify_column_checksum(*prev_item))) { if (OB_CHECKSUM_ERROR == tmp_ret) { - ObLSColumnReplicaCache dump_cache; int tmp_ret = OB_SUCCESS; - if (OB_TMP_FAIL(dump_cache.init())) { - LOG_WARN_RET(tmp_ret, "failed to init dump cache", K(MTL_ID())); - } else if (OB_TMP_FAIL(dump_cache.update(curr_item.ls_id_))) { - LOG_WARN_RET(tmp_ret, "failed to force refresh ls locality", K(curr_item)); - } - LOG_DBA_ERROR(OB_CHECKSUM_ERROR, "msg", "checksum error in tablet replica checksum", KR(tmp_ret), - K(curr_item), KPC(prev_item), K(ls_cs_replica_cache), K(is_cs_replica), K(dump_cache), K(data_checksum_checker)); + K(curr_item), KPC(prev_item), K(data_checksum_checker)); check_ret = OB_CHECKSUM_ERROR; if (curr_item.ls_id_ != prev_error_ls_id) { prev_error_ls_id = curr_item.ls_id_; @@ -1353,7 +1333,6 @@ int ObMediumCompactionScheduleFunc::check_tablet_checksum( int ObMediumCompactionScheduleFunc::check_replica_checksum_items( const ObReplicaCkmArray &checksum_items, - const ObLSColumnReplicaCache &ls_cs_replica_cache, const bool is_medium_checker) { int ret = OB_SUCCESS; @@ -1375,8 +1354,8 @@ int ObMediumCompactionScheduleFunc::check_replica_checksum_items( while (end_idx < count && tablet_id == checksum_items.at(end_idx).tablet_id_) { end_idx++; } - if (OB_FAIL(check_tablet_checksum(checksum_items, ls_cs_replica_cache, start_idx, end_idx, true /*is_medium_checker*/, error_pairs, check_ret))) { - LOG_WARN("failed to check tablet checksum", KR(ret), K(checksum_items), K(ls_cs_replica_cache), K(start_idx), K(end_idx), K(error_pairs)); + if (OB_FAIL(check_tablet_checksum(checksum_items, start_idx, end_idx, true /*is_medium_checker*/, error_pairs, check_ret))) { + LOG_WARN("failed to check tablet checksum", KR(ret), K(checksum_items), K(start_idx), K(end_idx), K(error_pairs)); } else { // update medium compaction info if (is_medium_checker && OB_SUCCESS == check_ret) { @@ -1425,8 +1404,7 @@ int ObMediumCompactionScheduleFunc::batch_check_medium_finish( const hash::ObHashMap &ls_info_map, ObIArray &finish_tablet_ls_infos, const ObIArray &tablet_ls_infos, - ObCompactionTimeGuard &time_guard, - const share::ObLSColumnReplicaCache &ls_cs_replica_cache) + ObCompactionTimeGuard &time_guard) { int ret = OB_SUCCESS; if (tablet_ls_infos.empty()) { @@ -1442,7 +1420,7 @@ int ObMediumCompactionScheduleFunc::batch_check_medium_finish( MTL_ID(), finish_tablet_ls_infos, checksum_items))) { LOG_WARN("failed to get tablet checksum", K(ret)); } else if (FALSE_IT(time_guard.click(ObCompactionScheduleTimeGuard::SEARCH_CHECKSUM))) { - } else if (OB_FAIL(check_replica_checksum_items(checksum_items, ls_cs_replica_cache, true /*is_medium_checker*/))) { + } else if (OB_FAIL(check_replica_checksum_items(checksum_items, true /*is_medium_checker*/))) { LOG_WARN("fail to check replica checksum items for medium checker", K(ret)); } else if (FALSE_IT(time_guard.click(ObCompactionScheduleTimeGuard::CHECK_CHECKSUM))) { } diff --git a/src/storage/compaction/ob_medium_compaction_func.h b/src/storage/compaction/ob_medium_compaction_func.h index 97921d1b7..0b013a1b8 100644 --- a/src/storage/compaction/ob_medium_compaction_func.h +++ b/src/storage/compaction/ob_medium_compaction_func.h @@ -93,11 +93,9 @@ public: const hash::ObHashMap &ls_info_map, ObIArray &finish_tablet_ls_infos, const ObIArray &tablet_ls_infos, - ObCompactionTimeGuard &time_guard, - const share::ObLSColumnReplicaCache &ls_cs_replica_cache); + ObCompactionTimeGuard &time_guard); static int check_replica_checksum_items( const ObReplicaCkmArray &checksum_items, - const ObLSColumnReplicaCache &ls_cs_replica_cache, const bool is_medium_checker); static int check_need_merge_and_schedule( ObLS &ls, @@ -166,7 +164,6 @@ protected: static int init_tablet_filters(share::ObTabletReplicaFilterHolder &filters); static int check_tablet_checksum( const share::ObReplicaCkmArray &checksum_items, - const ObLSColumnReplicaCache &ls_cs_replica_cache, const int64_t start_idx, const int64_t end_idx, const bool is_medium_checker, diff --git a/src/storage/compaction/ob_sstable_merge_history.cpp b/src/storage/compaction/ob_sstable_merge_history.cpp index 88d7cc27e..673b14780 100644 --- a/src/storage/compaction/ob_sstable_merge_history.cpp +++ b/src/storage/compaction/ob_sstable_merge_history.cpp @@ -120,6 +120,9 @@ ObMergeStaticInfo::ObMergeStaticInfo() participant_table_info_(), merge_level_(MERGE_LEVEL_MAX), exec_mode_(ObExecMode::EXEC_MODE_MAX), + merge_reason_(ObAdaptiveMergePolicy::NONE), + base_major_status_(ObCOMajorSSTableStatus::INVALID_CO_MAJOR_SSTABLE_STATUS), + co_major_merge_type_(ObCOMajorMergePolicy::INVALID_CO_MAJOR_MERGE_TYPE), is_full_merge_(false), is_fake_(false) {} @@ -145,6 +148,9 @@ void ObMergeStaticInfo::reset() participant_table_info_.reset(); merge_level_ = MERGE_LEVEL_MAX; exec_mode_ = ObExecMode::EXEC_MODE_MAX; + merge_reason_ = ObAdaptiveMergePolicy::NONE; + base_major_status_ = ObCOMajorSSTableStatus::INVALID_CO_MAJOR_SSTABLE_STATUS; + co_major_merge_type_ = ObCOMajorMergePolicy::INVALID_CO_MAJOR_MERGE_TYPE; is_full_merge_ = false; } @@ -161,6 +167,9 @@ void ObMergeStaticInfo::shallow_copy(const ObMergeStaticInfo &other) participant_table_info_ = other.participant_table_info_; merge_level_ = other.merge_level_; exec_mode_ = other.exec_mode_; + merge_reason_ = other.merge_reason_; + base_major_status_ = other.base_major_status_; + co_major_merge_type_ = other.co_major_merge_type_; is_full_merge_ = other.is_full_merge_; } /** @@ -171,6 +180,7 @@ ObMergeRunningInfo::ObMergeRunningInfo() merge_finish_time_(0), start_cg_idx_(0), end_cg_idx_(0), + io_percentage_(0), dag_id_(), parallel_merge_info_(), comment_("\0") @@ -182,6 +192,7 @@ void ObMergeRunningInfo::reset() merge_finish_time_ = 0; start_cg_idx_ = 0; end_cg_idx_ = 0; + io_percentage_ = 0; dag_id_.reset(); parallel_merge_info_.reset(); MEMSET(comment_, '\0', sizeof(comment_)); @@ -198,6 +209,7 @@ void ObMergeRunningInfo::shallow_copy(const ObMergeRunningInfo &other) merge_finish_time_ = other.merge_finish_time_; start_cg_idx_ = other.start_cg_idx_; end_cg_idx_ = other.end_cg_idx_; + io_percentage_ = other.io_percentage_; dag_id_ = other.dag_id_; parallel_merge_info_ = other.parallel_merge_info_; MEMSET(comment_, '\0', sizeof(comment_)); diff --git a/src/storage/compaction/ob_sstable_merge_history.h b/src/storage/compaction/ob_sstable_merge_history.h index 945011385..b867746ef 100644 --- a/src/storage/compaction/ob_sstable_merge_history.h +++ b/src/storage/compaction/ob_sstable_merge_history.h @@ -111,7 +111,11 @@ struct ObMergeStaticInfo TO_STRING_KV(K_(ls_id), K_(tablet_id), "merge_type", merge_type_to_str(merge_type_), K_(compaction_scn), K_(is_full_merge), K_(concurrent_cnt), "merge_level", merge_level_to_str(merge_level_), - "exec_mode", exec_mode_to_str(exec_mode_), K_(kept_snapshot_info), K_(participant_table_info), + "exec_mode", exec_mode_to_str(exec_mode_), + "merge_reason", ObAdaptiveMergePolicy::merge_reason_to_str(merge_reason_), + "base_major_status", co_major_sstable_status_to_str(base_major_status_), + "co_major_merge_type", ObCOMajorMergePolicy::co_major_merge_type_to_str(co_major_merge_type_), + K_(kept_snapshot_info), K_(participant_table_info), K_(progressive_merge_round), K_(progressive_merge_num), K_(is_fake)); share::ObLSID ls_id_; @@ -125,6 +129,9 @@ struct ObMergeStaticInfo PartTableInfo participant_table_info_; ObMergeLevel merge_level_; ObExecMode exec_mode_; + ObAdaptiveMergePolicy::AdaptiveMergeReason merge_reason_; + ObCOMajorSSTableStatus base_major_status_; + ObCOMajorMergePolicy::ObCOMajorMergeType co_major_merge_type_; bool is_full_merge_; bool is_fake_; }; @@ -140,12 +147,13 @@ struct ObMergeRunningInfo static const int64_t MERGE_INFO_COMMENT_LENGTH = 256; TO_STRING_KV(K_(merge_start_time), K_(merge_finish_time), K_(dag_id), "merge_cost_time", merge_finish_time_ - merge_start_time_, - K_(start_cg_idx), K_(end_cg_idx), K_(parallel_merge_info)); + K_(start_cg_idx), K_(end_cg_idx), K_(io_percentage), K_(parallel_merge_info)); int64_t merge_start_time_; int64_t merge_finish_time_; int64_t start_cg_idx_; int64_t end_cg_idx_; + int64_t io_percentage_; common::ObCurTraceId::TraceId dag_id_; ObParalleMergeInfo parallel_merge_info_; char comment_[MERGE_INFO_COMMENT_LENGTH]; diff --git a/src/storage/compaction/ob_tenant_medium_checker.cpp b/src/storage/compaction/ob_tenant_medium_checker.cpp index 521320d42..1009f171b 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.cpp +++ b/src/storage/compaction/ob_tenant_medium_checker.cpp @@ -219,20 +219,15 @@ int ObTenantMediumChecker::check_medium_finish_schedule() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObLSColumnReplicaCache cs_replica_cache; // a copy one from ls_locality_cache_ if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObTenantMediumChecker is not inited", K(ret)); - } else if (OB_FAIL(cs_replica_cache.init())) { - LOG_WARN("failed to init cs_replica_cache", K(ret)); } else { lib::ObMutexGuard guard(lock_); if (OB_FAIL(ls_locality_cache_.refresh_ls_locality(false /*force_refresh*/))) { LOG_WARN("failed to refresh ls locality"); ADD_COMMON_SUSPECT_INFO(MEDIUM_MERGE, share::ObDiagnoseTabletType::TYPE_MEDIUM_MERGE, SUSPECT_FAILED_TO_REFRESH_LS_LOCALITY, ret); - } else if (OB_FAIL(cs_replica_cache.assign(ls_locality_cache_.get_cs_replica_cache()))) { - LOG_WARN("failed to assign cs_replica_cache", K(ret)); } } @@ -275,7 +270,7 @@ int ObTenantMediumChecker::check_medium_finish_schedule() int64_t cost_ts = ObTimeUtility::fast_current_time(); ObBatchFinishCheckStat stat; while (start_idx < end_idx) { - if (OB_TMP_FAIL(check_medium_finish(tablet_ls_infos, start_idx, end_idx, batch_tablet_ls_infos, finish_tablet_ls_infos, stat, cs_replica_cache))) { + if (OB_TMP_FAIL(check_medium_finish(tablet_ls_infos, start_idx, end_idx, batch_tablet_ls_infos, finish_tablet_ls_infos, stat))) { LOG_WARN("failed to check medium finish", K(tmp_ret)); } start_idx = end_idx; @@ -298,8 +293,7 @@ int ObTenantMediumChecker::check_medium_finish( int64_t end_idx, ObIArray &check_tablet_ls_infos, ObIArray &finish_tablet_ls_infos, - ObBatchFinishCheckStat &stat, - const share::ObLSColumnReplicaCache &ls_cs_replica_cache) + ObBatchFinishCheckStat &stat) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -329,7 +323,7 @@ int ObTenantMediumChecker::check_medium_finish( ObCompactionScheduleTimeGuard time_guard; stat.filter_cnt_ += (end_idx - start_idx - check_tablet_ls_infos.count()); if (FAILEDx(ObMediumCompactionScheduleFunc::batch_check_medium_finish( - ls_info_map_, finish_tablet_ls_infos, check_tablet_ls_infos, time_guard, ls_cs_replica_cache))) { + ls_info_map_, finish_tablet_ls_infos, check_tablet_ls_infos, time_guard))) { LOG_WARN("failed to batch check medium finish", K(ret), K(tablet_ls_infos.count()), K(check_tablet_ls_infos.count()), K(tablet_ls_infos), K(check_tablet_ls_infos)); stat.fail_cnt_ += check_tablet_ls_infos.count(); diff --git a/src/storage/compaction/ob_tenant_medium_checker.h b/src/storage/compaction/ob_tenant_medium_checker.h index 342d67875..12cc6f2f9 100644 --- a/src/storage/compaction/ob_tenant_medium_checker.h +++ b/src/storage/compaction/ob_tenant_medium_checker.h @@ -95,8 +95,7 @@ public: int64_t end_idx, ObIArray &check_tablet_ls_infos, ObIArray &finish_tablet_ls_infos, - ObBatchFinishCheckStat &stat, - const share::ObLSColumnReplicaCache &ls_cs_replica_cache); + ObBatchFinishCheckStat &stat); int add_tablet_ls(const ObTabletID &tablet_id, const share::ObLSID &ls_id, const int64_t medium_scn); bool locality_cache_empty(); TO_STRING_KV(K_(is_inited), K_(ls_locality_cache)); diff --git a/src/storage/high_availability/ob_storage_ha_utils.cpp b/src/storage/high_availability/ob_storage_ha_utils.cpp index 8416ac450..ab559df26 100644 --- a/src/storage/high_availability/ob_storage_ha_utils.cpp +++ b/src/storage/high_availability/ob_storage_ha_utils.cpp @@ -210,7 +210,6 @@ int ObStorageHAUtils::check_tablet_replica_checksum_(const uint64_t tenant_id, c LOG_WARN("failed to batch get replica checksum item", K(ret), K(tenant_id), K(pairs), K(compaction_scn)); } else { ObArray filter_items; - ObLSColumnReplicaCache ls_cs_replica_cache; ObTabletDataChecksumChecker data_checksum_checker; for (int64_t i = 0; OB_SUCC(ret) && i < items.count(); ++i) { const ObTabletReplicaChecksumItem &item = items.at(i); @@ -221,38 +220,15 @@ int ObStorageHAUtils::check_tablet_replica_checksum_(const uint64_t tenant_id, c } } - if (FAILEDx(ls_cs_replica_cache.init())) { - LOG_WARN("failed to init ls column replica cache", K(ret)); - } - - for (int64_t i = 0; OB_SUCC(ret) && i < filter_items.count(); ++i) { - const ObTabletReplicaChecksumItem &item = filter_items.at(i); - if (OB_FAIL(ls_cs_replica_cache.update(item.ls_id_))) { - LOG_WARN("fail to update ls replica status", K(ret), K(item)); - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < filter_items.count(); ++i) { const ObTabletReplicaChecksumItem &first_item = filter_items.at(0); const ObTabletReplicaChecksumItem &item = filter_items.at(i); - const ObLSReplicaUniItem ls_item(item.ls_id_, item.server_); - bool is_cs_replica = false; - bool can_skip = false; - const ObLSReplica *replica = nullptr; - - if (OB_FAIL(ls_cs_replica_cache.check_can_skip(ls_item, can_skip))) { - LOG_WARN("failed to get ls replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); - } else if (can_skip) { - LOG_INFO("cur ls item can be skip", K(ret), K(ls_item), K(ls_cs_replica_cache)); - continue; - } else if (OB_FAIL(ls_cs_replica_cache.check_is_cs_replica(ls_item, is_cs_replica))) { - LOG_WARN("fail to check is cs replica", K(ret), K(ls_item), K(ls_cs_replica_cache)); - } else if (OB_FAIL(data_checksum_checker.check_data_checksum(item, is_cs_replica))) { + if (OB_FAIL(data_checksum_checker.check_data_checksum(item))) { LOG_ERROR("failed to verify data checksum", K(ret), K(tenant_id), K(tablet_id), - K(ls_id), K(compaction_scn), K(item), K(filter_items), K(is_cs_replica), K(ls_cs_replica_cache)); + K(ls_id), K(compaction_scn), K(item), K(filter_items)); } else if (OB_FAIL(item.verify_column_checksum(first_item))) { LOG_ERROR("failed to verify column checksum", K(ret), K(tenant_id), K(tablet_id), - K(ls_id), K(compaction_scn), K(first_item), K(item), K(filter_items), K(is_cs_replica), K(ls_cs_replica_cache)); + K(ls_id), K(compaction_scn), K(first_item), K(item), K(filter_items)); } } } diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 384e9951e..87505e387 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -5740,7 +5740,7 @@ int ObTablet::update_tablet_autoinc_seq(const uint64_t autoinc_seq) return ret; } -int ObTablet::check_cs_replica_compat_schema(bool &is_cs_replica_compat) +int ObTablet::check_cs_replica_compat_schema(bool &is_cs_replica_compat) const { int ret = OB_SUCCESS; is_cs_replica_compat = false; @@ -6150,6 +6150,7 @@ int ObTablet::get_tablet_report_info_by_sstable( ObArray column_checksums; column_checksums.set_attr(ObMemAttr(MTL_ID(), "tmpCkmArr")); ObSSTable *table = nullptr; + bool is_cs_replica_compat = false; if (OB_UNLIKELY(nullptr == main_major || report_major_snapshot != main_major->get_snapshot_version())) { if (GCTX.is_shared_storage_mode()) { ret = OB_EAGAIN; @@ -6193,6 +6194,8 @@ int ObTablet::get_tablet_report_info_by_sstable( LOG_WARN("fail to init a tablet replica", KR(ret), "tablet_id", get_tablet_id(), K(tablet_replica)); } else if (!need_checksums) { // do nothing + } else if (OB_FAIL(check_cs_replica_compat_schema(is_cs_replica_compat))) { + LOG_WARN("fail to check cs replica compat", K(ret), K(is_cs_replica_compat)); } else if (OB_FAIL(get_sstable_column_checksum(*main_major, column_checksums))) { LOG_WARN("fail to get sstable column checksum", K(ret), KPC(main_major)); } else if (OB_FAIL(tablet_checksum.set_tenant_id(MTL_ID()))) { @@ -6207,6 +6210,7 @@ int ObTablet::get_tablet_report_info_by_sstable( tablet_checksum.server_ = addr; tablet_checksum.row_count_ = get_tablet_meta().report_status_.row_count_; tablet_checksum.data_checksum_ = get_tablet_meta().report_status_.data_checksum_; + tablet_checksum.data_checksum_type_ = is_cs_replica_compat ? ObDataChecksumType::DATA_CHECKSUM_COLUMN_STORE : ObDataChecksumType::DATA_CHECKSUM_NORMAL; LOG_INFO("success to get tablet report info", KR(ret), "tablet_id", get_tablet_id(), "report_status", tablet_meta_.report_status_, K(tablet_checksum)); } @@ -7905,7 +7909,7 @@ int ObTablet::get_sstable_column_checksum( ObArenaAllocator allocator; if (OB_FAIL(load_storage_schema(allocator, storage_schema))) { LOG_WARN("fail to load storage schema", K(ret)); - } else if (OB_FAIL(static_cast(&sstable)->fill_column_ckm_array(*storage_schema, column_checksums))) { + } else if (OB_FAIL(static_cast(&sstable)->fill_column_ckm_array(*storage_schema, column_checksums, storage_schema->is_cs_replica_compat()))) { LOG_WARN("fail to fill_column_ckm_array", K(ret), K(sstable)); } ObTabletObjLoadHelper::free(allocator, storage_schema); diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 22dd789a1..16d25a9b0 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -467,7 +467,7 @@ public: int get_migration_sstable_size(int64_t &data_size); // column store replica - int check_cs_replica_compat_schema(bool &is_cs_replica_compat); + int check_cs_replica_compat_schema(bool &is_cs_replica_compat) const; int pre_process_cs_replica(ObTabletDirectLoadInsertParam &direct_load_param); // other diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index 7330cf68b..88668b539 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -2951,6 +2951,11 @@ END_CG_ID bigint(20) NO NULL KEPT_SNAPSHOT varchar(128) NO NULL MERGE_LEVEL varchar(64) NO NULL EXEC_MODE varchar(64) NO NULL +IS_FULL_MERGE varchar(5) NO NULL +IO_COST_TIME_PERCENTAGE bigint(20) NO NULL +MERGE_REASON varchar(32) NO NULL +BASE_MAJOR_STATUS varchar(64) NO NULL +CO_MERGE_TYPE varchar(64) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_TABLET_COMPACTION_HISTORY limit 1); cnt 1 @@ -2987,6 +2992,11 @@ END_CG_ID bigint(20) NO KEPT_SNAPSHOT varchar(128) NO MERGE_LEVEL varchar(64) NO EXEC_MODE varchar(64) NO +IS_FULL_MERGE varchar(5) NO +IO_COST_TIME_PERCENTAGE bigint(20) NO +MERGE_REASON varchar(32) NO +BASE_MAJOR_STATUS varchar(64) NO +CO_MERGE_TYPE varchar(64) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TABLET_COMPACTION_HISTORY limit 1); cnt 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 0b69e9329..d55600af8 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -4237,6 +4237,11 @@ END_CG_ID bigint(20) NO NULL KEPT_SNAPSHOT varchar(128) NO NULL MERGE_LEVEL varchar(64) NO NULL EXEC_MODE varchar(64) NO NULL +IS_FULL_MERGE varchar(5) NO NULL +IO_COST_TIME_PERCENTAGE bigint(20) NO NULL +MERGE_REASON varchar(32) NO NULL +BASE_MAJOR_STATUS varchar(64) NO NULL +CO_MERGE_TYPE varchar(64) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.GV$OB_TABLET_COMPACTION_HISTORY limit 1); cnt 1 @@ -4273,6 +4278,11 @@ END_CG_ID bigint(20) NO KEPT_SNAPSHOT varchar(128) NO MERGE_LEVEL varchar(64) NO EXEC_MODE varchar(64) NO +IS_FULL_MERGE varchar(5) NO +IO_COST_TIME_PERCENTAGE bigint(20) NO +MERGE_REASON varchar(32) NO +BASE_MAJOR_STATUS varchar(64) NO +CO_MERGE_TYPE varchar(64) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_TABLET_COMPACTION_HISTORY limit 1); cnt 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result index c4c69f080..d2926a06d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result @@ -1438,6 +1438,11 @@ end_cg_id bigint(20) NO NULL kept_snapshot varchar(128) NO NULL merge_level varchar(64) NO NULL exec_mode varchar(64) NO NULL +is_full_merge tinyint(4) NO NULL +io_cost_time_percentage bigint(20) NO NULL +merge_reason varchar(32) NO NULL +base_major_status varchar(64) NO NULL +co_merge_type varchar(64) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_tablet_compaction_history; IF(count(*) >= 0, 1, 0) 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index 9ebcd5bba..cf4b1756b 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -1862,6 +1862,11 @@ end_cg_id bigint(20) NO NULL kept_snapshot varchar(128) NO NULL merge_level varchar(64) NO NULL exec_mode varchar(64) NO NULL +is_full_merge tinyint(4) NO NULL +io_cost_time_percentage bigint(20) NO NULL +merge_reason varchar(32) NO NULL +base_major_status varchar(64) NO NULL +co_merge_type varchar(64) NO NULL select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_tablet_compaction_history; IF(count(*) >= 0, 1, 0) 1 @@ -5946,6 +5951,7 @@ row_count bigint(20) NO NULL data_checksum bigint(20) NO NULL column_checksums longtext YES NULL b_column_checksums varbinary(65536) YES NULL +data_checksum_type bigint(20) NO 0 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_tablet_replica_checksum; IF(count(*) >= 0, 1, 0) 1 diff --git a/unittest/share/compaction/CMakeLists.txt b/unittest/share/compaction/CMakeLists.txt index a1cd90d17..9f218b268 100644 --- a/unittest/share/compaction/CMakeLists.txt +++ b/unittest/share/compaction/CMakeLists.txt @@ -1 +1,2 @@ -storage_unittest(test_compaction_time_guard) \ No newline at end of file +storage_unittest(test_compaction_time_guard) +storage_unittest(test_checksum_operator) \ No newline at end of file diff --git a/unittest/share/compaction/test_checksum_operator.cpp b/unittest/share/compaction/test_checksum_operator.cpp new file mode 100644 index 000000000..c82a7d740 --- /dev/null +++ b/unittest/share/compaction/test_checksum_operator.cpp @@ -0,0 +1,84 @@ +/** + * Copyright (c) 2024 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include "share/ob_dml_sql_splicer.h" +#include +#define USING_LOG_PREFIX STORAGE +#define protected public +#define private public + +namespace oceanbase +{ +namespace unittest +{ + +TEST(ObPTSqlSplicer, batch) { + int ret = OB_SUCCESS; + share::ObDMLSqlSplicer splicer; + ObSqlString sql; + ObString result("INSERT INTO tname (gmt_modified, gmt_create, tenant_id, tablet_id, svr_ip, svr_port, ls_id, compaction_scn, row_count, data_checksum, column_checksums, b_column_checksums) VALUES " + "(now(6),now(6),1004,200001,'127.0.0.1',1001,1001,1234567890,100,1234567,'column_checksums-01','b_column_checksums-01'), " + "(now(6),now(6),1004,200001,'127.0.0.2',1002,1001,1234567890,100,1234567,'column_checksums-02','b_column_checksums-02'), " + "(now(6),now(6),1004,200001,'127.0.0.3',1003,1001,1234567890,100,1234567,'column_checksums-03','b_column_checksums-03') " + "ON DUPLICATE KEY UPDATE " + "gmt_modified=VALUES(gmt_modified)," + "gmt_create=VALUES(gmt_create)," + "tenant_id=VALUES(tenant_id)," + "tablet_id=VALUES(tablet_id)," + "svr_ip=VALUES(svr_ip)," + "svr_port=VALUES(svr_port)," + "ls_id=VALUES(ls_id)," + "compaction_scn=VALUES(compaction_scn)," + "row_count=VALUES(row_count)," + "data_checksum=VALUES(data_checksum)," + "column_checksums=VALUES(column_checksums)," + "b_column_checksums=VALUES(b_column_checksums)"); + + ObAddr ips[3]; + ObString column_checksums[3] = {"column_checksums-01", "column_checksums-02", "column_checksums-03"}; + ObString b_column_checksums[3] = {"b_column_checksums-01", "b_column_checksums-02", "b_column_checksums-03"}; + ASSERT_TRUE(ips[0].set_ip_addr("127.0.0.1", 1001)); + ASSERT_TRUE(ips[1].set_ip_addr("127.0.0.2", 1002)); + ASSERT_TRUE(ips[2].set_ip_addr("127.0.0.3", 1003)); + char ip[OB_MAX_SERVER_ADDR_SIZE] = ""; + + for (int64_t i = 0; i < 3; i++) { + ASSERT_TRUE(ips[i].ip_to_string(ip, sizeof(ip))); + ASSERT_EQ(OB_SUCCESS, splicer.add_gmt_modified()); + ASSERT_EQ(OB_SUCCESS, splicer.add_gmt_create()); + ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("tenant_id", 1004)); + ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("tablet_id", 200001)); + ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("svr_ip", ip)); + ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("svr_port", ips[i].get_port())); + ASSERT_EQ(OB_SUCCESS, splicer.add_pk_column("ls_id", 1001)); + ASSERT_EQ(OB_SUCCESS, splicer.add_column("compaction_scn", 1234567890)); + ASSERT_EQ(OB_SUCCESS, splicer.add_column("row_count", 100)); + ASSERT_EQ(OB_SUCCESS, splicer.add_column("data_checksum", 1234567)); + ASSERT_EQ(OB_SUCCESS, splicer.add_column("column_checksums", column_checksums[i])); + ASSERT_EQ(OB_SUCCESS, splicer.add_column("b_column_checksums", b_column_checksums[i])); + ASSERT_EQ(OB_SUCCESS, splicer.finish_row()); + } + ASSERT_EQ(OB_SUCCESS, splicer.splice_batch_insert_update_sql("tname", sql)); + LOG_INFO("finish splice batch insert update sql", K(ret), K(sql), K(result)); + ASSERT_EQ(0, result.compare(sql.string())); +} + +} // end unittest +} // end namespace oceanbase + +int main(int argc, char **argv) +{ + oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); + OB_LOGGER.set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}