From 9e1fc64a14949dbada649f6a30fb6efaa561924b Mon Sep 17 00:00:00 2001 From: obdev Date: Thu, 13 Apr 2023 06:24:58 +0000 Subject: [PATCH] optimize compaction statistic on tablet stat --- src/share/stat/ob_opt_stat_service.cpp | 2 +- .../compaction/ob_partition_merge_iter.cpp | 21 +++++++ .../compaction/ob_partition_merge_iter.h | 8 +++ .../compaction/ob_partition_merge_policy.cpp | 2 +- .../compaction/ob_partition_merger.cpp | 30 ++++++++++ src/storage/compaction/ob_partition_merger.h | 5 ++ .../compaction/ob_partition_rows_merger.cpp | 25 +++++++++ .../compaction/ob_partition_rows_merger.h | 9 +++ .../compaction/ob_tablet_merge_ctx.cpp | 3 +- src/storage/compaction/ob_tablet_merge_ctx.h | 2 + .../compaction/ob_tablet_merge_task.cpp | 34 +++-------- .../mvcc/ob_multi_version_iterator.cpp | 27 ++++++++- .../memtable/mvcc/ob_multi_version_iterator.h | 9 +++ src/storage/memtable/ob_memtable_iterator.cpp | 14 +++++ src/storage/memtable/ob_memtable_iterator.h | 7 +++ src/storage/ob_tenant_tablet_stat_mgr.cpp | 56 ++++++++++++++++--- src/storage/ob_tenant_tablet_stat_mgr.h | 29 ++++++++-- .../storage/test_tenant_tablet_stat_mgr.cpp | 8 --- 18 files changed, 241 insertions(+), 50 deletions(-) diff --git a/src/share/stat/ob_opt_stat_service.cpp b/src/share/stat/ob_opt_stat_service.cpp index 0350c3c41..c9a7a1d88 100644 --- a/src/share/stat/ob_opt_stat_service.cpp +++ b/src/share/stat/ob_opt_stat_service.cpp @@ -334,7 +334,7 @@ int ObOptStatService::get_table_rowcnt(const uint64_t tenant_id, } } LOG_TRACE("cache stat compare", KPC(handle.stat_), K(tablet_stat)); - if (handle.stat_->get_row_count() < tablet_stat.merge_logical_row_cnt_) { + if (handle.stat_->get_row_count() < tablet_stat.insert_row_cnt_ - tablet_stat.delete_row_cnt_) { if (OB_FAIL(reload_tablet_ids.push_back(all_tablet_ids.at(i))) || OB_FAIL(reload_ls_ids.push_back(all_ls_ids.at(i)))) { LOG_WARN("failed to push back", K(ret)); diff --git a/src/storage/compaction/ob_partition_merge_iter.cpp b/src/storage/compaction/ob_partition_merge_iter.cpp index d2b396745..1ed72f60a 100644 --- a/src/storage/compaction/ob_partition_merge_iter.cpp +++ b/src/storage/compaction/ob_partition_merge_iter.cpp @@ -1238,6 +1238,27 @@ bool ObPartitionMinorRowMergeIter::is_curr_row_commiting() const return bret; } +int ObPartitionMinorRowMergeIter::collect_tnode_dml_stat( + storage::ObTransNodeDMLStat &tnode_stat) const +{ + int ret = OB_SUCCESS; + memtable::ObMemtableMultiVersionScanIterator *iter = nullptr; + + if (OB_ISNULL(table_)) { + // do nothing + } else if (OB_UNLIKELY(!table_->is_memtable())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("only support to get mt stat from memtable", K(ret), KPC(table_)); + } else if (OB_ISNULL(iter = reinterpret_cast(row_iter_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null row iter", K(ret), KPC(row_iter_)); + } else if (OB_FAIL(iter->get_tnode_stat(tnode_stat))) { + LOG_WARN("failed to get mt stat", K(ret)); + } + return ret; +} + + /* *ObPartitionMinorMacroMergeIter */ diff --git a/src/storage/compaction/ob_partition_merge_iter.h b/src/storage/compaction/ob_partition_merge_iter.h index 53ba85adf..d45cc10e7 100644 --- a/src/storage/compaction/ob_partition_merge_iter.h +++ b/src/storage/compaction/ob_partition_merge_iter.h @@ -30,6 +30,12 @@ namespace oceanbase { using namespace blocksstable; + +namespace storage +{ +struct ObTransNodeDMLStat; +} + namespace compaction { struct ObMergeParameter; @@ -82,6 +88,7 @@ public: virtual int get_curr_micro_block(const blocksstable::ObMicroBlock *µ_block) {UNUSED(micro_block); return OB_NOT_SUPPORTED; } virtual int64_t get_iter_row_count() const { return iter_row_count_; } virtual int64_t get_ghost_row_count() const { return 0; } + virtual int collect_tnode_dml_stat(storage::ObTransNodeDMLStat &tnode_stat) const { UNUSED(tnode_stat); return OB_NOT_SUPPORTED; } OB_INLINE bool is_compact_completed_row() const { bool bret = false; @@ -221,6 +228,7 @@ public: virtual int64_t get_ghost_row_count() const override { return ghost_row_count_; } virtual OB_INLINE bool is_multi_version_minor_iter() const { return true; } virtual bool is_curr_row_commiting() const; + virtual int collect_tnode_dml_stat(storage::ObTransNodeDMLStat &tnode_stat) const override; virtual void reset_first_multi_version_row_flag() { if (nullptr != curr_row_) { diff --git a/src/storage/compaction/ob_partition_merge_policy.cpp b/src/storage/compaction/ob_partition_merge_policy.cpp index f530d118c..1033dc2b2 100644 --- a/src/storage/compaction/ob_partition_merge_policy.cpp +++ b/src/storage/compaction/ob_partition_merge_policy.cpp @@ -1522,7 +1522,7 @@ int ObAdaptiveMergePolicy::check_tombstone_situation( || ls_id.id() != tablet_stat.ls_id_ || tablet_id.id() != tablet_stat.tablet_id_) { ret = OB_INVALID_ARGUMENT; LOG_WARN("get invalid arguments", K(ret), K(tablet), K(tablet_stat)); - } else if (tablet_stat.is_hot_tablet() && tablet_stat.is_update_mostly()) { + } else if (tablet_stat.is_hot_tablet() && (tablet_stat.is_update_mostly() || tablet_stat.is_delete_mostly())) { reason = AdaptiveMergeReason::TOMBSTONE_SCENE; } LOG_DEBUG("check_tombstone_situation", K(ret), K(ls_id), K(tablet_id), K(reason), K(tablet_stat)); diff --git a/src/storage/compaction/ob_partition_merger.cpp b/src/storage/compaction/ob_partition_merger.cpp index d98907e09..e8a522d22 100644 --- a/src/storage/compaction/ob_partition_merger.cpp +++ b/src/storage/compaction/ob_partition_merger.cpp @@ -1161,11 +1161,41 @@ int ObPartitionMinorMerger::merge_partition(ObTabletMergeCtx &ctx, const int64_t } } else if (OB_FAIL(close())){ STORAGE_LOG(WARN, "failed to close partition merger", K(ret)); + } else { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(collect_merge_stat(merge_param.merge_type_, merge_helper, ctx))) { + STORAGE_LOG(WARN, "failed to collect merge stat", K(tmp_ret), K(merge_param)); + } } } return ret; } +int ObPartitionMinorMerger::collect_merge_stat( + const ObMergeType &merge_type, + ObPartitionMinorMergeHelper &merge_helper, + ObTabletMergeCtx &ctx) +{ + int ret = OB_SUCCESS; + ObTransNodeDMLStat tnode_stat; + + if (OB_UNLIKELY(!is_mini_merge(merge_type))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid argument", K(ret), K(merge_type)); + } else if (OB_FAIL(merge_helper.collect_tnode_dml_stat(merge_type, tnode_stat))) { + STORAGE_LOG(WARN, "failed to get memtable stat", K(ret)); + } else if (tnode_stat.empty()) { + // do nothing + } else if (OB_LIKELY(1 >= ctx.get_concurrent_cnt())) { + // serial mini compaction + ctx.tnode_stat_ = tnode_stat; + } else { + // parallel mini compaction + ctx.tnode_stat_.atomic_inc(tnode_stat); + } + return ret; +} + int ObPartitionMinorMerger::merge_single_iter(ObPartitionMergeIter &merge_iter) { int ret = OB_SUCCESS; diff --git a/src/storage/compaction/ob_partition_merger.h b/src/storage/compaction/ob_partition_merger.h index daded6ab8..04c4e9974 100644 --- a/src/storage/compaction/ob_partition_merger.h +++ b/src/storage/compaction/ob_partition_merger.h @@ -40,6 +40,7 @@ namespace compaction { struct ObTabletMergeCtx; struct ObMergeParameter; +class ObPartitionMinorMergeHelper; class ObPartitionMerger { @@ -144,6 +145,10 @@ protected: virtual int append_bloom_filter(const blocksstable::ObDatumRow &row); virtual int rewrite_macro_block(MERGE_ITER_ARRAY &minimum_iters) override; private: + int collect_merge_stat( + const storage::ObMergeType &merge_type, + compaction::ObPartitionMinorMergeHelper &merge_helper, + ObTabletMergeCtx &ctx); int check_add_shadow_row(MERGE_ITER_ARRAY &merge_iters, const bool contain_multi_trans, bool &add_shadow_row); int merge_single_iter(ObPartitionMergeIter &merge_ite); int check_first_committed_row(const MERGE_ITER_ARRAY &merge_iters); diff --git a/src/storage/compaction/ob_partition_rows_merger.cpp b/src/storage/compaction/ob_partition_rows_merger.cpp index 61789e67d..c4a0a2d87 100644 --- a/src/storage/compaction/ob_partition_rows_merger.cpp +++ b/src/storage/compaction/ob_partition_rows_merger.cpp @@ -952,6 +952,31 @@ ObPartitionMergeIter *ObPartitionMajorMergeHelper::alloc_merge_iter(const ObMerg * ---------------------------------------------------------ObPartitionMinorMergeHelper-------------------------------------------------------------- */ +int ObPartitionMinorMergeHelper::collect_tnode_dml_stat( + const ObMergeType &merge_type, + storage::ObTransNodeDMLStat &tnode_stat) const +{ + int ret = OB_SUCCESS; + tnode_stat.reset(); + + if (OB_UNLIKELY(!is_mini_merge(merge_type))) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "get invalid argument", K(ret), K(merge_type)); + } + + const MERGE_ITER_ARRAY &merge_iters = get_merge_iters(); + for (int64_t i = 0; OB_SUCC(ret) && i < merge_iters.count(); ++i) { + const ObPartitionMergeIter *iter = merge_iters.at(i); + if (OB_ISNULL(iter)) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "get unexpected null merge iter", K(ret)); + } else if (OB_FAIL(iter->collect_tnode_dml_stat(tnode_stat))) { + STORAGE_LOG(WARN, "failed to collect mt stat", K(ret)); + } + } + return ret; +} + ObPartitionMergeIter *ObPartitionMinorMergeHelper::alloc_merge_iter(const ObMergeParameter &merge_param, const bool is_base_iter, const bool is_small_sstable) diff --git a/src/storage/compaction/ob_partition_rows_merger.h b/src/storage/compaction/ob_partition_rows_merger.h index f031adf3f..28889fd76 100644 --- a/src/storage/compaction/ob_partition_rows_merger.h +++ b/src/storage/compaction/ob_partition_rows_merger.h @@ -21,6 +21,12 @@ namespace oceanbase { + +namespace storage +{ +struct ObTransNodeDMLStat; +} + namespace compaction { struct ObPartitionMergeLoserTreeItem @@ -205,6 +211,9 @@ public: ObPartitionMinorMergeHelper() {} virtual ~ObPartitionMinorMergeHelper() { reset(); } + int collect_tnode_dml_stat( + const ObMergeType &merge_type, + storage::ObTransNodeDMLStat &tnode_stat) const; protected: ObPartitionMergeIter *alloc_merge_iter(const ObMergeParameter &merge_param, const bool is_base_iter, const bool is_small_sstable) override; }; diff --git a/src/storage/compaction/ob_tablet_merge_ctx.cpp b/src/storage/compaction/ob_tablet_merge_ctx.cpp index aee11d364..8af904991 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_tablet_merge_ctx.cpp @@ -529,7 +529,8 @@ ObTabletMergeCtx::ObTabletMergeCtx( time_guard_(), rebuild_seq_(-1), data_version_(0), - merge_list_() + merge_list_(), + tnode_stat_() { merge_scn_.set_max(); } diff --git a/src/storage/compaction/ob_tablet_merge_ctx.h b/src/storage/compaction/ob_tablet_merge_ctx.h index f9fa191ee..6a20edf20 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.h +++ b/src/storage/compaction/ob_tablet_merge_ctx.h @@ -24,6 +24,7 @@ #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_handle.h" #include "share/scn.h" +#include "storage/ob_tenant_tablet_stat_mgr.h" namespace oceanbase { @@ -252,6 +253,7 @@ public: int64_t rebuild_seq_; uint64_t data_version_; ObMediumCompactionInfoList merge_list_; + ObTransNodeDMLStat tnode_stat_; // collect trans node dml stat on memtable, only worked in mini compaction. TO_STRING_KV(K_(param), K_(sstable_version_range), K_(create_snapshot_version), K_(is_full_merge), K_(merge_level), diff --git a/src/storage/compaction/ob_tablet_merge_task.cpp b/src/storage/compaction/ob_tablet_merge_task.cpp index 181fb4e6f..0ada754b7 100644 --- a/src/storage/compaction/ob_tablet_merge_task.cpp +++ b/src/storage/compaction/ob_tablet_merge_task.cpp @@ -1193,37 +1193,21 @@ int ObTabletMergeFinishTask::try_report_tablet_stat_after_mini(ObTabletMergeCtx { int ret = OB_SUCCESS; const ObTabletID &tablet_id = ctx.param_.tablet_id_; - ObQueryFlag query_flag(ObQueryFlag::Forward, - true, /*is daily merge scan*/ - true, /*is read multiple macro block*/ - true, /*sys task scan, read one macro block in single io*/ - false, /*full row scan flag, obsoleted*/ - false, /*index back*/ - false); /*query_stat*/ - ObTableEstimateBaseInput base_input(query_flag, - tablet_id.id(), - ctx.tables_handle_.get_tables(), - ctx.tablet_handle_); - ObDatumRange whole_range; - whole_range.set_whole_range(); - ObSEArray ranges; - ObPartitionEst part_estimate; - ObSEArray records; - if (OB_FAIL(ranges.push_back(whole_range))) { - LOG_WARN("failed to add ranges", K(ret), K(ranges), K(whole_range)); - } else if (OB_FAIL(ObTableEstimator::estimate_row_count_for_scan( - base_input, ranges, part_estimate, records))) { - LOG_WARN("failed to estimate row counts", K(ret), K(part_estimate), K(records)); - } else if (0 == part_estimate.logical_row_count_ && - ObTabletStat::MERGE_REPORT_MIN_ROW_CNT >= part_estimate.physical_row_count_) { + const ObTransNodeDMLStat &tnode_stat = ctx.tnode_stat_; + + if (tablet_id.is_special_merge_tablet()) { + // no need report + } else if (ObTabletStat::MERGE_REPORT_MIN_ROW_CNT >= tnode_stat.get_dml_count()) { + // insufficient data, skip to report } else { ObTabletStat report_stat; report_stat.ls_id_ = ctx.param_.ls_id_.id(), report_stat.tablet_id_ = ctx.param_.tablet_id_.id(); report_stat.merge_cnt_ = 1; - report_stat.merge_logical_row_cnt_ = part_estimate.logical_row_count_; - report_stat.merge_physical_row_cnt_ = part_estimate.physical_row_count_; + report_stat.insert_row_cnt_ = tnode_stat.insert_row_count_; + report_stat.update_row_cnt_ = tnode_stat.update_row_count_; + report_stat.delete_row_cnt_ = tnode_stat.delete_row_count_; if (OB_FAIL(MTL(ObTenantTabletStatMgr *)->report_stat(report_stat))) { STORAGE_LOG(WARN, "failed to report tablet stat", K(ret)); } diff --git a/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp b/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp index a96133d80..6201c96f8 100644 --- a/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp +++ b/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp @@ -23,6 +23,7 @@ #include "storage/tx/ob_trans_part_ctx.h" #include "storage/tx/ob_tx_data_functor.h" #include "storage/tx_table/ob_tx_table.h" +#include "storage/ob_tenant_tablet_stat_mgr.h" namespace oceanbase { @@ -477,7 +478,10 @@ ObMultiVersionRowIterator::ObMultiVersionRowIterator() version_range_(), value_iter_(), query_engine_(NULL), - query_engine_iter_(NULL) + query_engine_iter_(NULL), + insert_row_count_(0), + update_row_count_(0), + delete_row_count_(0) { } @@ -592,11 +596,28 @@ int ObMultiVersionRowIterator::try_cleanout_tx_node_(ObMvccRow *value, *tnode, false /*need_row_latch*/))) { TRANS_LOG(WARN, "cleanout tx state failed", K(ret), K(*value), K(*tnode)); + } else if (!tnode->is_aborted()) { + const blocksstable::ObDmlFlag dml_flag = tnode->get_dml_flag(); + + if (blocksstable::ObDmlFlag::DF_INSERT == dml_flag) { + ++insert_row_count_; + } else if (blocksstable::ObDmlFlag::DF_UPDATE == dml_flag) { + ++update_row_count_; + } else if (blocksstable::ObDmlFlag::DF_DELETE == dml_flag) { + ++delete_row_count_; + } } return ret; } +void ObMultiVersionRowIterator::get_tnode_dml_stat(storage::ObTransNodeDMLStat &stat) const +{ + stat.insert_row_count_ += insert_row_count_; + stat.update_row_count_ += update_row_count_; + stat.delete_row_count_ += delete_row_count_; +} + void ObMultiVersionRowIterator::reset() { is_inited_ = false; @@ -609,6 +630,10 @@ void ObMultiVersionRowIterator::reset() query_engine_iter_ = NULL; } query_engine_ = NULL; + + insert_row_count_ = 0; + update_row_count_ = 0; + delete_row_count_ = 0; } diff --git a/src/storage/memtable/mvcc/ob_multi_version_iterator.h b/src/storage/memtable/mvcc/ob_multi_version_iterator.h index ef99aedc6..b80470d1c 100644 --- a/src/storage/memtable/mvcc/ob_multi_version_iterator.h +++ b/src/storage/memtable/mvcc/ob_multi_version_iterator.h @@ -24,6 +24,11 @@ namespace oceanbase { +namespace storage +{ +struct ObTransNodeDMLStat; +} + namespace memtable { class ObMvccAccessCtx; @@ -112,6 +117,7 @@ public: const common::ObVersionRange &version_range, const ObMvccScanRange &range); int get_next_row(const ObMemtableKey *&key, ObMultiVersionValueIterator *&value_iter); + void get_tnode_dml_stat(storage::ObTransNodeDMLStat &mt_stat) const; void reset(); private: int try_cleanout_mvcc_row_(ObMvccRow *value); @@ -124,6 +130,9 @@ private: ObMultiVersionValueIterator value_iter_; ObQueryEngine *query_engine_; ObIQueryEngineIterator *query_engine_iter_; + int64_t insert_row_count_; + int64_t update_row_count_; + int64_t delete_row_count_; }; } diff --git a/src/storage/memtable/ob_memtable_iterator.cpp b/src/storage/memtable/ob_memtable_iterator.cpp index 8c9811111..98045d1b5 100644 --- a/src/storage/memtable/ob_memtable_iterator.cpp +++ b/src/storage/memtable/ob_memtable_iterator.cpp @@ -26,6 +26,7 @@ #include "ob_memtable_context.h" #include "ob_memtable.h" #include "storage/blocksstable/ob_datum_row.h" +#include "storage/ob_tenant_tablet_stat_mgr.h" namespace oceanbase { @@ -792,6 +793,19 @@ int ObMemtableMultiVersionScanIterator::init( return ret; } +int ObMemtableMultiVersionScanIterator::get_tnode_stat( + storage::ObTransNodeDMLStat &tnode_stat) const +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", K(ret), KPC(this)); + } else { + row_iter_.get_tnode_dml_stat(tnode_stat); + } + return ret; +} + // The row format is as follows: //rowkey｜opposite number of trans_version｜opposite number of Sql_seuqnce | non-rowkey columns // diff --git a/src/storage/memtable/ob_memtable_iterator.h b/src/storage/memtable/ob_memtable_iterator.h index d26869f17..687db3bf5 100644 --- a/src/storage/memtable/ob_memtable_iterator.h +++ b/src/storage/memtable/ob_memtable_iterator.h @@ -32,6 +32,11 @@ namespace oceanbase { +namespace storage +{ +struct ObTransNodeDMLStat; +} + namespace memtable { @@ -246,6 +251,8 @@ public: storage::ObTableAccessContext &context, storage::ObITable *table, const void *query_range) override; + int get_tnode_stat(storage::ObTransNodeDMLStat &tnode_stat) const; + public: virtual int inner_get_next_row(const blocksstable::ObDatumRow *&row); virtual void reset(); diff --git a/src/storage/ob_tenant_tablet_stat_mgr.cpp b/src/storage/ob_tenant_tablet_stat_mgr.cpp index 83ed4ba18..e2a211698 100644 --- a/src/storage/ob_tenant_tablet_stat_mgr.cpp +++ b/src/storage/ob_tenant_tablet_stat_mgr.cpp @@ -15,6 +15,25 @@ using namespace oceanbase::common; using namespace oceanbase::storage; +bool ObTransNodeDMLStat::empty() const +{ + return 0 >= insert_row_count_ && + 0 >= update_row_count_ && + 0 >= delete_row_count_; +} + +void ObTransNodeDMLStat::atomic_inc(const ObTransNodeDMLStat &other) +{ + if (other.empty()) { + // do nothing + } else { + (void) ATOMIC_AAFx(&insert_row_count_, other.insert_row_count_, 0/*placeholder*/); + (void) ATOMIC_AAFx(&update_row_count_, other.update_row_count_, 0/*placeholder*/); + (void) ATOMIC_AAFx(&delete_row_count_, other.delete_row_count_, 0/*placeholder*/); + } +} + + /************************************* ObTabletStatKey *************************************/ ObTabletStatKey::ObTabletStatKey( const int64_t ls_id, @@ -88,7 +107,7 @@ bool ObTabletStat::check_need_report() const bret = true; } } else if (0 != merge_cnt_) { // report by compaction - bret = MERGE_REPORT_MIN_ROW_CNT <= merge_physical_row_cnt_; + bret = MERGE_REPORT_MIN_ROW_CNT <= insert_row_cnt_ + update_row_cnt_ + delete_row_cnt_; } else { // invalid tablet stat bret = false; } @@ -116,8 +135,9 @@ ObTabletStat& ObTabletStat::operator+=(const ObTabletStat &other) pushdown_micro_block_cnt_ += other.pushdown_micro_block_cnt_; exist_row_total_table_cnt_ += other.exist_row_total_table_cnt_; exist_row_read_table_cnt_ += other.exist_row_read_table_cnt_; - merge_physical_row_cnt_ += other.merge_physical_row_cnt_; - merge_logical_row_cnt_ += other.merge_logical_row_cnt_; + insert_row_cnt_ += other.insert_row_cnt_; + update_row_cnt_ += other.update_row_cnt_; + delete_row_cnt_ += other.delete_row_cnt_; } return *this; } @@ -133,8 +153,9 @@ ObTabletStat& ObTabletStat::archive(int64_t factor) pushdown_micro_block_cnt_ /= factor; exist_row_total_table_cnt_ /= factor; exist_row_read_table_cnt_ /= factor; - merge_physical_row_cnt_ /= factor; - merge_logical_row_cnt_ /= factor; + insert_row_cnt_ /= factor; + update_row_cnt_ /= factor; + delete_row_cnt_ /= factor; } return *this; } @@ -147,9 +168,11 @@ bool ObTabletStat::is_hot_tablet() const bool ObTabletStat::is_insert_mostly() const { bool bret = false; - if (merge_physical_row_cnt_ < BASIC_ROW_CNT_THRESHOLD) { + uint64_t total_row_cnt = insert_row_cnt_ + update_row_cnt_ + delete_row_cnt_; + if (total_row_cnt < BASIC_ROW_CNT_THRESHOLD) { + // do nothing } else { - bret = merge_logical_row_cnt_ >= (merge_physical_row_cnt_ / BASE_FACTOR * INSERT_PIVOT_FACTOR); + bret = insert_row_cnt_ * BASE_FACTOR / total_row_cnt >= INSERT_PIVOT_FACTOR; } return bret; } @@ -157,13 +180,28 @@ bool ObTabletStat::is_insert_mostly() const bool ObTabletStat::is_update_mostly() const { bool bret = false; - if (0 == merge_physical_row_cnt_ || merge_physical_row_cnt_ < BASIC_ROW_CNT_THRESHOLD) { + uint64_t total_row_cnt = insert_row_cnt_ + update_row_cnt_ + delete_row_cnt_; + if (total_row_cnt < BASIC_ROW_CNT_THRESHOLD) { + // do nothing } else { - bret = merge_logical_row_cnt_ >= (merge_physical_row_cnt_ / BASE_FACTOR * UPDATE_PIVOT_FACTOR); + bret = update_row_cnt_ * BASE_FACTOR / total_row_cnt >= UPDATE_PIVOT_FACTOR; } return bret; } +bool ObTabletStat::is_delete_mostly() const +{ + bool bret = false; + uint64_t total_row_cnt = insert_row_cnt_ + update_row_cnt_ + delete_row_cnt_; + if (total_row_cnt < BASIC_ROW_CNT_THRESHOLD) { + // do nothing + } else { + bret = delete_row_cnt_ * BASE_FACTOR / total_row_cnt >= DELETE_PIVOT_FACTOR; + } + return bret; +} + + bool ObTabletStat::is_inefficient_scan() const { bool bret = false; diff --git a/src/storage/ob_tenant_tablet_stat_mgr.h b/src/storage/ob_tenant_tablet_stat_mgr.h index 71cf4f4a3..fe4f67cc6 100644 --- a/src/storage/ob_tenant_tablet_stat_mgr.h +++ b/src/storage/ob_tenant_tablet_stat_mgr.h @@ -23,6 +23,24 @@ namespace oceanbase namespace storage { +struct ObTransNodeDMLStat +{ +public: + ObTransNodeDMLStat() { reset(); } + ~ObTransNodeDMLStat() { reset(); } + void reset() { MEMSET(this, 0, sizeof(*this)); } + bool empty() const; + void atomic_inc(const ObTransNodeDMLStat &other); + int64_t get_dml_count() const { return insert_row_count_ + update_row_count_ + delete_row_count_; } + + TO_STRING_KV(K_(insert_row_count), K_(update_row_count), K_(delete_row_count)); +public: + int64_t insert_row_count_; + int64_t update_row_count_; + int64_t delete_row_count_; +}; + + struct ObTabletStatKey { public: @@ -56,19 +74,21 @@ public: bool is_hot_tablet() const; bool is_insert_mostly() const; bool is_update_mostly() const; + bool is_delete_mostly() const; bool is_inefficient_scan() const; bool is_inefficient_insert() const; bool is_inefficient_pushdown() const; TO_STRING_KV(K_(ls_id), K_(tablet_id), K_(query_cnt), K_(merge_cnt), K_(scan_logical_row_cnt), K_(scan_physical_row_cnt), K_(scan_micro_block_cnt), K_(pushdown_micro_block_cnt), - K_(exist_row_total_table_cnt), K_(exist_row_read_table_cnt), K_(merge_physical_row_cnt), - K_(merge_logical_row_cnt)); + K_(exist_row_total_table_cnt), K_(exist_row_read_table_cnt), K_(insert_row_cnt), + K_(update_row_cnt), K_(delete_row_cnt)); public: static constexpr int64_t ACCESS_FREQUENCY = 5; static constexpr int64_t BASE_FACTOR = 10; static constexpr int64_t INSERT_PIVOT_FACTOR = 5; static constexpr int64_t UPDATE_PIVOT_FACTOR = 4; + static constexpr int64_t DELETE_PIVOT_FACTOR = 3; static constexpr int64_t SCAN_READ_FACTOR = 2; static constexpr int64_t EXIST_READ_FACTOR = 7; static constexpr int64_t BASIC_TABLE_CNT_THRESHOLD = 5; @@ -89,8 +109,9 @@ public: uint64_t pushdown_micro_block_cnt_; uint64_t exist_row_total_table_cnt_; uint64_t exist_row_read_table_cnt_; - uint64_t merge_physical_row_cnt_; - uint64_t merge_logical_row_cnt_; + uint64_t insert_row_cnt_; + uint64_t update_row_cnt_; + uint64_t delete_row_cnt_; }; diff --git a/unittest/storage/test_tenant_tablet_stat_mgr.cpp b/unittest/storage/test_tenant_tablet_stat_mgr.cpp index fba68606e..704553d8c 100644 --- a/unittest/storage/test_tenant_tablet_stat_mgr.cpp +++ b/unittest/storage/test_tenant_tablet_stat_mgr.cpp @@ -139,8 +139,6 @@ TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_bucket) tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 100; tablet_stat.scan_physical_row_cnt_ = 100; - tablet_stat.merge_physical_row_cnt_ = 100; - tablet_stat.merge_logical_row_cnt_ = 100; { int64_t step = 1; @@ -205,8 +203,6 @@ TEST_F(TestTenantTabletStatMgr, basic_tablet_stream) tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 100; tablet_stat.scan_physical_row_cnt_ = 100; - tablet_stat.merge_physical_row_cnt_ = 100; - tablet_stat.merge_logical_row_cnt_ = 100; ObTabletStream stream; auto &curr_buckets = stream.curr_buckets_; @@ -249,8 +245,6 @@ TEST_F(TestTenantTabletStatMgr, get_all_tablet_stat) tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 100; tablet_stat.scan_physical_row_cnt_ = 100; - tablet_stat.merge_physical_row_cnt_ = 100; - tablet_stat.merge_logical_row_cnt_ = 100; ObTabletStream stream; auto &curr_buckets = stream.curr_buckets_; @@ -388,8 +382,6 @@ TEST_F(TestTenantTabletStatMgr, basic_tablet_stat_mgr) tablet_stat.query_cnt_ = 100; tablet_stat.scan_logical_row_cnt_ = 100; tablet_stat.scan_physical_row_cnt_ = 100; - tablet_stat.merge_physical_row_cnt_ = 100; - tablet_stat.merge_logical_row_cnt_ = 100; ret = stat_mgr_->report_stat(tablet_stat); ASSERT_EQ(OB_SUCCESS, ret);