diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 2d09016e81..7b72ef2cfc 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -5293,6 +5293,7 @@ int64_t ObEstPartArgElement::get_serialize_size(void) const OB_UNIS_ADD_LEN(tablet_id_); OB_UNIS_ADD_LEN(ls_id_); OB_UNIS_ADD_LEN(tenant_id_); + OB_UNIS_ADD_LEN(tx_id_); return len; } @@ -5309,6 +5310,7 @@ int ObEstPartArgElement::serialize(char *buf, OB_UNIS_ENCODE(tablet_id_); OB_UNIS_ENCODE(ls_id_); OB_UNIS_ENCODE(tenant_id_); + OB_UNIS_ENCODE(tx_id_); return ret; } @@ -5330,6 +5332,7 @@ int ObEstPartArgElement::deserialize(common::ObIAllocator &allocator, OB_UNIS_DECODE(tablet_id_); OB_UNIS_DECODE(ls_id_); OB_UNIS_DECODE(tenant_id_); + OB_UNIS_DECODE(tx_id_); return ret; } diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index cfc53db703..f6d4c2abdc 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -6595,7 +6595,7 @@ public: struct ObEstPartArgElement { ObEstPartArgElement() : batch_(), scan_flag_(), - index_id_(common::OB_INVALID_ID), range_columns_count_(0), tablet_id_(), ls_id_(), tenant_id_(0) + index_id_(common::OB_INVALID_ID), range_columns_count_(0), tablet_id_(), ls_id_(), tenant_id_(0), tx_id_() {} // Essentially, we can use ObIArray here // For compatibility reason, we still use ObSimpleBatch @@ -6606,6 +6606,7 @@ struct ObEstPartArgElement ObTabletID tablet_id_; share::ObLSID ls_id_; uint64_t tenant_id_; + transaction::ObTransID tx_id_; TO_STRING_KV( K(scan_flag_), @@ -6614,7 +6615,8 @@ struct ObEstPartArgElement K(range_columns_count_), K(tablet_id_), K(ls_id_), - K(tenant_id_)); + K(tenant_id_), + K(tx_id_)); int64_t get_serialize_size(void) const; int serialize(char *buf, const int64_t buf_len, int64_t &pos) const; int deserialize(common::ObIAllocator &allocator, diff --git a/src/sql/optimizer/ob_access_path_estimation.cpp b/src/sql/optimizer/ob_access_path_estimation.cpp index f604bd553e..6c6cedeae6 100644 --- a/src/sql/optimizer/ob_access_path_estimation.cpp +++ b/src/sql/optimizer/ob_access_path_estimation.cpp @@ -587,6 +587,7 @@ int ObAccessPathEstimation::add_index_info(ObOptimizerContext &ctx, index_est_arg->tablet_id_ = part.tablet_id_; index_est_arg->ls_id_ = part.ls_id_; index_est_arg->tenant_id_ = ctx.get_session_info()->get_effective_tenant_id(); + index_est_arg->tx_id_ = ctx.get_session_info()->get_tx_id(); } // FIXME, move following codes if (OB_SUCC(ret)) { @@ -1101,6 +1102,7 @@ int ObAccessPathEstimation::storage_estimate_full_table_rowcount(ObOptimizerCont path_arg.tablet_id_ = best_index_part.tablet_id_; path_arg.ls_id_ = best_index_part.ls_id_; path_arg.tenant_id_ = ctx.get_session_info()->get_effective_tenant_id(); + path_arg.tx_id_ = ctx.get_session_info()->get_tx_id(); if (OB_FAIL(ObSQLUtils::make_whole_range(arena, meta.ref_table_id_, meta.table_rowkey_count_, diff --git a/src/sql/optimizer/ob_storage_estimator.cpp b/src/sql/optimizer/ob_storage_estimator.cpp index dfaad09ca4..cbd87514c5 100644 --- a/src/sql/optimizer/ob_storage_estimator.cpp +++ b/src/sql/optimizer/ob_storage_estimator.cpp @@ -36,6 +36,7 @@ int ObStorageEstimator::estimate_row_count(const obrpc::ObEstPartArg &arg, param.scan_flag_ = arg.index_params_.at(i).scan_flag_; param.tablet_id_ = arg.index_params_.at(i).tablet_id_; param.ls_id_ = arg.index_params_.at(i).ls_id_; + param.tx_id_ = arg.index_params_.at(i).tx_id_; if (OB_FAIL(storage_estimate_rowcount( arg.index_params_.at(i).tenant_id_, param, diff --git a/src/storage/access/ob_table_estimator.cpp b/src/storage/access/ob_table_estimator.cpp index c04f637c1b..73c23b5c06 100644 --- a/src/storage/access/ob_table_estimator.cpp +++ b/src/storage/access/ob_table_estimator.cpp @@ -117,7 +117,7 @@ int ObTableEstimator::estimate_multi_scan_row_count( LOG_WARN("failed to estimate sstable row count", K(ret), K(*current_table)); } } else if (current_table->is_data_memtable()) { - if (OB_FAIL(estimate_memtable_scan_row_count(base_input.query_flag_, base_input.table_id_, + if (OB_FAIL(estimate_memtable_scan_row_count(base_input, static_cast(current_table), range, tmp_cost))) { LOG_WARN("failed to estimate memtable row count", K(ret), K(*current_table)); } @@ -157,12 +157,13 @@ int ObTableEstimator::estimate_sstable_scan_row_count( K(key_range), K(part_est), K(sizeof(scan_estimator))); } } + LOG_DEBUG("[STORAGE ESTIMATE ROW]", K(ret), K(base_input.table_id_), + K(base_input.tx_id_), K(part_est), K(key_range), KPC(sstable)); return ret; } int ObTableEstimator::estimate_memtable_scan_row_count( - const common::ObQueryFlag query_flag, - const uint64_t table_id, + const ObTableEstimateBaseInput &base_input, const memtable::ObMemtable *memtable, const ObDatumRange &key_range, storage::ObPartitionEst &part_est) @@ -179,15 +180,17 @@ int ObTableEstimator::estimate_memtable_scan_row_count( } else { memtable::ObMvccScanRange mvcc_scan_range; ObDatumRange real_range; - memtable->m_get_real_range(real_range, key_range, query_flag.is_reverse_scan()); + memtable->m_get_real_range(real_range, key_range, base_input.query_flag_.is_reverse_scan()); memtable::ObMemtableKey start_key(&(real_range.get_start_key().get_store_rowkey())); memtable::ObMemtableKey end_key(&(real_range.get_end_key().get_store_rowkey())); mvcc_scan_range.border_flag_ = real_range.get_border_flag(); mvcc_scan_range.start_key_ = &start_key; mvcc_scan_range.end_key_ = &end_key; - if (OB_FAIL(memtable->get_mvcc_engine().estimate_scan_row_count(mvcc_scan_range, part_est))){ - LOG_WARN("Fail to estimate cost of scan.", K(ret), K(table_id)); + if (OB_FAIL(memtable->get_mvcc_engine().estimate_scan_row_count(base_input.tx_id_, mvcc_scan_range, part_est))){ + LOG_WARN("Fail to estimate cost of scan.", K(ret), K(base_input.table_id_)); } + LOG_DEBUG("[STORAGE ESTIMATE ROW]", K(ret), K(base_input.table_id_), K(base_input.tx_id_), + K(part_est), K(key_range), KPC(memtable)); } return ret; diff --git a/src/storage/access/ob_table_estimator.h b/src/storage/access/ob_table_estimator.h index 6b5d32a7fe..39604ea625 100644 --- a/src/storage/access/ob_table_estimator.h +++ b/src/storage/access/ob_table_estimator.h @@ -32,10 +32,12 @@ struct ObTableEstimateBaseInput { ObTableEstimateBaseInput( const common::ObQueryFlag query_flag, const uint64_t table_id, + const transaction::ObTransID tx_id, const ObTableArray &tables, const ObTabletHandle &tablet_handle) : query_flag_(query_flag), table_id_(table_id), + tx_id_(tx_id), tables_(tables), tablet_handle_(tablet_handle) {} @@ -45,6 +47,7 @@ struct ObTableEstimateBaseInput { const common::ObQueryFlag query_flag_; const uint64_t table_id_; + const transaction::ObTransID tx_id_; const ObTableArray &tables_; const ObTabletHandle &tablet_handle_; }; @@ -69,8 +72,7 @@ public: ObPartitionEst &part_est); static int estimate_memtable_scan_row_count( - const common::ObQueryFlag query_flag, - const uint64_t table_id, + const ObTableEstimateBaseInput &base_input, const memtable::ObMemtable *memtable, const blocksstable::ObDatumRange &key_range, ObPartitionEst &part_est); diff --git a/src/storage/access/ob_table_scan_iterator.cpp b/src/storage/access/ob_table_scan_iterator.cpp index 7fced76586..30990250b8 100644 --- a/src/storage/access/ob_table_scan_iterator.cpp +++ b/src/storage/access/ob_table_scan_iterator.cpp @@ -486,7 +486,8 @@ int ObTableScanIterator::can_retire_to_row_sample(bool &retire) } else if (memtables.count() > 0) { ObPartitionEst batch_est; ObSEArray est_records; - ObTableEstimateBaseInput base_input(scan_param_->scan_flag_, memtables.at(0)->get_key().tablet_id_.id(), memtables, get_table_param_.tablet_iter_.tablet_handle_); + ObTableEstimateBaseInput base_input(scan_param_->scan_flag_, memtables.at(0)->get_key().tablet_id_.id(), + transaction::ObTransID(), memtables, get_table_param_.tablet_iter_.tablet_handle_); if (OB_FAIL(ObTableEstimator::estimate_row_count_for_scan(base_input, table_scan_range_.get_ranges(), batch_est, est_records))) { STORAGE_LOG(WARN, "Failed to estimate row count for scan", K(ret), KPC(scan_param_), K(table_scan_range_)); } else { diff --git a/src/storage/compaction/ob_partition_merge_progress.cpp b/src/storage/compaction/ob_partition_merge_progress.cpp index e0fe257402..1630492bdb 100644 --- a/src/storage/compaction/ob_partition_merge_progress.cpp +++ b/src/storage/compaction/ob_partition_merge_progress.cpp @@ -149,7 +149,7 @@ int ObPartitionMergeProgress::estimate(ObTabletMergeCtx *ctx) false, /*full row scan flag, obsoleted*/ false, /*index back*/ false); /*query_stat*/ - ObTableEstimateBaseInput base_input(query_flag, tables.at(0)->get_key().tablet_id_.id(), tables, ctx->tablet_handle_); + ObTableEstimateBaseInput base_input(query_flag, tables.at(0)->get_key().tablet_id_.id(), transaction::ObTransID(), tables, ctx->tablet_handle_); ObDatumRange whole_range; whole_range.set_whole_range(); diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 077ea8f0fa..69e02c2527 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -5648,7 +5648,7 @@ int ObLSTabletService::estimate_row_count( } } if (OB_SUCC(ret) && tables.count() > 0) { - ObTableEstimateBaseInput base_input(param.scan_flag_, param.index_id_, tables, tablet_iter.tablet_handle_); + ObTableEstimateBaseInput base_input(param.scan_flag_, param.index_id_, param.tx_id_, tables, tablet_iter.tablet_handle_); if (scan_range.is_get()) { if (OB_FAIL(ObTableEstimator::estimate_row_count_for_get(base_input, scan_range.get_rowkeys(), batch_est))) { LOG_WARN("failed to estimate row count", K(ret), K(param), K(scan_range)); diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp index dfaeca5584..6eb7505774 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.cpp @@ -202,6 +202,7 @@ int ObMvccEngine::scan( } int ObMvccEngine::estimate_scan_row_count( + const transaction::ObTransID &tx_id, const ObMvccScanRange &range, ObPartitionEst &part_est) const { @@ -214,6 +215,7 @@ int ObMvccEngine::estimate_scan_row_count( TRANS_LOG(WARN, "invalid param"); ret = OB_INVALID_ARGUMENT; } else if (OB_FAIL(query_engine_->estimate_row_count( + tx_id, range.start_key_, !range.border_flag_.inclusive_start(), range.end_key_, !range.border_flag_.inclusive_end(), part_est.logical_row_count_, part_est.physical_row_count_))) { diff --git a/src/storage/memtable/mvcc/ob_mvcc_engine.h b/src/storage/memtable/mvcc/ob_mvcc_engine.h index 2f5557b822..2b358d8f5d 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_engine.h +++ b/src/storage/memtable/mvcc/ob_mvcc_engine.h @@ -113,7 +113,8 @@ public: const ObMemtableKey *key, storage::ObStoreRowLockState &lock_state); // estimate_scan_row_count estimate the row count for the range - int estimate_scan_row_count(const ObMvccScanRange &range, + int estimate_scan_row_count(const transaction::ObTransID &tx_id, + const ObMvccScanRange &range, storage::ObPartitionEst &part_est) const; private: int try_compact_row_when_mvcc_read_(const share::SCN &snapshot_version, diff --git a/src/storage/memtable/mvcc/ob_query_engine.cpp b/src/storage/memtable/mvcc/ob_query_engine.cpp index 4eed95102a..71ab60c537 100644 --- a/src/storage/memtable/mvcc/ob_query_engine.cpp +++ b/src/storage/memtable/mvcc/ob_query_engine.cpp @@ -425,6 +425,7 @@ void ObQueryEngine::revert_iter(ObIQueryEngineIterator *iter) int ObQueryEngine::sample_rows(Iterator *iter, const ObMemtableKey *start_key, const int start_exclude, const ObMemtableKey *end_key, const int end_exclude, + const transaction::ObTransID &tx_id, int64_t &logical_row_count, int64_t &physical_row_count, double &ratio) { int ret = OB_SUCCESS; @@ -470,7 +471,12 @@ int ObQueryEngine::sample_rows(Iterator *iter, const ObMemtabl } gap_size = 0; } - if (blocksstable::ObDmlFlag::DF_INSERT == value->first_dml_flag_ + if (blocksstable::ObDmlFlag::DF_NOT_EXIST == value->first_dml_flag_ && + blocksstable::ObDmlFlag::DF_NOT_EXIST == value->last_dml_flag_ && + nullptr != value->list_head_ && + value->list_head_->tx_id_ == tx_id) { + ++logical_row_count; + } else if (blocksstable::ObDmlFlag::DF_INSERT == value->first_dml_flag_ && blocksstable::ObDmlFlag::DF_DELETE != value->last_dml_flag_) { // insert new row ++logical_row_count; @@ -654,7 +660,8 @@ int ObQueryEngine::split_range(const ObMemtableKey *start_key, } -int ObQueryEngine::estimate_row_count(const ObMemtableKey *start_key, const int start_exclude, +int ObQueryEngine::estimate_row_count(const transaction::ObTransID &tx_id, + const ObMemtableKey *start_key, const int start_exclude, const ObMemtableKey *end_key, const int end_exclude, int64_t &logical_row_count, int64_t &physical_row_count) { @@ -681,12 +688,12 @@ int ObQueryEngine::estimate_row_count(const ObMemtableKey *start_key, const int } else if (OB_ISNULL((iter = raw_iter_alloc_.alloc()))) { TRANS_LOG(WARN, "alloc raw iter fail"); ret = OB_ALLOCATE_MEMORY_FAILED; - } else if (OB_FAIL(sample_rows(iter, end_key, end_exclude, start_key, start_exclude, + } else if (OB_FAIL(sample_rows(iter, end_key, end_exclude, start_key, start_exclude, tx_id, log_row_count1, phy_row_count1, ratio1))) { if (OB_ITER_END != ret) { TRANS_LOG(WARN, "failed to sample rows reverse", KR(ret), K(*start_key), K(*end_key)); } - } else if (OB_FAIL(sample_rows(iter, start_key, start_exclude, end_key, end_exclude, + } else if (OB_FAIL(sample_rows(iter, start_key, start_exclude, end_key, end_exclude, tx_id, log_row_count2, phy_row_count2, ratio2))) { if (OB_ITER_END != ret) { TRANS_LOG(WARN, "failed to sample rows", KR(ret), K(*start_key), K(*end_key)); diff --git a/src/storage/memtable/mvcc/ob_query_engine.h b/src/storage/memtable/mvcc/ob_query_engine.h index b356b14f99..3c6b88dade 100644 --- a/src/storage/memtable/mvcc/ob_query_engine.h +++ b/src/storage/memtable/mvcc/ob_query_engine.h @@ -201,7 +201,8 @@ public: const ObMemtableKey *end_key, int64_t part_count, common::ObIArray &range_array); - int estimate_row_count(const ObMemtableKey *start_key, const int start_exclude, + int estimate_row_count(const transaction::ObTransID &tx_id, + const ObMemtableKey *start_key, const int start_exclude, const ObMemtableKey *end_key, const int end_exclude, int64_t &logical_row_count, int64_t &physical_row_count); int dump_keyhash(FILE *fd) const @@ -246,6 +247,7 @@ public: private: int sample_rows(Iterator *iter, const ObMemtableKey *start_key, const int start_exclude, const ObMemtableKey *end_key, const int end_exclude, + const transaction::ObTransID &tx_id, int64_t &logical_row_count, int64_t &physical_row_count, double &ratio); int init_raw_iter_for_estimate(Iterator*& iter, const ObMemtableKey *start_key,