hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

reduce the memory of defensive check mgr

Browse Source
This commit is contained in:
obdev
2022-11-02 20:11:05 +00:00
committed by wangzelin.wzl
parent 4a9e45d350
commit 5077661704
9 changed files with 129 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/storage/access/ob_single_merge.cpp
View File

@ -322,16 +322,17 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row)
#ifdef ENABLE_DEBUG_LOG
if (OB_SUCC(ret)) {
access_ctx_->defensive_check_record_.query_flag_ = access_ctx_->query_flag_;
if (NULL != access_ctx_->store_ctx_
&& NULL != access_ctx_->store_ctx_->mvcc_acc_ctx_.get_mem_ctx()
&& NULL != access_ctx_->store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_defensive_check_mgr()) {
(void)access_ctx_->store_ctx_
->mvcc_acc_ctx_
.get_mem_ctx()
->get_defensive_check_mgr()->put(tablet_meta.tablet_id_,
row,
*rowkey_,
access_ctx_->defensive_check_record_);
transaction::ObTransService *trx = MTL(transaction::ObTransService *);
bool trx_id_valid = (NULL != access_ctx_->store_ctx_
&& access_ctx_->store_ctx_->mvcc_acc_ctx_.tx_id_.is_valid());
if (OB_NOT_NULL(trx)
&& trx_id_valid
&& NULL != trx->get_defensive_check_mgr()) {
(void)trx->get_defensive_check_mgr()->put(tablet_meta.tablet_id_,
access_ctx_->store_ctx_->mvcc_acc_ctx_.tx_id_,
row,
*rowkey_,
access_ctx_->defensive_check_record_);
}
}
access_ctx_->defensive_check_record_.reset();

9
src/storage/ls/ob_ls_tablet_service.cpp
View File

@ -4707,9 +4707,12 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss(
}
#ifdef ENABLE_DEBUG_LOG
// print single row check info
if (NULL != store_ctx.mvcc_acc_ctx_.get_mem_ctx()
&& NULL != store_ctx.mvcc_acc_ctx_.get_mem_ctx()->get_defensive_check_mgr()) {
(void)store_ctx.mvcc_acc_ctx_.get_mem_ctx()->get_defensive_check_mgr()->dump(access_param.iter_param_.tablet_id_);
if (store_ctx.mvcc_acc_ctx_.tx_id_.is_valid()) {
transaction::ObTransService *trx = MTL(transaction::ObTransService *);
if (OB_NOT_NULL(trx)
&& NULL != trx->get_defensive_check_mgr()) {
(void)trx->get_defensive_check_mgr()->dump(store_ctx.mvcc_acc_ctx_.tx_id_);
}
}
#endif
}

21
src/storage/memtable/ob_memtable_context.cpp
View File

@ -28,7 +28,6 @@
#include "storage/tablelock/ob_table_lock_callback.h"
#include "storage/tablelock/ob_table_lock_common.h"
#include "storage/tx/ob_trans_deadlock_adapter.h"
#include "storage/tx/ob_defensive_check_mgr.h"
namespace oceanbase
{
@ -64,9 +63,6 @@ ObMemtableCtx::ObMemtableCtx()
is_master_(true),
read_elr_data_(false),
lock_mem_ctx_(ctx_cb_allocator_),
#ifdef ENABLE_DEBUG_LOG
defensive_check_mgr_(NULL),
#endif
is_inited_(false)
{
}
@ -93,17 +89,6 @@ int ObMemtableCtx::init(const uint64_t tenant_id)
TRANS_LOG(ERROR, "ctx_allocator_ init error", K(ret));
} else if (OB_FAIL(reset_log_generator_())) {
TRANS_LOG(ERROR, "fail to reset log generator", K(ret));
#ifdef ENABLE_DEBUG_LOG
} else if (!GCONF.enable_defensive_check()) {
// do nothing
} else if (NULL == (defensive_check_mgr_ = op_alloc(ObDefensiveCheckMgr))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
TRANS_LOG(ERROR, "memory alloc failed", K(ret), KP(defensive_check_mgr_));
} else if (OB_FAIL(defensive_check_mgr_->init(lib::ObMemAttr(tenant_id, "MemtableCtx")))) {
TRANS_LOG(ERROR, "defensive check mgr init failed", K(ret), KP(defensive_check_mgr_));
op_free(defensive_check_mgr_);
defensive_check_mgr_ = NULL;
#endif
} else {
// do nothing
}
@ -173,12 +158,6 @@ void ObMemtableCtx::reset()
tx_table_guard_.reset();
//FIXME: ObIMemtableCtx don't have resetfunction,
//thus ObIMvccCtx::reset is called, so resource_link_is not reset
#ifdef ENABLE_DEBUG_LOG
if (NULL != defensive_check_mgr_) {
op_free(defensive_check_mgr_);
defensive_check_mgr_ = NULL;
}
#endif
ObIMemtableCtx::reset();
}
}

6
src/storage/memtable/ob_memtable_context.h
View File

@ -484,9 +484,6 @@ public:
// for deadlock detect.
void set_table_lock_killed() { lock_mem_ctx_.set_killed(); }
bool is_table_lock_killed() const { return lock_mem_ctx_.is_killed(); }
#ifdef ENABLE_DEBUG_LOG
transaction::ObDefensiveCheckMgr *get_defensive_check_mgr() { return defensive_check_mgr_; }
#endif
private:
int do_trans_end(
const bool commit,
@ -559,9 +556,6 @@ private:
common::ObArray<transaction::ObTransID> conflict_trans_ids_;
// table lock mem ctx.
transaction::tablelock::ObLockMemCtx lock_mem_ctx_;
#ifdef ENABLE_DEBUG_LOG
transaction::ObDefensiveCheckMgr *defensive_check_mgr_;
#endif
bool is_inited_;
};

82
src/storage/tx/ob_defensive_check_mgr.cpp
View File

@ -27,12 +27,14 @@ void SingleRowDefensiveRecord::reset()
generate_ts_ = 0;
rowkey_.reset();
row_.reset();
tablet_id_.reset();
allocator_.reset();
}
int SingleRowDefensiveRecord::deep_copy(const blocksstable::ObDatumRow &row,
const blocksstable::ObDatumRowkey &rowkey,
const ObDefensiveCheckRecordExtend &extend_info)
const ObDefensiveCheckRecordExtend &extend_info,
const ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
@ -44,6 +46,7 @@ int SingleRowDefensiveRecord::deep_copy(const blocksstable::ObDatumRow &row,
TRANS_LOG(WARN, "datum row deep copy error", K(ret), K(row));
} else {
extend_info_ = extend_info;
tablet_id_ = tablet_id;
generate_ts_ = ObTimeUtility::current_time();
}
@ -52,7 +55,7 @@ int SingleRowDefensiveRecord::deep_copy(const blocksstable::ObDatumRow &row,
void ObSingleTabletDefensiveCheckInfo::reset()
{
tablet_id_.reset();
tx_id_.reset();
for (int64_t i = 0; i < record_arr_.count(); ++i) {
if (NULL != record_arr_.at(i)) {
op_free(record_arr_.at(i));
@ -61,15 +64,15 @@ void ObSingleTabletDefensiveCheckInfo::reset()
}
}
int ObSingleTabletDefensiveCheckInfo::init(const ObTabletID &tablet_id)
int ObSingleTabletDefensiveCheckInfo::init(const ObTransID &tx_id)
{
int ret = OB_SUCCESS;
if (!tablet_id.is_valid()) {
if (!tx_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
TRANS_LOG(WARN, "invalid argument", K(ret), K(tablet_id));
TRANS_LOG(WARN, "invalid argument", K(ret), K(tx_id));
} else {
tablet_id_ = tablet_id;
tx_id_ = tx_id;
}
return ret;
@ -117,6 +120,7 @@ void ObDefensiveCheckMgr::reset()
}
int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id,
const ObTransID &tx_id,
const blocksstable::ObDatumRow &row,
const blocksstable::ObDatumRowkey &rowkey,
const ObDefensiveCheckRecordExtend &extend_info)
@ -130,27 +134,31 @@ int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id,
SingleRowDefensiveRecord *r = NULL;
if (NULL == (r = op_alloc(SingleRowDefensiveRecord))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
TRANS_LOG(WARN, "alloc memory fail", K(ret), K(tablet_id));
} else if (OB_FAIL(r->deep_copy(row, rowkey, extend_info))) {
TRANS_LOG(WARN, "Defensive reocrd deep copy error", K(ret), K(row), K(rowkey));
TRANS_LOG(WARN, "alloc memory fail", K(ret), K(tablet_id), K(tx_id));
} else if (OB_FAIL(r->deep_copy(row, rowkey, extend_info, tablet_id))) {
TRANS_LOG(WARN, "Defensive reocrd deep copy error", K(ret),
K(row),
K(rowkey),
K(extend_info),
K(tablet_id));
} else {
Guard spin_guard(lock_);
ObSingleTabletDefensiveCheckInfo *info = NULL;
if (OB_FAIL(map_.get(tablet_id, info))) {
if (OB_FAIL(map_.get(tx_id, info))) {
if (OB_ENTRY_NOT_EXIST != ret) {
TRANS_LOG(WARN, "map get error", K(ret), K(tablet_id));
TRANS_LOG(WARN, "map get error", K(ret), K(tx_id), K(tablet_id));
} else if (NULL == (info = op_alloc(ObSingleTabletDefensiveCheckInfo))
|| OB_FAIL(info->init(tablet_id))) {
|| OB_FAIL(info->init(tx_id))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
TRANS_LOG(WARN, "Defensive info alloc or init error", K(tablet_id));
TRANS_LOG(WARN, "Defensive info alloc or init error", K(tx_id), K(tablet_id));
} else if (OB_FAIL(info->add_record(r))) {
TRANS_LOG(WARN, "add defensive record error", K(ret));
} else if (OB_FAIL(map_.insert_and_get(tablet_id, info, NULL))) {
TRANS_LOG(WARN, "defensive check info insert map error", K(ret), K(tablet_id), KP(info));
} else if (OB_FAIL(map_.insert_and_get(tx_id, info, NULL))) {
TRANS_LOG(WARN, "defensive check info insert map error", K(ret), K(tx_id), K(tablet_id), KP(info));
} else {
TRANS_LOG(DEBUG, "add record", K(tablet_id), K(row), K(rowkey), K(extend_info));
TRANS_LOG(DEBUG, "add record", K(tx_id), K(tablet_id), K(row), K(rowkey), K(extend_info));
// do nothing
}
if (OB_FAIL(ret)) {
@ -165,7 +173,7 @@ int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id,
} else if (OB_FAIL(info->add_record(r))) {
TRANS_LOG(WARN, "add defensive record error", K(ret));
} else {
TRANS_LOG(DEBUG, "add record", K(tablet_id), K(row), K(rowkey), K(extend_info));
TRANS_LOG(DEBUG, "add record", K(tx_id), K(tablet_id), K(row), K(rowkey), K(extend_info));
// do nothing
}
if (OB_SUCC(ret) && NULL != info) {
@ -181,7 +189,7 @@ int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id,
return ret;
}
void ObDefensiveCheckMgr::dump(const ObTabletID &tablet_id)
void ObDefensiveCheckMgr::del(const ObTransID &tx_id)
{
int ret = OB_SUCCESS;
@ -193,23 +201,43 @@ void ObDefensiveCheckMgr::dump(const ObTabletID &tablet_id)
Guard spin_guard(lock_);
ObSingleTabletDefensiveCheckInfo *info = NULL;
if (OB_FAIL(map_.get(tablet_id, info))) {
if (OB_FAIL(map_.get(tx_id, info))) {
if (OB_ENTRY_NOT_EXIST != ret) {
TRANS_LOG(WARN, "map get error", K(ret), K(tablet_id));
TRANS_LOG(WARN, "map get error", K(ret), K(tx_id));
}
} else {
(void)map_.del(tx_id, info);
(void)map_.revert(info);
}
}
UNUSED(ret);
}
void ObDefensiveCheckMgr::dump(const ObTransID &tx_id)
{
int ret = OB_SUCCESS;
if (!is_inited_) {
ret = OB_NOT_INIT;
TRANS_LOG(WARN, "ObDefensiveCheckMgr not init ", K(ret));
} else {
Guard spin_guard(lock_);
ObSingleTabletDefensiveCheckInfo *info = NULL;
if (OB_FAIL(map_.get(tx_id, info))) {
if (OB_ENTRY_NOT_EXIST != ret) {
TRANS_LOG(WARN, "map get error", K(ret), K(tx_id));
}
} else {
const int64_t max_count = std::min(max_record_cnt_, info->get_record_arr().count());
TRANS_LOG(INFO, "###################### start to print defensice check info##########################",
K(max_count), "tablet_id", info->get_tablet_id());
K(max_count), K(tx_id), K(max_count));
for (int64_t i = 0; i < max_count; ++i) {
TRANS_LOG(INFO, "print single row defensive check info",
"rowkey", info->get_record_arr().at(i)->rowkey_,
"row", info->get_record_arr().at(i)->row_,
"extend_info", info->get_record_arr().at(i)->extend_info_);
TRANS_LOG(INFO, "print single row defensive check info", "record", *(info->get_record_arr().at(i)));
}
TRANS_LOG(INFO, "##################### print defensice check info finished #########################",
K(max_count),
"tablet_id", info->get_tablet_id());
K(max_count), K(tx_id), K(max_count));
map_.revert(info);
}

22
src/storage/tx/ob_defensive_check_mgr.h
View File

@ -59,14 +59,16 @@ public:
void destroy() { reset(); }
int deep_copy(const blocksstable::ObDatumRow &row,
const blocksstable::ObDatumRowkey &rowkey,
const ObDefensiveCheckRecordExtend &extend_info);
const ObDefensiveCheckRecordExtend &extend_info,
const ObTabletID &tablet_id);
TO_STRING_KV(K_(row), K_(generate_ts), K_(rowkey), K_(extend_info));
TO_STRING_KV(K_(row), K_(generate_ts), K_(rowkey), K_(tablet_id), K_(extend_info));
blocksstable::ObDatumRow row_;
int64_t generate_ts_;
ObArenaAllocator allocator_;
blocksstable::ObDatumRowkey rowkey_;
ObTabletID tablet_id_;
ObDefensiveCheckRecordExtend extend_info_;
};
@ -77,15 +79,15 @@ class ObSingleTabletDefensiveCheckInfo : public ObTransHashLink<ObSingleTabletDe
public:
ObSingleTabletDefensiveCheckInfo() { }
~ObSingleTabletDefensiveCheckInfo() { reset(); }
int init(const ObTabletID &tablet_id);
int init(const ObTransID &tx_id);
void reset();
void destroy() { reset(); }
bool contain(const ObTabletID &tablet_id) { return tablet_id_ == tablet_id; }
bool contain(const ObTransID &tx_id) { return tx_id_ == tx_id; }
int add_record(SingleRowDefensiveRecord *record);
ObSingleRowDefensiveRecordArray &get_record_arr() { return record_arr_; }
const ObTabletID &get_tablet_id() const { return tablet_id_; }
const ObTransID &get_tx_id() const { return tx_id_; }
private:
ObTabletID tablet_id_;
ObTransID tx_id_;
ObSingleRowDefensiveRecordArray record_arr_;
private:
DISALLOW_COPY_AND_ASSIGN(ObSingleTabletDefensiveCheckInfo);
@ -109,10 +111,10 @@ public:
}
};
typedef ObTransHashMap<ObTabletID,
typedef ObTransHashMap<ObTransID,
ObSingleTabletDefensiveCheckInfo,
ObSingleTabletDefensiveCheckInfoAlloc,
common::SpinRWLock, 64 /*bucket_num*/> ObTxDefensiveCheckInfoMap;
common::SpinRWLock, 2 << 16 /*bucket_num*/> ObTxDefensiveCheckInfoMap;
class ObDefensiveCheckMgr
{
@ -123,10 +125,12 @@ public:
void reset();
void destroy() { reset(); }
int put(const ObTabletID &tablet_id,
const ObTransID &tx_id,
const blocksstable::ObDatumRow &row,
const blocksstable::ObDatumRowkey &rowkey,
const ObDefensiveCheckRecordExtend &extend_info);
void dump(const ObTabletID &tablet_id);
void del(const ObTransID &tx_id);
void dump(const ObTransID &tx_id);
private:
static int64_t max_record_cnt_;
typedef ObSmallSpinLockGuard<common::ObByteLock> Guard;

6
src/storage/tx/ob_trans_part_ctx.cpp
View File

@ -184,6 +184,12 @@ void ObPartTransCtx::destroy()
if (OB_LIKELY(is_inited_)) {
TRANS_LOG(DEBUG, "part_ctx_destroy", K(*this));
#ifdef ENABLE_DEBUG_LOG
if (NULL != trans_service_->get_defensive_check_mgr()) {
trans_service_->get_defensive_check_mgr()->del(trans_id_);
}
#endif
// Defensive Check 1 : earse ctx id descriptor
mt_ctx_.reset();

32
src/storage/tx/ob_trans_service.cpp
View File

@ -61,6 +61,9 @@ ObTransService::ObTransService()
server_tracer_(NULL),
input_queue_count_(0),
output_queue_count_(0),
#ifdef ENABLE_DEBUG_LOG
defensive_check_mgr_(NULL),
#endif
tx_desc_mgr_(*this)
{
check_env_();
@ -159,6 +162,28 @@ int ObTransService::init(const ObAddr &self,
is_inited_ = true;
TRANS_LOG(INFO, "transaction service inited success", KP(this));
}
if (OB_SUCC(ret)) {
#ifdef ENABLE_DEBUG_LOG
void *p = NULL;
if (!GCONF.enable_defensive_check()) {
// do nothing
} else if (NULL == (p = ob_malloc(sizeof(ObDefensiveCheckMgr),
lib::ObMemAttr(tenant_id, "ObDefenCheckMgr")))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
TRANS_LOG(WARN, "memory alloc failed", KR(ret));
} else {
defensive_check_mgr_ = new(p) ObDefensiveCheckMgr();
if (OB_FAIL(defensive_check_mgr_->init(lib::ObMemAttr(tenant_id, "ObDefenCheckMgr")))) {
TRANS_LOG(ERROR, "defensive check mgr init failed", K(ret), KP(defensive_check_mgr_));
defensive_check_mgr_->destroy();
ob_free(defensive_check_mgr_);
defensive_check_mgr_ = NULL;
} else {
// do nothing
}
}
#endif
}
return ret;
}
@ -267,6 +292,13 @@ void ObTransService::destroy()
tx_ctx_mgr_.destroy();
tx_desc_mgr_.destroy();
dup_table_rpc_->destroy();
#ifdef ENABLE_DEBUG_LOG
if (NULL != defensive_check_mgr_) {
defensive_check_mgr_->destroy();
ob_free(defensive_check_mgr_);
defensive_check_mgr_ = NULL;
}
#endif
is_inited_ = false;
TRANS_LOG(INFO, "transaction service destroyed", KPC(this));
}

6
src/storage/tx/ob_trans_service.h
View File

@ -198,6 +198,9 @@ public:
const char *buf,
const int64_t buf_len);
ObTxELRUtil &get_tx_elr_util() { return elr_util_; }
#ifdef ENABLE_DEBUG_LOG
transaction::ObDefensiveCheckMgr *get_defensive_check_mgr() { return defensive_check_mgr_; }
#endif
private:
void check_env_();
bool can_create_ctx_(const int64_t trx_start_ts, const common::ObTsWindows &changing_leader_windows);
@ -266,6 +269,9 @@ private:
// account task qeuue's inqueue and dequeue
uint32_t input_queue_count_;
uint32_t output_queue_count_;
#ifdef ENABLE_DEBUG_LOG
transaction::ObDefensiveCheckMgr *defensive_check_mgr_;
#endif
// txDesc's manager
ObTxDescMgr tx_desc_mgr_;