hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

impl ob_admin log_tool for dup table log

Browse Source
This commit is contained in:
obdev 2023-09-08 11:09:50 +08:00 committed by ob-robot
parent 7feac6b5e7
commit 570c401f51
8 changed files with 382 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/storage/CMakeLists.txt
View File

@ -317,6 +317,7 @@ ob_set_subtarget(ob_storage tx
tx/ob_dup_table_tablets.cpp
tx/ob_dup_table_ts_sync.cpp
tx/ob_dup_table_stat.cpp
tx/ob_dup_table_dump.cpp
tx/ob_xa_ctx.cpp
tx/ob_xa_ctx_mgr.cpp
tx/ob_xa_dblink_service.cpp

10
src/storage/tx/ob_dup_table_base.cpp
View File

@ -513,7 +513,7 @@ int ObDupTableLSCheckpoint::online()
// OB_SERIALIZE_MEMBER(ObDupTableLogBlockHeader, position_, remain_length_);
OB_SERIALIZE_MEMBER(DupTableLogEntryHeader, entry_type_);
OB_SERIALIZE_MEMBER(DupTableStatLog, lease_addr_cnt_, readable_cnt_, all_tablet_set_cnt_);
OB_SERIALIZE_MEMBER(DupTableStatLog, lease_addr_cnt_, leader_readable_cnt_, all_tablet_set_cnt_);
// OB_SERIALIZE_MEMBER(ObLSDupTabletsMgr, max_submitted_tablet_change_ts_);
@ -754,8 +754,8 @@ int ObDupTableLogOperator::sync_log_succ_(const bool for_replay)
}
}
}
if (stat_log_.readable_cnt_ > 0 && logging_readable_cnt > 0
&& stat_log_.readable_cnt_ == logging_readable_cnt) {
if (stat_log_.leader_readable_cnt_ > 0 && logging_readable_cnt > 0
&& stat_log_.leader_readable_cnt_ == logging_readable_cnt) {
contain_all_readable = true;
}
if (!contain_all_readable) {
@ -804,7 +804,7 @@ int ObDupTableLogOperator::prepare_serialize_log_entry_(int64_t &max_ser_size,
DupTableStatLog max_stat_log;
max_stat_log.lease_addr_cnt_ = INT64_MAX;
max_stat_log.readable_cnt_ = INT64_MAX;
max_stat_log.leader_readable_cnt_ = INT64_MAX;
max_stat_log.all_tablet_set_cnt_ = INT64_MAX;
if (big_segment_buf_.is_active()) {
@ -913,7 +913,7 @@ int ObDupTableLogOperator::serialize_log_entry_(const int64_t max_ser_size,
case DupTableLogEntryType::DuptableStatLog: {
DupTableStatLog stat_log;
stat_log.lease_addr_cnt_ = logging_lease_addrs_.count();
stat_log.readable_cnt_ = tablet_mgr_ptr_->get_readable_tablet_set_count();
stat_log.leader_readable_cnt_ = tablet_mgr_ptr_->get_readable_tablet_set_count();
stat_log.all_tablet_set_cnt_ = tablet_mgr_ptr_->get_all_tablet_set_count();
if (OB_FAIL(stat_log.serialize(big_segment_buf_.get_serialize_buf(),
big_segment_buf_.get_serialize_buf_len(), data_pos))) {

7
src/storage/tx/ob_dup_table_base.h
View File

@ -901,11 +901,11 @@ struct DupTableLogEntryHeader
struct DupTableStatLog
{
int64_t lease_addr_cnt_;
int64_t readable_cnt_;
int64_t leader_readable_cnt_;
int64_t all_tablet_set_cnt_;
int64_t logging_readable_cnt_;
TO_STRING_KV(K(lease_addr_cnt_), K(readable_cnt_), K(all_tablet_set_cnt_),
TO_STRING_KV(K(lease_addr_cnt_), K(leader_readable_cnt_), K(all_tablet_set_cnt_),
K(logging_readable_cnt_));
DupTableStatLog() { reset(); }
@ -913,7 +913,7 @@ struct DupTableStatLog
void reset()
{
lease_addr_cnt_ = -1;
readable_cnt_ = -1;
leader_readable_cnt_ = -1;
all_tablet_set_cnt_ = -1;
logging_readable_cnt_ = -1;
}
@ -979,6 +979,7 @@ private:
private:
void after_submit_log(const bool submit_result, const bool for_replay);
#define LOG_OPERATOR_INIT_CHECK \
if (OB_SUCC(ret)) { \
if (OB_ISNULL(log_handler_) || OB_ISNULL(lease_mgr_ptr_) || OB_ISNULL(tablet_mgr_ptr_)) { \

277
src/storage/tx/ob_dup_table_dump.cpp Normal file
View File

@ -0,0 +1,277 @@
// Copyright (c) 2021 OceanBase
// OceanBase is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
// http://license.coscl.org.cn/MulanPubL-2.0
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PubL v2 for more details.
#include "storage/tx/ob_dup_table_dump.h"
namespace oceanbase
{
namespace transaction
{
void ObDupTableLogDumpIterator::reset()
{
str_arg_ = nullptr;
iter_buf_ = nullptr;
iter_buf_len_ = 0;
iter_buf_pos_ = 0;
big_segment_.reset();
dup_tablet_map_.reset();
dup_table_lease_map_.destroy();
stat_log_.reset();
}
bool ObDupTableLogDumpIterator::is_inited()
{
bool inited = false;
inited = OB_NOT_NULL(str_arg_) && OB_NOT_NULL(iter_buf_) && big_segment_.is_active()
&& dup_tablet_map_.created() && dup_table_lease_map_.created();
return inited;
}
int ObDupTableLogDumpIterator::init_with_log_buf(const char *buf,
const int64_t buf_len,
share::ObAdminMutatorStringArg *str_arg_ptr)
{
int ret = OB_SUCCESS;
logservice::ObLogBaseHeader base_header;
int64_t replay_pos = 0;
if (OB_ISNULL(buf) || buf_len < 0 || OB_ISNULL(str_arg_ptr)) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_len), KP(str_arg_ptr));
} else if (OB_FAIL(big_segment_.collect_one_part(buf, buf_len, replay_pos))) {
if (OB_ITER_END == ret) {
// need clear big_segment after collected all part for replay
ret = OB_ERR_UNEXPECTED;
DUP_TABLE_LOG(WARN, "Log entry is completed, can not merge new block", K(ret),
K(big_segment_));
} else if (OB_START_LOG_CURSOR_INVALID == ret) {
DUP_TABLE_LOG(INFO, "start replay from the middle of a big log entry", K(ret),
K(big_segment_));
} else if (big_segment_.is_completed()) {
ret = OB_ITER_END;
}
} else if (!dup_tablet_map_.created()
&& OB_FAIL(dup_tablet_map_.create(OB_SYS_TENANT_ID, 1024))) {
DUP_TABLE_LOG(WARN, "create dup tablet map failed", K(ret), KPC(this),
K(dup_tablet_map_.created()));
} else if (!dup_table_lease_map_.created()
&& OB_FAIL(dup_table_lease_map_.create(64, "DupTableDump"))) {
DUP_TABLE_LOG(WARN, "create dup table lease map failed", K(ret), K(this),
K(dup_table_lease_map_.created()));
} else {
str_arg_ = str_arg_ptr;
dup_tablet_map_.clear();
dup_table_lease_map_.clear();
}
return ret;
}
int ObDupTableLogDumpIterator::dump_dup_table_log()
{
int ret = OB_SUCCESS;
if (OB_SUCC(ret)) {
if (!big_segment_.is_completed()) {
ret = OB_STATE_NOT_MATCH;
DUP_TABLE_LOG(WARN, "need collect more parts of log entry", K(ret), K(big_segment_));
} else {
int64_t deser_pos = big_segment_.get_deserialize_buf_pos();
const int64_t deser_buf_len = big_segment_.get_deserialize_buf_len();
if (OB_NOT_NULL(str_arg_)) {
share::ObAdminLogNormalDumper normal_writer;
str_arg_->writer_ptr_ = &normal_writer;
str_arg_->writer_ptr_->dump_key("###<DUP_TABLE_LOG>");
str_arg_->writer_ptr_->start_object();
}
while (OB_SUCC(ret) && deser_pos < deser_buf_len) {
DupTableLogEntryHeader entry_header;
int64_t log_entry_size = 0;
int64_t after_header_pos = 0;
stat_log_.reset();
if (OB_FAIL(entry_header.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len,
deser_pos))) {
DUP_TABLE_LOG(WARN, "deserialize entry header failed", K(ret), K(deser_pos),
K(big_segment_), K(entry_header));
} else if (OB_FAIL(serialization::decode_i64(big_segment_.get_deserialize_buf(),
deser_buf_len, deser_pos, &log_entry_size))) {
DUP_TABLE_LOG(WARN, "decode log entry size failed", K(ret), K(log_entry_size),
K(big_segment_));
} else if (OB_FALSE_IT(after_header_pos = deser_pos)) {
// do nothing
} else {
switch (entry_header.entry_type_) {
case DupTableLogEntryType::TabletChangeLog: {
if (OB_FAIL(iter_tablet_log_(deser_pos + log_entry_size, deser_pos))) {
DUP_TABLE_LOG(WARN, "iter dup table tablet log failed", K(ret), K(deser_pos),
K(log_entry_size));
}
break;
}
case DupTableLogEntryType::LeaseListLog: {
if (OB_FAIL(iter_lease_log_(deser_pos + log_entry_size, deser_pos))) {
DUP_TABLE_LOG(WARN, "iter dup table lease log failed", K(ret), K(deser_pos),
K(log_entry_size));
}
break;
}
case DupTableLogEntryType::DuptableStatLog: {
if (OB_FAIL(iter_stat_log_(deser_pos + log_entry_size, deser_pos))) {
DUP_TABLE_LOG(WARN, "iter dup table stat log failed", K(ret), K(deser_pos),
K(log_entry_size));
}
break;
}
default: {
ret = OB_ERR_UNEXPECTED;
DUP_TABLE_LOG(WARN, "unexpected log entry type", K(ret), K(entry_header), K(deser_pos));
break;
}
}
}
}
if (OB_SUCC(ret)) {
iter_buf_pos_ = deser_pos;
if (OB_NOT_NULL(str_arg_)) {
str_arg_->writer_ptr_->end_object();
}
}
}
}
return ret;
}
int ObDupTableLogDumpIterator::iter_stat_log_(const int64_t deser_buf_len, int64_t &deser_pos)
{
int ret = OB_SUCCESS;
int64_t tmp_pos = deser_pos;
if (OB_FAIL(stat_log_.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len, tmp_pos))) {
DUP_TABLE_LOG(WARN, "deserialize stat log failed", K(ret), K(deser_pos), K(deser_buf_len),
K(stat_log_));
} else if (OB_ISNULL(str_arg_)) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid ObAdminMutatorStringArg", K(ret), KPC(this));
} else {
str_arg_->writer_ptr_->dump_key("##<STAT_LOG>");
str_arg_->writer_ptr_->dump_string(to_cstring(stat_log_));
}
if (OB_SUCC(ret)) {
deser_pos = tmp_pos;
}
return ret;
}
int ObDupTableLogDumpIterator::iter_tablet_log_(const int64_t deser_buf_len, int64_t &deser_pos)
{
int ret = OB_SUCCESS;
int64_t tmp_pos = deser_pos;
if (OB_ISNULL(str_arg_)) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid ObAdminMutatorStringArg", K(ret), KPC(this));
} else {
str_arg_->writer_ptr_->dump_key("##<TABLET_LOG>");
str_arg_->writer_ptr_->start_object();
while (OB_SUCC(ret) && tmp_pos < deser_buf_len) {
dup_tablet_map_.reuse();
DupTabletLogBody dup_log_body(dup_tablet_map_);
if (OB_FAIL(dup_log_body.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len,
tmp_pos))) {
DUP_TABLE_LOG(WARN, "deserialize dup log body failed", K(ret), K(dup_log_body));
} else {
str_arg_->writer_ptr_->dump_key("DupTabletSet");
str_arg_->writer_ptr_->start_object();
str_arg_->writer_ptr_->dump_key("#(TabletSetAttribute)");
str_arg_->writer_ptr_->dump_string(to_cstring(dup_tablet_map_.get_RO_attribute()));
str_arg_->writer_ptr_->dump_key("#(RelatedTabletSetAttribute)");
str_arg_->writer_ptr_->dump_string(to_cstring(dup_tablet_map_.get_RO_related_attribute()));
str_arg_->writer_ptr_->dump_key("#(TabletCount)");
str_arg_->writer_ptr_->dump_string(to_cstring(dup_tablet_map_.size()));
str_arg_->writer_ptr_->dump_key("#(TabletIdList)");
str_arg_->writer_ptr_->start_object();
for (DupTabletChangeMap::const_iterator iter = dup_tablet_map_.begin();
iter != dup_tablet_map_.end(); iter++) {
str_arg_->writer_ptr_->dump_string(to_cstring(iter->first));
}
str_arg_->writer_ptr_->end_object();
str_arg_->writer_ptr_->end_object();
}
}
str_arg_->writer_ptr_->end_object();
}
if (OB_SUCC(ret)) {
deser_pos = tmp_pos;
}
return ret;
}
int ObDupTableLogDumpIterator::iter_lease_log_(const int64_t deser_buf_len, int64_t &deser_pos)
{
int ret = OB_SUCCESS;
int64_t tmp_pos = deser_pos;
if (OB_ISNULL(str_arg_)) {
ret = OB_INVALID_ARGUMENT;
DUP_TABLE_LOG(WARN, "invalid ObAdminMutatorStringArg", K(ret), KPC(this));
} else {
str_arg_->writer_ptr_->dump_key("##<LEASE_LOG>");
str_arg_->writer_ptr_->start_object();
while (OB_SUCC(ret) && tmp_pos < deser_buf_len) {
DupTableLeaseLogHeader lease_log_header;
DupTableLeaderLeaseInfo leader_lease_info;
DupTableDurableLeaseLogBody durable_lease_log_body(leader_lease_info.confirmed_lease_info_);
if (OB_FAIL(lease_log_header.deserialize(big_segment_.get_deserialize_buf(), deser_buf_len,
tmp_pos))) {
DUP_TABLE_LOG(WARN, "deserialize lease log header failed", K(ret), K(lease_log_header),
K(tmp_pos), K(deser_buf_len));
} else if (lease_log_header.is_durable_lease_log()) {
if (OB_FAIL(durable_lease_log_body.deserialize(big_segment_.get_deserialize_buf(),
deser_buf_len, tmp_pos))) {
DUP_TABLE_LOG(WARN, "deserialize leader lease info failed", K(ret));
} else {
str_arg_->writer_ptr_->start_object();
str_arg_->writer_ptr_->dump_key("#(LeaseKey)");
str_arg_->writer_ptr_->dump_string(to_cstring(lease_log_header));
str_arg_->writer_ptr_->dump_key("#(ConfirmedLeaseInfo)");
str_arg_->writer_ptr_->dump_string(to_cstring(leader_lease_info.confirmed_lease_info_));
str_arg_->writer_ptr_->end_object();
}
}
}
str_arg_->writer_ptr_->end_object();
}
if (OB_SUCC(ret)) {
deser_pos = tmp_pos;
}
return ret;
}
} // namespace transaction
} // namespace oceanbase

69
src/storage/tx/ob_dup_table_dump.h Normal file
View File

@ -0,0 +1,69 @@
// Copyright (c) 2021 OceanBase
// OceanBase is licensed under Mulan PubL v2.
// You can use this software according to the terms and conditions of the Mulan PubL v2.
// You may obtain a copy of Mulan PubL v2 at:
// http://license.coscl.org.cn/MulanPubL-2.0
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
// See the Mulan PubL v2 for more details.
#ifndef OCEANBASE_TRANSACTION_DUP_TABLE_DUMP_H
#define OCEANBASE_TRANSACTION_DUP_TABLE_DUMP_H
#include "src/share/ob_admin_dump_helper.h"
#include "storage/tx/ob_dup_table_lease.h"
#include "storage/tx/ob_dup_table_tablets.h"
namespace oceanbase
{
namespace transaction
{
class ObDupTableLogDumpIterator
{
public:
ObDupTableLogDumpIterator()
: str_arg_(nullptr), iter_buf_(nullptr), iter_buf_len_(0), dup_tablet_map_(1),
dup_table_lease_map_()
{}
void reset();
bool is_inited();
int init_with_log_buf(const char *buf,
const int64_t buf_len,
share::ObAdminMutatorStringArg *str_arg_ptr);
int dump_dup_table_log();
int64_t get_iter_buf_pos() { return iter_buf_pos_; }
TO_STRING_KV(KP(str_arg_),
KP(iter_buf_),
K(iter_buf_len_),
K(dup_tablet_map_.size()),
K(dup_table_lease_map_.size()));
private:
int iter_stat_log_(const int64_t deser_buf_len, int64_t &deser_pos);
int iter_tablet_log_(const int64_t deser_buf_len, int64_t &deser_pos);
int iter_lease_log_(const int64_t deser_buf_len, int64_t &deser_pos);
private:
share::ObAdminMutatorStringArg *str_arg_;
const char *iter_buf_;
int64_t iter_buf_len_;
int64_t iter_buf_pos_;
ObTxBigSegmentBuf big_segment_;
DupTabletChangeMap dup_tablet_map_;
DupTableLeaderLeaseMap dup_table_lease_map_;
DupTableStatLog stat_log_;
};
} // namespace transaction
} // namespace oceanbase
#endif

3
src/storage/tx/ob_dup_table_tablets.h
View File

@ -441,6 +441,9 @@ public:
}
int64_t get_last_gc_scan_ts() { return last_gc_scan_ts_; }
const DupTabletSetAttribute & get_RO_attribute() const {return dup_set_attr_;}
const RelatedSetAttribute & get_RO_related_attribute() const {return related_set_attr_;}
bool is_logging() const { return dup_set_attr_.change_status_.check_logging(); }
void set_logging() { dup_set_attr_.change_status_.set_logging(); }
void clean_logging() { dup_set_attr_.change_status_.clean_logging(); }

22
tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.cpp
View File

@ -22,6 +22,7 @@
#include "storage/tx/ob_id_service.h"
#include "storage/memtable/ob_memtable_mutator.h"
#include "storage/tx/ob_keep_alive_ls_handler.h"
#include "storage/tx/ob_dup_table_dump.h"
#include "logservice/ob_log_base_header.h"
#include "logservice/ob_garbage_collector.h"
#include "logservice/data_dictionary/ob_data_dict_iterator.h" // ObDataDictIterator
@ -578,6 +579,23 @@ int ObAdminParserLogEntry::parse_medium_log_()
return ret;
}
int ObAdminParserLogEntry::parse_dup_table_log_()
{
int ret = OB_SUCCESS;
oceanbase::transaction::ObDupTableLogDumpIterator dup_table_log_dump_iter;
if (OB_FAIL(dup_table_log_dump_iter.init_with_log_buf(buf_ + pos_, buf_len_, &str_arg_))) {
LOG_WARN("fail to init dup table log dump_iter", K(ret));
} else if (OB_FAIL(dup_table_log_dump_iter.dump_dup_table_log())) {
LOG_WARN("fail to dump dup table log", K(ret));
} else {
pos_ += dup_table_log_dump_iter.get_iter_buf_pos();
}
return ret;
}
int ObAdminParserLogEntry::parse_different_entry_type_(const logservice::ObLogBaseHeader &header)
{
int ret = OB_SUCCESS;
@ -656,6 +674,10 @@ int ObAdminParserLogEntry::parse_different_entry_type_(const logservice::ObLogBa
ret = parse_medium_log_();
break;
}
case oceanbase::logservice::ObLogBaseType::DUP_TABLE_LOG_BASE_TYPE: {
ret = parse_dup_table_log_();
break;
}
default: {
fprintf(stdout, " Unknown Base Log Type : %d\n", header.get_log_type());

1
tools/ob_admin/log_tool/parser/ob_admin_parser_log_entry.h
View File

@ -64,6 +64,7 @@ private:
int parse_data_dict_log_();
int parse_reserved_snapshot_log_();
int parse_medium_log_();
int parse_dup_table_log_();
//log type belong to trans_service
int parse_trans_redo_log_(transaction::ObTxLogBlock &tx_log_block,