hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[opensource] add clog tool

Browse Source
This commit is contained in:
gjw2284740
2021-08-23 21:24:34 +08:00
committed by wangzelin.wzl
parent f4def44a03
commit 5ae88eacbd
19 changed files with 4317 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
CMakeLists.txt
View File

@ -29,7 +29,6 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
add_subdirectory(deps/easy) add_subdirectory(deps/easy)
add_subdirectory(deps/oblib) add_subdirectory(deps/oblib)
add_subdirectory(src) add_subdirectory(src)
add_subdirectory(tools)
include(CMakeDependentOption) include(CMakeDependentOption)
# OB_BUILD_RPM => include tools and build them. # OB_BUILD_RPM => include tools and build them.
@ -59,4 +58,10 @@ elseif(OB_INCLUDE_UNITTEST)
add_subdirectory(unittest EXCLUDE_FROM_ALL) add_subdirectory(unittest EXCLUDE_FROM_ALL)
endif() endif()
if (OB_BUILD_TOOLS)
add_subdirectory(tools)
elseif (OB_INCLUDE_TOOLS)
add_subdirectory(tools EXCLUDE_FROM_ALL)
endif()
include(cmake/RPM.cmake) include(cmake/RPM.cmake)

72
src/storage/transaction/ob_trans_log.cpp
View File

@ -382,6 +382,30 @@ bool ObTransRedoLog::is_xa_trans() const
return xid_.is_valid() && !xid_.empty(); return xid_.is_valid() && !xid_.empty();
} }
int ObTransRedoLog::init_for_deserialize(const bool use_mutator_buf)
{
int ret = OB_SUCCESS;
if (OB_FAIL(mutator_.init(use_mutator_buf))) {
TRANS_LOG(WARN, "redo log mutator init failed", K(ret));
}
return ret;
}
int ObTransRedoLog::replace_encrypt_info_tenant_id(const uint64_t real_tenant_id)
{
int ret = OB_SUCCESS;
UNUSED(real_tenant_id);
// no implementation
return ret;
}
int ObTransRedoLog::decrypt_table_key()
{
int ret = OB_SUCCESS;
// no implementation
return ret;
}
// prepare log generated by observers before version 2.0 is replayed on observers of version 2.0 // prepare log generated by observers before version 2.0 is replayed on observers of version 2.0
// checkpoint has not been deserialized, no need to check at this time // checkpoint has not been deserialized, no need to check at this time
bool ObTransPrepareLog::is_valid() const bool ObTransPrepareLog::is_valid() const
@ -707,6 +731,30 @@ int ObSpTransRedoLog::replace_tenant_id(const uint64_t new_tenant_id)
return ret; return ret;
} }
int ObSpTransRedoLog::init_for_deserialize(const bool use_mutator_buf)
{
int ret = OB_SUCCESS;
if (OB_FAIL(mutator_.init(use_mutator_buf))) {
TRANS_LOG(WARN, "sp trans redo log mutator init failed", K(ret));
}
return ret;
}
int ObSpTransRedoLog::replace_encrypt_info_tenant_id(const uint64_t real_tenant_id)
{
int ret = OB_SUCCESS;
UNUSED(real_tenant_id);
// no implementation
return ret;
}
int ObSpTransRedoLog::decrypt_table_key()
{
int ret = OB_SUCCESS;
// no implementation
return ret;
}
int ObSpTransCommitLog::init(const int64_t log_type, const ObPartitionKey& partition, const uint64_t tenant_id, int ObSpTransCommitLog::init(const int64_t log_type, const ObPartitionKey& partition, const uint64_t tenant_id,
const ObTransID& trans_id, const uint64_t checksum, const uint64_t cluster_id, const ObRedoLogIdArray& redo_log_ids, const ObTransID& trans_id, const uint64_t checksum, const uint64_t cluster_id, const ObRedoLogIdArray& redo_log_ids,
const ObStartTransParam& trans_param, const int64_t log_no, const ObString& app_trace_id_str, const ObStartTransParam& trans_param, const int64_t log_no, const ObString& app_trace_id_str,
@ -933,6 +981,30 @@ int ObTransMutatorLog::replace_tenant_id(const uint64_t new_tenant_id)
return ret; return ret;
} }
int ObTransMutatorLog::init_for_deserialize(const bool use_mutator_buf)
{
int ret = OB_SUCCESS;
if (OB_FAIL(mutator_.init(use_mutator_buf))) {
TRANS_LOG(WARN, "mutator log mutator init failed", K(ret));
}
return ret;
}
int ObTransMutatorLog::replace_encrypt_info_tenant_id(const uint64_t real_tenant_id)
{
int ret = OB_SUCCESS;
UNUSED(real_tenant_id);
// no implementation
return ret;
}
int ObTransMutatorLog::decrypt_table_key()
{
int ret = OB_SUCCESS;
// no implementation
return ret;
}
int ObTransMutatorAbortLog::init( int ObTransMutatorAbortLog::init(
const int64_t log_type, const ObPartitionKey& pkey, const ObTransID& trans_id, const uint64_t cluster_id) const int64_t log_type, const ObPartitionKey& pkey, const ObTransID& trans_id, const uint64_t cluster_id)
{ {

97
src/storage/transaction/ob_trans_log.h
View File

@ -319,6 +319,9 @@ public:
return xid_; return xid_;
} }
bool is_xa_trans() const; bool is_xa_trans() const;
int init_for_deserialize(const bool use_mutator_buf = true);
int replace_encrypt_info_tenant_id(const uint64_t real_tenant_id);
int decrypt_table_key();
VIRTUAL_TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(tenant_id), K_(log_no), K_(scheduler), VIRTUAL_TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(tenant_id), K_(log_no), K_(scheduler),
K_(coordinator), K_(participants), K_(trans_param), K_(cluster_id), K_(active_memstore_version), K_(coordinator), K_(participants), K_(trans_param), K_(cluster_id), K_(active_memstore_version),
@ -664,6 +667,30 @@ public:
bool is_valid() const override; bool is_valid() const override;
}; };
class ObSpTransRedoLogHelper {
public:
ObSpTransRedoLogHelper()
: tenant_id_(common::OB_INVALID_TENANT_ID),
log_no_(-1),
trans_param_(),
mutator_(),
active_memstore_version_(),
prev_trans_arr_(),
can_elr_(false)
{}
~ObSpTransRedoLogHelper()
{}
public:
uint64_t tenant_id_;
int64_t log_no_;
ObStartTransParam trans_param_;
ObTransMutator mutator_;
common::ObVersion active_memstore_version_;
ObElrTransInfoArray prev_trans_arr_;
bool can_elr_;
};
class ObSpTransRedoLog : public ObTransLog { class ObSpTransRedoLog : public ObTransLog {
OB_UNIS_VERSION(1); OB_UNIS_VERSION(1);
@ -678,6 +705,16 @@ public:
prev_trans_arr_(), prev_trans_arr_(),
can_elr_(false) can_elr_(false)
{} {}
ObSpTransRedoLog(ObSpTransRedoLogHelper& helper)
: ObTransLog(),
tenant_id_(helper.tenant_id_),
log_no_(helper.log_no_),
trans_param_(helper.trans_param_),
mutator_(),
active_memstore_version_(helper.active_memstore_version_),
prev_trans_arr_(helper.prev_trans_arr_),
can_elr_(helper.can_elr_)
{}
~ObSpTransRedoLog() ~ObSpTransRedoLog()
{} {}
int init(const int64_t log_type, const common::ObPartitionKey& partition, const ObTransID& trans_id, int init(const int64_t log_type, const common::ObPartitionKey& partition, const ObTransID& trans_id,
@ -721,6 +758,9 @@ public:
} }
bool is_valid() const override; bool is_valid() const override;
virtual int replace_tenant_id(const uint64_t new_tenant_id) override; virtual int replace_tenant_id(const uint64_t new_tenant_id) override;
int init_for_deserialize(const bool use_mutator_buf = true);
int replace_encrypt_info_tenant_id(const uint64_t real_tenant_id);
int decrypt_table_key();
VIRTUAL_TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(tenant_id), K_(log_no), K_(trans_param), VIRTUAL_TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(tenant_id), K_(log_no), K_(trans_param),
K_(cluster_id), K_(active_memstore_version), K_(prev_trans_arr), K_(can_elr)); K_(cluster_id), K_(active_memstore_version), K_(prev_trans_arr), K_(can_elr));
@ -738,6 +778,14 @@ protected:
bool can_elr_; bool can_elr_;
}; };
class ObSpTransCommitLogHelper : public ObSpTransRedoLogHelper {
public:
ObSpTransCommitLogHelper() : ObSpTransRedoLogHelper()
{}
~ObSpTransCommitLogHelper()
{}
};
class ObSpTransCommitLog : public ObSpTransRedoLog { class ObSpTransCommitLog : public ObSpTransRedoLog {
OB_UNIS_VERSION(1); OB_UNIS_VERSION(1);
@ -751,6 +799,15 @@ public:
checkpoint_(0), checkpoint_(0),
app_trace_info_() app_trace_info_()
{} {}
ObSpTransCommitLog(ObSpTransCommitLogHelper& helper)
: ObSpTransRedoLog(helper),
global_trans_version_(-1),
checksum_(0),
prev_redo_log_ids_(common::ObModIds::OB_TRANS_REDO_LOG_ID_ARRAY, common::OB_MALLOC_NORMAL_BLOCK_SIZE),
app_trace_id_str_(),
checkpoint_(0),
app_trace_info_()
{}
~ObSpTransCommitLog() ~ObSpTransCommitLog()
{} {}
int init(const int64_t log_type, const common::ObPartitionKey& partition, const uint64_t tenant_id, int init(const int64_t log_type, const common::ObPartitionKey& partition, const uint64_t tenant_id,
@ -1009,6 +1066,32 @@ private:
ObXATransID xid_; ObXATransID xid_;
}; };
class ObTransMutatorLogHelper {
public:
ObTransMutatorLogHelper()
: tenant_id_(common::OB_INVALID_TENANT_ID),
trans_expired_time_(0),
trans_param_(),
log_no_(0),
mutator_(),
prev_trans_arr_(),
can_elr_(false),
cluster_version_(0)
{}
~ObTransMutatorLogHelper()
{}
public:
int64_t tenant_id_;
int64_t trans_expired_time_;
ObStartTransParam trans_param_;
int64_t log_no_;
ObTransMutator mutator_;
ObElrTransInfoArray prev_trans_arr_;
bool can_elr_;
uint64_t cluster_version_;
};
class ObTransMutatorLog : public ObTransLog { class ObTransMutatorLog : public ObTransLog {
OB_UNIS_VERSION(1); OB_UNIS_VERSION(1);
@ -1024,6 +1107,17 @@ public:
can_elr_(false), can_elr_(false),
cluster_version_(0) cluster_version_(0)
{} {}
ObTransMutatorLog(ObTransMutatorLogHelper& helper)
: ObTransLog(),
tenant_id_(helper.tenant_id_),
trans_expired_time_(helper.trans_expired_time_),
trans_param_(),
log_no_(helper.log_no_),
mutator_(),
prev_trans_arr_(),
can_elr_(helper.can_elr_),
cluster_version_(helper.cluster_version_)
{}
~ObTransMutatorLog() ~ObTransMutatorLog()
{ {
destroy(); destroy();
@ -1073,6 +1167,9 @@ public:
return cluster_version_; return cluster_version_;
} }
virtual int replace_tenant_id(const uint64_t new_tenant_id) override; virtual int replace_tenant_id(const uint64_t new_tenant_id) override;
int init_for_deserialize(const bool use_mutator_buf = true);
int replace_encrypt_info_tenant_id(const uint64_t real_tenant_id);
int decrypt_table_key();
public: public:
TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(cluster_id), K_(tenant_id), K_(trans_expired_time), TO_STRING_KV(K_(log_type), K_(partition), K_(trans_id), K_(cluster_id), K_(tenant_id), K_(trans_expired_time),

1
tools/CMakeLists.txt
View File

@ -1 +1,2 @@
add_subdirectory(ob_error) add_subdirectory(ob_error)
add_subdirectory(ob_admin)

32
tools/ob_admin/CMakeLists.txt Normal file
View File

@ -0,0 +1,32 @@
add_executable(ob_admin
clog_tool/cmd_args_parser.h
clog_tool/ob_admin_clog_v2_executor.cpp
clog_tool/ob_admin_clog_v2_executor.h
clog_tool/ob_func_utils.cpp
clog_tool/ob_func_utils.h
clog_tool/ob_ilog_entry_parser.cpp
clog_tool/ob_ilog_entry_parser.h
clog_tool/ob_log_entry_filter.cpp
clog_tool/ob_log_entry_filter.h
clog_tool/ob_log_entry_parser.cpp
clog_tool/ob_log_entry_parser.h
ob_admin_executor.h
ob_admin_executor.cpp
main.cpp)
if (OB_STATIC_LINK_LGPL_DEPS)
set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -l:libmariadbclient.a -l:libaio.a")
endif()
target_link_libraries(ob_admin
PRIVATE
oceanbase_static
-static-libgcc
-static-libstdc++
${LGPL_DEPS}
)
target_include_directories(ob_admin
PRIVATE
${CMAKE_CURRENT_SOURCE_DIR})

150
tools/ob_admin/clog_tool/cmd_args_parser.h Normal file
View File

@ -0,0 +1,150 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef __OB_TOOLS_CMD_ARGS_PARSER_H__
#define __OB_TOOLS_CMD_ARGS_PARSER_H__
class CmdArgsParser {
const static int64_t MAX_N_ARGS = 1 << 10;
struct arg_t {
arg_t() : name_(NULL), value_(null_), default_value_(NULL)
{}
~arg_t()
{}
const char* name_;
const char* value_;
const char* default_value_;
};
public:
CmdArgsParser() : parse_seq_(0), n_args_(0)
{
default_arg_.name_ = "*default*";
default_arg_.value_ = null_;
default_arg_.default_value_ = null_;
}
~CmdArgsParser()
{}
bool reset()
{
memset(args_, 0, sizeof(args_));
n_args_ = 0;
parse_seq_ |= 1;
return true;
}
bool check(int argc, char** argv, ...)
{
bool args_is_valid = true;
char* p = NULL;
parse_seq_ = (parse_seq_ & ~1) + 2;
for (int64_t i = 0; i < n_args_ / 2; i++) {
arg_t arg = args_[i];
args_[i] = args_[n_args_ - 1 - i];
args_[n_args_ - 1 - i] = arg;
}
for (int64_t i = 0; i < argc; i++) {
if (argv[i][0] == ':' || NULL == (p = strchr(argv[i], '=')))
continue;
*p++ = 0;
arg_t* arg = get_arg(argv[i]);
if (arg && &default_arg_ != arg)
arg->value_ = p;
*--p = '=';
}
for (int64_t i = 0; i < argc; i++) {
if (argv[i][0] != ':' && (p = strchr(argv[i], '=')))
continue;
p = argv[i][0] == ':' ? argv[i] + 1 : argv[i];
arg_t* arg = get_next_unset_arg();
if (arg && arg->name_)
arg->value_ = p;
}
for (int64_t i = 0; i < n_args_; i++) {
if (null_ == args_[i].value_ && args_[i].default_value_)
args_[i].value_ = args_[i].default_value_;
if (null_ == args_[i].value_)
args_is_valid = false;
}
if (0 == strcmp("true", getenv("dump_args") ?: "false")) {
dump(argc, argv);
}
return args_is_valid;
}
void dump(int argc, char** argv)
{
printf("cmd_args_parser.dump:\n");
for (int64_t i = 0; i < argc; i++) {
printf("argv[%ld]=%s\n", i, argv[i]);
}
for (int64_t i = 0; i < n_args_; i++) {
printf(
"args[%ld]={name=%s, value=%s, default=%s}\n", i, args_[i].name_, args_[i].value_, args_[i].default_value_);
}
}
arg_t* get_next_unset_arg()
{
for (int64_t i = 0; i < n_args_; i++) {
if (null_ == args_[i].value_)
return args_ + i;
}
return NULL;
}
arg_t* get_arg(const char* name, const char* default_value = NULL)
{
assert(n_args_ < MAX_N_ARGS && name);
if (parse_seq_ & 1) {
args_[n_args_].name_ = name;
args_[n_args_].default_value_ = default_value;
args_[n_args_].value_ = null_;
return args_ + (n_args_++);
}
for (int64_t i = 0; i < n_args_; i++) {
if (0 == strcmp(args_[i].name_, name))
return args_ + i;
}
return &default_arg_;
}
private:
static const char* null_;
int64_t parse_seq_;
int64_t n_args_;
arg_t default_arg_;
arg_t args_[MAX_N_ARGS];
};
inline bool argv1_match_func(const char* argv1, const char* func)
{
const char* last_part = strrchr(func, '.');
if (NULL != last_part) {
last_part++;
} else {
last_part = func;
}
return 0 == strcmp(last_part, argv1);
}
const char* CmdArgsParser::null_ __attribute__((weak)) = "*null*";
CmdArgsParser __cmd_args_parser __attribute__((weak));
#define _Arg(name, ...) __cmd_args_parser.get_arg(#name, ##__VA_ARGS__)
#define BoolArg(name, ...) (0 == atoll(__cmd_args_parser.get_arg(#name, ##__VA_ARGS__)->value_)) ? true : false
#define IntArg(name, ...) atoll(__cmd_args_parser.get_arg(#name, ##__VA_ARGS__)->value_)
#define StrArg(name, ...) __cmd_args_parser.get_arg(#name, ##__VA_ARGS__)->value_
#define CmdCall(argc, argv, func, ...) \
(argc >= 2 && argv1_match_func(argv[1], #func) && __cmd_args_parser.reset() && \
__cmd_args_parser.check(argc - 2, argv + 2, ##__VA_ARGS__)) \
? func(__VA_ARGS__)
#define CmdCallSimple(argc, argv, func) (argc >= 3 && argv1_match_func(argv[1], #func)) ? func(argc - 2, argv + 2)
#endif /* __OB_TOOLS_CMD_ARGS_PARSER_H__ */

587
tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.cpp Normal file
View File

@ -0,0 +1,587 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX CLOG
#include <unistd.h>
#include <getopt.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include "ob_log_entry_parser.h"
#include "ob_ilog_entry_parser.h"
#include "cmd_args_parser.h"
#include "ob_admin_clog_v2_executor.h"
#include "ob_func_utils.h"
#include "share/ob_srv_rpc_proxy.h"
#include "share/ob_version.h"
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::clog;
namespace oceanbase {
namespace tools {
#define getcfg(key) getenv(key)
ObAdminClogV2Executor::ObAdminClogV2Executor() : DB_port_(-1), is_ofs_open_(false)
{}
ObAdminClogV2Executor::~ObAdminClogV2Executor()
{}
int ObAdminClogV2Executor::execute(int argc, char* argv[])
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(argc < 4) || OB_ISNULL(argv)) {
print_usage();
ret = OB_INVALID_ARGUMENT;
} else if (OB_FAIL(parse_options(argc, argv))) {
LOG_WARN("failed to parse options", K(ret));
} else {
int new_argc = argc - optind;
char** new_argv = argv + optind;
if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_all) : OB_NEED_RETRY)) {
LOG_INFO("finish dump_all ", K(ret));
} else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_filter) : OB_NEED_RETRY)) {
LOG_INFO("finish dump_filter ", K(ret));
} else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_hex) : OB_NEED_RETRY)) {
LOG_INFO("finish dump_hex ", K(ret));
} else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_format) : OB_NEED_RETRY)) {
LOG_INFO("finish dump_format ", K(ret));
} else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, stat_clog) : OB_NEED_RETRY)) {
LOG_INFO("finish stat_clog ", K(ret));
} else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, grep) : OB_NEED_RETRY)) {
LOG_INFO("finish encode", K(ret));
} else if (OB_NEED_RETRY != (ret = CmdCallSimple(new_argc, new_argv, dump_ilog) : OB_NEED_RETRY)) {
LOG_INFO("finish encode", K(ret));
} else {
fprintf(stderr, "failed %d", ret);
print_usage();
}
}
return ret;
}
int ObAdminClogV2Executor::parse_options(int argc, char* argv[])
{
int ret = OB_SUCCESS;
int option_index = 0;
struct option long_options[] = {{"host", 1, NULL, 'h'}, {"port", 1, NULL, 'p'}, {NULL, 0, NULL, 0}};
int c;
while (-1 != (c = getopt_long(argc, argv, "h:p:", long_options, &option_index))) {
switch (c) {
case 'h':
DB_host_.assign_ptr(optarg, strlen(optarg));
break;
case 'p':
DB_port_ = static_cast<int32_t>(strtol(optarg, NULL, 10));
break;
case '?':
case ':':
ret = OB_ERR_UNEXPECTED;
break;
default:
break;
}
}
return ret;
}
void ObAdminClogV2Executor::print_usage()
{
fprintf(stdout,
"Usages:\n"
"$ob_admin clog_tool dump_ilog ilog_files ## ./ob_admin clog_tool dump_ilog 1 2 3\n"
"$ob_admin clog_tool dump_all log_files ## ./ob_admin clog_tool dump_all 1 2 3\n"
"$ob_admin clog_tool dump_filter filter_str log_files ## ./ob_admin clog_tool dump_filter "
"'table_id=123;partition_id=123;log_id=123' 1 2 3\n"
"$ob_admin clog_tool dump_hex log_files ## ./ob_admin clog_tool dump_hex 1 2 3\n"
"$ob_admin clog_tool dump_format log_files ## ./ob_admin clog_tool dump_format 1 2 3\n");
}
int ObAdminClogV2Executor::dump_all(int argc, char* argv[])
{
int ret = OB_SUCCESS;
const bool is_hex = false;
if (OB_FAIL(dump_inner(argc, argv, is_hex))) {
LOG_WARN("failed to dump all", K(ret));
}
return ret;
}
int ObAdminClogV2Executor::dump_filter(int argc, char* argv[])
{
int ret = OB_SUCCESS;
const bool is_hex = false;
if (OB_FAIL(filter_.parse(argv[0]))) {
LOG_WARN("parse filter failed", K(ret), K(argv[0]));
} else {
LOG_INFO("dump with filter", K_(filter), K(argv[0]));
if (OB_FAIL(dump_inner(argc - 1, argv + 1, is_hex))) {
LOG_WARN("failed to dump all", K(ret));
}
}
return ret;
}
int ObAdminClogV2Executor::dump_hex(int argc, char* argv[])
{
int ret = OB_SUCCESS;
const bool is_hex = true;
if (OB_FAIL(dump_inner(argc, argv, is_hex))) {
LOG_WARN("failed to dump hex", K(ret));
}
return ret;
}
int ObAdminClogV2Executor::dump_inner(int argc, char* argv[], bool is_hex)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(argc), K(ret));
} else {
LOG_INFO("begin to dump all ", K(is_hex), K(ret));
for (int64_t i = 0; i < argc; ++i) {
if (OB_FAIL(dump_single_clog(argv[i], is_hex)) && OB_ITER_END != ret) {
LOG_WARN("failed to dump log ", K(argv[i]), K(ret));
} else if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
} else { /*do nothing*/
}
}
}
return ret;
}
int ObAdminClogV2Executor::dump_format(int argc, char* argv[])
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(argc), K(ret));
} else {
LOG_INFO("begin to dump format", K(ret));
for (int64_t i = 0; i < argc; ++i) {
if (OB_FAIL(dump_format_single_file(argv[i])) && (OB_ITER_END != ret)) {
LOG_WARN("failed to dump format ", K(argv[i]), K(ret));
} else if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
} else { /*do nothing*/
}
}
}
return ret;
}
int ObAdminClogV2Executor::stat_clog(int argc, char* argv[])
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(argc), K(ret));
} else {
LOG_INFO("begin to stat_clog", K(ret));
for (int64_t i = 0; i < argc; ++i) {
if (OB_FAIL(stat_single_log(argv[i])) && (OB_ITER_END != ret)) {
LOG_WARN("failed to stat_single_log", K(argv[i]), K(ret));
} else if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
} else { /*do nothing*/
}
}
}
return ret;
}
int ObAdminClogV2Executor::dump_ilog(int argc, char* argv[])
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(argc < 1) || OB_ISNULL(argv)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(argc), K(ret));
} else {
LOG_INFO("begin to dump ilog ", K(ret));
for (int64_t i = 0; i < argc; ++i) {
if (OB_FAIL(dump_single_ilog(argv[i])) && OB_ITER_END != ret) {
LOG_WARN("failed to dump ilog ", K(argv[i]), K(ret));
} else if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
} else { /*do nothing*/
}
}
}
return ret;
}
int ObAdminClogV2Executor::grep(int argc, char* argv[])
{
int ret = OB_SUCCESS;
const int64_t BUF_LEN = 1024;
char result[BUF_LEN] = {0};
if (OB_UNLIKELY(1 > argc) || OB_ISNULL(argv)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(argc), K(ret));
} else {
LOG_INFO("begin to grep", K(ret));
char arg_str[BUF_LEN] = {0};
int64_t pos = 0;
memcpy(arg_str, argv[0], strlen(argv[0]));
char* input_str = arg_str;
char encode_type[64] = {0};
int64_t int_value = 0;
char origin_str[BUF_LEN] = {0};
char* token = NULL;
int32_t local_ret = 0;
while (OB_SUCC(ret) && NULL != (token = STRSEP(&input_str, "%"))) {
MEMSET(origin_str, '\0', BUF_LEN);
if (0 == STRLEN(token)) {
// do nothing
} else if (0 == (local_ret = sscanf(token, "%[0-9a-z]:%ld%s", encode_type, &int_value, origin_str)) ||
1 == local_ret) {
if (0 > (local_ret = snprintf(result + pos, BUF_LEN - pos, "%s", token))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("buf is not enough", "token_len", STRLEN(token), K(local_ret), K(BUF_LEN), K(ret));
} else {
pos += local_ret;
LOG_DEBUG("match normal str", K(token), K(pos));
}
} else if (2 == local_ret || 3 == local_ret) {
if (0 != STRCMP("i4", encode_type) && 0 != STRCMP("i8", encode_type) && 0 != STRCMP("v4", encode_type) &&
0 != STRCMP("v8", encode_type)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid encode type", K(token), K(encode_type), K(ret));
print_usage();
} else if (OB_FAIL(encode_int(result, pos, BUF_LEN, encode_type, int_value))) {
LOG_WARN("failed to encode_int", K(result), K(pos), K(BUF_LEN), K(encode_type), K(int_value), K(ret));
} else {
if (0 > (local_ret = snprintf(result + pos, BUF_LEN - pos, "%s", origin_str))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("buf is not enough", "token_len", STRLEN(token), K(BUF_LEN), K(ret));
} else {
pos += local_ret;
LOG_DEBUG("match normal str", K(token), K(pos));
}
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid token", K(token), K(ret));
}
}
}
if (OB_SUCC(ret)) {
if (1 == argc) {
int fd = 0;
struct stat sb;
if (-1 == fstat(fd, &sb)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to fstat", K(errno), K(ret));
} else {
if (S_ISFIFO(sb.st_mode) || S_ISREG(sb.st_mode)) {
execl("/usr/bin/xargs", "xargs", "-n", "1", "-P", "30", "grep", "-UP", result, NULL);
if (-1 == errno) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to execve", K(errno), K(ret));
}
} else {
fprintf(stdout, "ls store/clog/{1..3} |xargs -n 1 -P 30 grep -UP '%s' \n", result);
}
}
} else if (argc > 1) {
const int64_t COMMAND_BUF_SIZE = 1024;
char command[COMMAND_BUF_SIZE] = {0};
snprintf(command, COMMAND_BUF_SIZE, "xargs -n 1 -P 30 grep -UP '%s' \n", result);
FILE* file = popen(command, "w");
if (NULL == file) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to popen", K(errno), K(ret));
} else {
for (int64_t i = 1; OB_SUCC(ret) && i < argc; ++i) {
size_t w_len = 0;
size_t arg_len = STRLEN(argv[i]);
const size_t element_size = 1;
if (arg_len != (w_len = fwrite(argv[i], element_size, arg_len, file))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to fwrite", K(argv[i]), K(i), K(ret));
} else {
fwrite((char*)"\n", 1, 1, file);
}
}
if (OB_SUCC(ret)) {
if (-1 == pclose(file)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to pclose", K(errno), K(ret));
}
}
}
}
}
return ret;
}
int ObAdminClogV2Executor::encode_int(
char* buf, int64_t& pos, int64_t buf_len, const char* encode_type, int64_t int_value)
{
int ret = OB_SUCCESS;
const int64_t BUF_LEN = 128;
char buf_local[BUF_LEN] = {0};
int64_t local_pos = 0;
if (0 == STRCMP("i8", encode_type)) {
if (OB_FAIL(serialization::encode_i64(buf_local, BUF_LEN, local_pos, int_value))) {
LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret));
} else {
CLOG_LOG(DEBUG, "succ to encode i64", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret));
}
} else if (0 == STRCMP("i4", encode_type)) {
if (OB_FAIL(serialization::encode_i32(buf_local, BUF_LEN, local_pos, static_cast<int32_t>(int_value)))) {
LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret));
} else {
CLOG_LOG(DEBUG, "succ to encode i32", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret));
}
} else if (0 == STRCMP("v8", encode_type)) {
if (OB_FAIL(serialization::encode_vi64(buf_local, BUF_LEN, local_pos, int_value))) {
LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret));
} else {
CLOG_LOG(DEBUG, "succ to encode vi64", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret));
}
} else if (0 == STRCMP("v4", encode_type)) {
if (OB_FAIL(serialization::encode_vi32(buf_local, BUF_LEN, local_pos, static_cast<int32_t>(int_value)))) {
LOG_WARN("failed to encode i64", K(BUF_LEN), K(local_pos), K(int_value), K(ret));
} else {
CLOG_LOG(DEBUG, "succ to encode vi32", K(buf_local), K(buf_len), K(local_pos), K(int_value), K(ret));
}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid encode_type", K(encode_type), K(ret));
}
if (OB_SUCC(ret)) {
int ret_len = 0;
for (int64_t i = 0; OB_SUCC(ret) && i < local_pos; ++i) {
if (4 != (ret_len = snprintf(buf + pos, buf_len - pos, "\\x%02x", (unsigned int)(unsigned char)(buf_local[i])))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to snprinf", K(buf), K(buf_local[i]), K(ret_len), K(ret));
} else {
pos += 4;
}
}
}
return ret;
}
int ObAdminClogV2Executor::dump_single_clog(const char* path, bool is_hex)
{
int ret = OB_SUCCESS;
int fd = -1;
char* buf = NULL;
int64_t buf_len = -1;
uint64_t file_id = -1;
if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) {
LOG_WARN("failed to mmap_log_file", K(path), K(ret));
} else if (OB_FAIL(file_name_parser(path, file_id))) {
LOG_WARN("failed to parse file name", K(path), K(ret));
} else {
const bool is_ofs = is_ofs_file(path);
ObLogEntryParser entry_parser;
if (OB_FAIL(entry_parser.init(file_id, buf, buf_len, filter_, DB_host_, DB_port_, config_file_, is_ofs))) {
LOG_WARN("failed to init entry parser", K(path), K(ret));
} else if (OB_FAIL(entry_parser.dump_all_entry(is_hex))) {
if (OB_ITER_END == ret) {
LOG_INFO("succ to dump_all_entry", K(path));
} else {
LOG_WARN("failed to dump_all_entry", K(path), K(ret));
}
} else { /*do nothing*/
}
}
close_fd(path, fd, buf, buf_len);
return ret;
}
int ObAdminClogV2Executor::dump_single_ilog(const char* path)
{
int ret = OB_SUCCESS;
int fd = -1;
char* buf = NULL;
int64_t buf_len = -1;
uint64_t file_id = -1;
if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) {
LOG_WARN("failed to mmap_log_file", K(path), K(ret));
} else if (OB_FAIL(file_name_parser(path, file_id))) {
LOG_WARN("failed to parse file name", K(path), K(ret));
} else {
ObILogEntryParser entry_parser;
if (OB_FAIL(entry_parser.init(file_id, buf, buf_len))) {
LOG_WARN("failed to init entry parser", K(path), K(ret));
} else if (OB_FAIL(entry_parser.parse_all_entry())) {
if (OB_ITER_END == ret) {
LOG_INFO("succ to dump_all_entry", K(path));
} else {
LOG_WARN("failed to dump_all_entry", K(path), K(ret));
}
} else { /*do nothing*/
}
}
close_fd(path, fd, buf, buf_len);
return ret;
}
int ObAdminClogV2Executor::dump_format_single_file(const char* path)
{
int ret = OB_SUCCESS;
int fd = -1;
char* buf = NULL;
int64_t buf_len = -1;
uint64_t file_id = -1;
if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) {
LOG_WARN("failed to mmap_log_file", K(path), K(ret));
} else if (OB_FAIL(file_name_parser(path, file_id))) {
LOG_WARN("failed to parser file name ", K(path), K(ret));
} else {
const bool is_ofs = is_ofs_file(path);
ObLogEntryParser entry_parser;
if (OB_FAIL(entry_parser.init(file_id, buf, buf_len, filter_, DB_host_, DB_port_, config_file_, is_ofs))) {
LOG_WARN("failed to init entry parser", K(path), K(ret));
} else if (OB_FAIL(entry_parser.format_dump_entry())) {
if (OB_ITER_END == ret) {
LOG_INFO("succ to format_dump_all_entry", K(path));
} else {
LOG_WARN("failed to format_dump_entry", K(path), K(ret));
}
} else { /*do nothing*/
}
}
close_fd(path, fd, buf, buf_len);
return ret;
}
int ObAdminClogV2Executor::stat_single_log(const char* path)
{
int ret = OB_SUCCESS;
int fd = -1;
char* buf = NULL;
int64_t buf_len = -1;
uint64_t file_id = -1;
if (OB_FAIL(mmap_log_file(path, buf, buf_len, fd))) {
LOG_WARN("failed to mmap_log_file", K(path), K(ret));
} else if (OB_FAIL(file_name_parser(path, file_id))) {
LOG_WARN("failed to parser file name", K(path), K(ret));
} else {
const bool is_ofs = is_ofs_file(path);
ObLogEntryParser entry_parser;
if (OB_FAIL(entry_parser.init(file_id, buf, buf_len, filter_, DB_host_, DB_port_, config_file_, is_ofs))) {
LOG_WARN("failed to init entry parser", K(ret));
} else if (OB_FAIL(entry_parser.stat_log()) && (OB_ITER_END != ret)) {
LOG_WARN("failed to stat log", K(path), K(ret));
} else {
const ObLogStat& log_stat = entry_parser.get_log_stat();
fprintf(stdout, "log_file:%s\t stat_info:%s\n ", path, to_cstring(log_stat));
fprintf(stdout,
"log_file:%s\t stat_info:\n data_block_header_size = %lf M;\n log_header_size = %lf M;\n "
" log_size = %lf M;\n "
" trans_log_size = %lf M;\n mutator_size = %lf M;\n padding_size = %lf M;\n "
" new_row_size = %lf M;\n old_row_size = %lf M;\n total_row_size = %lf M;\n"
" new_primary_row_size = %lf M;\n primary_row_count = %ld;\n total_row_count = "
"%ld;\n total_log_count = %ld;\n dist_trans_count = %ld;\n sp_trans_count = %ld;\n"
" non_compressed_log_cnt = %ld;\n compressed_log_cnt = %ld;\n compressed_log_size "
"= %lf M;\n original_log_size = %lf M;\n compress_ratio = %lf;\n "
"compressed_tenant_ids:[%s];\n",
path,
(double)log_stat.data_block_header_size_ / 1024 / 1024,
(double)log_stat.log_header_size_ / 1024 / 1024,
(double)log_stat.log_size_ / 1024 / 1024,
(double)log_stat.trans_log_size_ / 1024 / 1024,
(double)log_stat.mutator_size_ / 1024 / 1024,
(double)log_stat.padding_size_ / 1024 / 1024,
(double)log_stat.new_row_size_ / 1024 / 1024,
(double)log_stat.old_row_size_ / 1024 / 1024,
(double)(log_stat.old_row_size_ + log_stat.new_row_size_) / 1024 / 1024,
(double)log_stat.new_primary_row_size_ / 1024 / 1024,
log_stat.primary_row_count_,
log_stat.total_row_count_,
log_stat.total_log_count_,
log_stat.dist_trans_count_,
log_stat.sp_trans_count_,
log_stat.non_compressed_log_cnt_,
log_stat.compressed_log_cnt_,
(double)log_stat.compressed_log_size_ / 1024 / 1024,
(double)log_stat.original_log_size_ / 1024 / 1024,
(0 == log_stat.original_log_size_) ? 1 : (double)log_stat.compressed_log_size_ / log_stat.original_log_size_,
to_cstring(log_stat.compressed_tenant_ids_));
}
}
close_fd(path, fd, buf, buf_len);
return ret;
}
int ObAdminClogV2Executor::mmap_log_file(const char* path, char*& buf_out, int64_t& buf_len, int& fd)
{
int ret = OB_SUCCESS;
void* buf = NULL;
struct stat stat_buf;
if (OB_ISNULL(path)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid log file path is NULL", K(ret));
} else if (is_ofs_file(path)) { // OFS file
ret = OB_NOT_SUPPORTED;
LOG_WARN("not support ofs", K(ret));
} else { // Local file
if (-1 == (fd = open(path, O_RDONLY))) {
ret = OB_IO_ERROR;
CLOG_LOG(ERROR, "open file fail", K(path), KERRMSG, K(ret));
} else if (-1 == fstat(fd, &stat_buf)) {
ret = OB_IO_ERROR;
CLOG_LOG(ERROR, "stat_buf error", K(path), KERRMSG, K(ret));
} else if (stat_buf.st_size > LOG_FILE_MAX_SIZE) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(ERROR, "invalid file size", K(path), K(stat_buf.st_size), K(ret));
} else if (MAP_FAILED == (buf = mmap(NULL, stat_buf.st_size, PROT_READ, MAP_SHARED, fd, 0)) || NULL == buf) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to mmap file", K(path), K(errno), KERRMSG, K(ret));
} else {
buf_out = static_cast<char*>(buf);
buf_len = stat_buf.st_size;
}
}
return ret;
}
int ObAdminClogV2Executor::close_fd(const char* path, const int fd, char* buf, const int64_t buf_len)
{
int ret = OB_SUCCESS;
if (is_ofs_file(path)) {
if (fd > 0) {
// close fd for ofs
}
if (nullptr != buf) {
ob_free(buf);
}
} else {
if (nullptr != buf) {
munmap(buf, buf_len);
}
if (fd >= 0) {
close(fd);
}
}
return ret;
}
} // namespace tools
} // namespace oceanbase

63
tools/ob_admin/clog_tool/ob_admin_clog_v2_executor.h Normal file
View File

@ -0,0 +1,63 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_ADMIN_CLOG_V2_EXECUTOR_H_
#define OB_ADMIN_CLOG_V2_EXECUTOR_H_
#include "../ob_admin_executor.h"
#include "ob_log_entry_filter.h"
#include "lib/string/ob_string.h"
namespace oceanbase {
using namespace clog;
namespace tools {
class ObAdminClogV2Executor : public ObAdminExecutor {
public:
ObAdminClogV2Executor();
virtual ~ObAdminClogV2Executor();
virtual int execute(int argc, char* argv[]);
private:
int dump_all(int argc, char* argv[]);
int dump_filter(int argc, char* argv[]);
int dump_hex(int argc, char* argv[]);
int dump_inner(int argc, char* argv[], bool is_hex);
int dump_format(int argc, char* argv[]);
int stat_clog(int argc, char* argv[]);
int dump_ilog(int argc, char* argv[]);
void print_usage();
int parse_options(int argc, char* argv[]);
int grep(int argc, char* argv[]);
int encode_int(char* buf, int64_t& pos, int64_t buf_len, const char* encode_type, int64_t int_value);
int dump_single_clog(const char* path, bool is_hex);
int dump_single_ilog(const char* path);
int dump_format_single_file(const char* path);
int stat_single_log(const char* path);
int mmap_log_file(const char* path, char*& buf_out, int64_t& buf_len, int& fd);
int close_fd(const char* path, const int fd, char* buf, const int64_t buf_len);
private:
ObLogEntryFilter filter_;
common::ObString DB_host_;
int32_t DB_port_;
bool is_ofs_open_;
};
} // namespace tools
} // namespace oceanbase
#endif /* OB_ADMIN_CLOG_EXECUTOR_V2_H_ */

205
tools/ob_admin/clog_tool/ob_func_utils.cpp Normal file
View File

@ -0,0 +1,205 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_func_utils.h"
const char* get_submit_log_type(const int64_t submit_log_type)
{
const char* char_ret = NULL;
switch (submit_log_type) {
case OB_LOG_SP_TRANS_REDO:
char_ret = "OB_LOG_SP_TRANS_REDO";
break;
case OB_LOG_TRANS_REDO:
char_ret = "OB_LOG_TRANS_REDO";
break;
case OB_LOG_TRANS_PREPARE:
char_ret = "OB_LOG_TRANS_PREPARE";
break;
case OB_LOG_TRANS_REDO_WITH_PREPARE:
char_ret = "OB_LOG_TRANS_REDO_WITH_PREPARE";
break;
case OB_LOG_TRANS_PREPARE_WITH_COMMIT:
char_ret = "OB_LOG_TRANS_PREPARE_WITH_COMMIT";
break;
case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT:
char_ret = "OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT";
break;
case OB_LOG_SP_TRANS_COMMIT:
char_ret = "OB_LOG_SP_TRANS_COMMIT";
break;
case OB_LOG_SP_ELR_TRANS_COMMIT:
char_ret = "OB_LOG_SP_ELR_TRANS_COMMIT";
break;
case OB_LOG_TRANS_COMMIT:
char_ret = "OB_LOG_TRANS_COMMIT";
break;
case OB_LOG_SP_TRANS_ABORT:
char_ret = "OB_LOG_SP_TRANS_ABORT";
break;
case OB_LOG_TRANS_ABORT:
char_ret = "OB_LOG_TRANS_ABORT";
break;
case OB_LOG_TRANS_CLEAR:
char_ret = "OB_LOG_TRANS_CLEAR";
break;
case OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT_WITH_CLEAR:
char_ret = "OB_LOG_TRANS_REDO_WITH_PREPARE_WITH_COMMIT_WITH_CLEAR";
break;
case OB_LOG_MUTATOR:
char_ret = "OB_LOG_MUTATOR";
break;
case OB_LOG_TRANS_STATE:
char_ret = "OB_LOG_TRANS_STATE";
break;
case OB_LOG_MUTATOR_WITH_STATE:
char_ret = "OB_LOG_MUTATOR_WITH_STATE";
break;
case OB_LOG_MUTATOR_ABORT:
char_ret = "OB_LOG_MUTATOR_ABORT";
break;
case OB_LOG_SPLIT_SOURCE_PARTITION:
char_ret = "OB_LOG_SPLIT_SOURCE_PARTITION";
break;
case OB_LOG_SPLIT_DEST_PARTITION:
char_ret = "OB_LOG_SPLIT_DEST_PARTITION";
break;
case OB_LOG_TRANS_CHECKPOINT:
char_ret = "OB_LOG_TRANS_CHECKPOINT";
break;
case OB_LOG_MAJOR_FREEZE:
char_ret = "OB_LOG_MAJOR_FREEZE";
break;
case OB_LOG_ADD_PARTITION_TO_PG:
char_ret = "OB_LOG_ADD_PARTITION_TO_PG";
break;
case OB_LOG_REMOVE_PARTITION_FROM_PG:
char_ret = "OB_LOG_REMOVE_PARTITION_FROM_PG";
break;
case OB_PARTITION_SCHEMA_VERSION_CHANGE_LOG:
char_ret = "OB_PARTITION_SCHEMA_VERSION_CHANGE_LOG";
break;
default:
char_ret = "OB_LOG_UNKNOWN";
break;
}
return char_ret;
}
int file_name_parser(const char* path, uint64_t& file_id)
{
int ret = oceanbase::OB_SUCCESS;
struct stat _stat;
if (OB_ISNULL(path)) {
ret = oceanbase::common::OB_INVALID_ARGUMENT;
} else if (!is_ofs_file(path) && 0 != stat(path, &_stat)) {
ret = OB_IO_ERROR;
_LOGTOOL_LOG(ERROR, "fstate:%s", strerror(errno));
} else {
file_id = 0;
int i = 0;
int path_len = static_cast<int>(strlen(path));
for (--path_len; path_len >= 0 && path[path_len] >= '0' && path[path_len] <= '9'; --path_len) {
file_id += (path[path_len] - '0') * static_cast<int>(pow(10, i++));
}
}
return ret;
}
const char* get_log_type(const enum ObLogType log_type)
{
const char* char_ret = NULL;
switch (log_type) {
case OB_LOG_SUBMIT:
char_ret = "OB_LOG_SUBMIT";
break;
case OB_LOG_MEMBERSHIP:
char_ret = "OB_LOG_MEMBERSHIP";
break;
case OB_LOG_PREPARED:
char_ret = "OB_LOG_PREPARED";
break;
case oceanbase::clog::OB_LOG_NOP:
char_ret = "OB_LOG_NOP";
break;
case OB_LOG_START_MEMBERSHIP:
char_ret = "OB_LOG_START_MEMBERSHIP";
break;
case OB_LOG_NOT_EXIST:
char_ret = "OB_LOG_NOT_EXIST";
break;
case OB_LOG_AGGRE:
char_ret = "OB_LOG_AGGRE";
break;
case OB_LOG_ARCHIVE_CHECKPOINT:
char_ret = "OB_LOG_ARCHIVE_CHECKPOINT";
break;
case OB_LOG_ARCHIVE_KICKOFF:
char_ret = "OB_LOG_ARCHIVE_KICKOFF";
break;
default:
char_ret = "OB_LOG_UNKNOWN";
break;
}
return char_ret;
}
const char* get_freeze_type(ObFreezeType freeze_type)
{
const char* char_ret = NULL;
switch (freeze_type) {
case INVALID_FREEZE:
char_ret = "INVALID_FREEZE";
break;
case MAJOR_FREEZE:
char_ret = "MAJOR_FREEZE";
break;
case MINOR_FREEZE:
char_ret = "MINOR_FREEZE";
break;
}
return char_ret;
}
const char* get_row_dml_type_str(const ObRowDml& dml_type)
{
const char* dml_str = "UNKNOWN";
switch (dml_type) {
case T_DML_INSERT:
dml_str = "INSERT";
break;
case T_DML_UPDATE:
dml_str = "UPDATE";
break;
case T_DML_DELETE:
dml_str = "DELETE";
break;
case T_DML_REPLACE:
dml_str = "REPLACE";
break;
case T_DML_LOCK:
dml_str = "LOCK";
break;
default:
dml_str = "UNKNOWN";
CLOG_LOG(ERROR, "unknown dml_type", K(dml_type));
break;
}
return dml_str;
}
bool is_ofs_file(const char* path)
{
UNUSED(path);
return false;
}

49
tools/ob_admin/clog_tool/ob_func_utils.h Normal file
View File

@ -0,0 +1,49 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_LOG_TOOL_OB_FUN_H
#define OCEANBASE_LOG_TOOL_OB_FUN_H
#include <bitset>
#include <libgen.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/mman.h>
#include "share/ob_define.h"
#include "common/storage/ob_freeze_define.h"
#include "common/cell/ob_cell_reader.h"
#include "clog/ob_log_entry.h"
#include "clog/ob_log_type.h"
#include "storage/transaction/ob_trans_log.h"
#include "storage/memtable/ob_memtable_mutator.h"
using namespace oceanbase;
using namespace oceanbase::clog;
using namespace oceanbase::common;
using namespace oceanbase::transaction;
using namespace oceanbase::memtable;
using namespace oceanbase::storage;
// get the type of clog entry
const char* get_log_type(const enum ObLogType log_type);
const char* get_submit_log_type(const int64_t submit_log_type);
const char* get_freeze_type(ObFreezeType freeze_type);
// this func is used to parse file name
int file_name_parser(const char* path, uint64_t& file_id);
const char* get_row_dml_type_str(const ObRowDml& dml_type);
bool is_ofs_file(const char* path);
const int64_t LOG_FILE_MAX_SIZE = 64 * 1024 * 1024;
#endif // OCEANBASE_LOG_TOOL_OB_FUN_H

118
tools/ob_admin/clog_tool/ob_ilog_entry_parser.cpp Normal file
View File

@ -0,0 +1,118 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <string.h>
#include "ob_ilog_entry_parser.h"
namespace oceanbase {
using namespace common;
using namespace memtable;
namespace clog {
class ObILogEntryParser::DumpIlogEntryFunctor {
public:
DumpIlogEntryFunctor(file_id_t file_id, char* buf, int64_t len) : file_id_(file_id), buf_(buf), buf_len_(len)
{}
~DumpIlogEntryFunctor()
{}
bool operator()(const common::ObPartitionKey& partition_key, const IndexInfoBlockEntry& index_info_block_entry)
{
int ret = OB_SUCCESS;
bool bool_ret = true;
offset_t start_offset = index_info_block_entry.start_offset_;
int64_t cur_pos = start_offset;
ObLogCursorExt cursor;
// int64_t cursor_size = cursor.get_serialize_size();
for (uint64_t log_id = index_info_block_entry.min_log_id_;
OB_SUCC(ret) && bool_ret && log_id <= index_info_block_entry.max_log_id_;
++log_id) {
if (OB_FAIL(cursor.deserialize(buf_, buf_len_, cur_pos))) {
bool_ret = false;
} else {
fprintf(stdout,
"ilog_file_id: %d INDEX_LOG: pk:%s log_id:%lu %s||\n",
file_id_,
to_cstring(partition_key),
log_id,
to_cstring(cursor));
}
}
return bool_ret;
}
private:
file_id_t file_id_;
char* buf_;
int64_t buf_len_;
};
int ObILogEntryParser::init(file_id_t file_id, char* buf, int64_t buf_len)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
} else if (OB_ISNULL(buf) || OB_UNLIKELY(buf_len <= 0) || OB_UNLIKELY(OB_INVALID_FILE_ID == file_id)) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalie buf or buf_len", KP(buf), K(buf_len), K(file_id), K(ret));
} else {
is_inited_ = true;
file_id_ = file_id;
buf_ = buf;
buf_len_ = buf_len;
}
return ret;
}
int ObILogEntryParser::parse_all_entry()
{
int ret = OB_SUCCESS;
if (OB_FAIL(resolve_trailer_and_info_block_map())) {
CLOG_LOG(ERROR, "failed to resolve trailer and info block map", K(ret));
} else if (OB_FAIL(dump_all_entry())) {
CLOG_LOG(ERROR, "failed to dump all entry", K(ret));
}
return ret;
}
int ObILogEntryParser::resolve_trailer_and_info_block_map()
{
int ret = OB_SUCCESS;
int64_t pos = buf_len_ - CLOG_TRAILER_SIZE;
if (OB_FAIL(trailer_.deserialize(buf_, buf_len_, pos))) {
CLOG_LOG(ERROR, "index_info_block_map init failed", K(ret));
} else {
CLOG_LOG(INFO, "DEBUG", K(trailer_));
const int64_t info_block_size = upper_align(trailer_.get_info_block_size(), CLOG_DIO_ALIGN_SIZE);
const int64_t MAX_ENTRY_NUM = info_block_size / index_info_block_map_.item_size() + 1000;
int64_t local_pos = 0;
if (OB_FAIL(index_info_block_map_.init(ObModIds::OB_CLOG_INFO_BLK_HNDLR, MAX_ENTRY_NUM))) {
CLOG_LOG(ERROR, "index_info_block_map init failed", K(ret));
} else if (OB_FAIL(index_info_block_map_.deserialize(
buf_ + trailer_.get_info_block_start_offset(), info_block_size, local_pos))) {
CLOG_LOG(ERROR, "index_info_block_map deserialize failed", K(ret));
} else { /*do nothing*/
}
}
return ret;
}
int ObILogEntryParser::dump_all_entry()
{
int ret = OB_SUCCESS;
DumpIlogEntryFunctor functor(file_id_, buf_, trailer_.get_info_block_start_offset());
if (OB_FAIL(index_info_block_map_.for_each(functor))) {
CLOG_LOG(ERROR, "index_info_block_map_ for_each failed", K(ret));
}
return ret;
}
} // namespace clog
} // end of namespace oceanbase

58
tools/ob_admin/clog_tool/ob_ilog_entry_parser.h Normal file
View File

@ -0,0 +1,58 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_TOOL_iLOG_ENTRY_PARSER
#define OCEANBASE_TOOL_iLOG_ENTRY_PARSER
#include "ob_func_utils.h"
#include "clog/ob_log_block.h"
#include "clog/ob_log_file_trailer.h"
#include "clog/ob_info_block_handler.h"
//#include "lib/allocator/page_arena.h"
namespace oceanbase {
namespace clog {
class ObILogEntryParser {
public:
ObILogEntryParser()
: is_inited_(false), file_id_(OB_INVALID_FILE_ID), buf_(NULL), buf_len_(0), trailer_(), index_info_block_map_()
{}
virtual ~ObILogEntryParser()
{}
int init(file_id_t file_id, char* buf, int64_t buf_len);
int parse_all_entry();
private:
class DumpIlogEntryFunctor;
private:
int resolve_trailer_and_info_block_map();
int dump_all_entry();
private:
static const int64_t PRINT_BUF_SIZE = 5 * 1024 * 1024;
bool is_inited_;
file_id_t file_id_;
char* buf_;
int64_t buf_len_;
ObIlogFileTrailerV2 trailer_;
IndexInfoBlockMap index_info_block_map_;
DISALLOW_COPY_AND_ASSIGN(ObILogEntryParser);
};
} // end namespace clog
} // end namespace oceanbase
#endif // OCEANBASE_CLOG_OB_RAW_ENTRY_ITERATOR_

76
tools/ob_admin/clog_tool/ob_log_entry_filter.cpp Normal file
View File

@ -0,0 +1,76 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <string.h>
#include <stdlib.h>
#include "lib/ob_errno.h"
#include "ob_log_entry_filter.h"
namespace oceanbase {
namespace clog {
int ObLogEntryFilter::parse(const char* str)
{
int ret = OB_SUCCESS;
char* ptr1 = NULL;
char* saveptr1 = NULL;
char* token1 = NULL;
char buf[1024];
char tmp[128];
const char* TABLE_ID_STR = "table_id";
const char* PARTITION_ID_STR = "partition_id";
const char* LOG_ID_STR = "log_id";
const char* TRANS_ID_STR = "trans_id";
if (NULL != str) {
strncpy(buf, str, sizeof(buf));
for (ptr1 = buf;; ptr1 = NULL) {
token1 = strtok_r(ptr1, ";", &saveptr1);
if (NULL == token1) {
break;
} else {
int i = 0;
char* ptr2 = NULL;
char* saveptr2 = NULL;
char* token2 = NULL;
for (i = 1, ptr2 = token1;; ptr2 = NULL, i++) {
token2 = strtok_r(ptr2, "=", &saveptr2);
if (NULL == token2) {
break;
} else if (1 == (i % 2)) {
strncpy(tmp, token2, sizeof(tmp));
} else {
if (0 == strcmp(tmp, TABLE_ID_STR)) {
table_id_ = atol(token2);
is_table_id_valid_ = true;
} else if (0 == strcmp(tmp, PARTITION_ID_STR)) {
partition_id_ = atol(token2);
is_partition_id_valid_ = true;
} else if (0 == strcmp(tmp, LOG_ID_STR)) {
log_id_ = atol(token2);
is_log_id_valid_ = true;
} else if (0 == strcmp(tmp, TRANS_ID_STR)) {
trans_id_ = atol(token2);
is_trans_id_valid_ = true;
} else {
// do nothing
}
}
}
}
}
}
return ret;
}
} // end of namespace clog
} // end of namespace oceanbase

91
tools/ob_admin/clog_tool/ob_log_entry_filter.h Normal file
View File

@ -0,0 +1,91 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_TOOL_LOG_ENTRY_FILTER_H_
#define OCEANBASE_TOOL_LOG_ENTRY_FILTER_H_
#include <stdint.h>
#include "lib/utility/utility.h"
namespace oceanbase {
namespace clog {
class ObLogEntryFilter {
public:
ObLogEntryFilter()
: table_id_(0),
partition_id_(0),
trans_id_(0),
log_id_(0),
is_table_id_valid_(false),
is_partition_id_valid_(false),
is_trans_id_valid_(false),
is_log_id_valid_(false)
{}
~ObLogEntryFilter()
{}
int parse(const char* str);
bool is_table_id_valid() const
{
return is_table_id_valid_;
}
bool is_partition_id_valid() const
{
return is_partition_id_valid_;
}
bool is_trans_id_valid() const
{
return is_trans_id_valid_;
}
bool is_log_id_valid() const
{
return is_log_id_valid_;
}
uint64_t get_table_id() const
{
return table_id_;
}
int64_t get_partition_id() const
{
return partition_id_;
}
uint64_t get_trans_id() const
{
return trans_id_;
}
int64_t get_log_id() const
{
return log_id_;
}
bool is_valid() const
{
return is_table_id_valid_ || is_partition_id_valid_ || is_trans_id_valid_ || is_log_id_valid_;
}
public:
TO_STRING_KV(K_(table_id), K_(partition_id), K_(trans_id), K_(log_id), K_(is_table_id_valid),
K_(is_partition_id_valid), K_(is_trans_id_valid), K_(is_log_id_valid));
private:
uint64_t table_id_;
int64_t partition_id_;
uint64_t trans_id_;
int64_t log_id_;
bool is_table_id_valid_;
bool is_partition_id_valid_;
bool is_trans_id_valid_;
bool is_log_id_valid_;
};
} // namespace clog
} // end namespace oceanbase
#endif // OCEANBASE_TOOL_LOG_ENTRY_FILTER_H_

2299
tools/ob_admin/clog_tool/ob_log_entry_parser.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

252
tools/ob_admin/clog_tool/ob_log_entry_parser.h Normal file
View File

@ -0,0 +1,252 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_TOOL_LOG_ENTRY_PARSER
#define OCEANBASE_TOOL_LOG_ENTRY_PARSER
#include "ob_func_utils.h"
#include "ob_log_entry_filter.h"
#include "clog/ob_log_block.h"
#include "clog/ob_raw_entry_iterator.h"
#include "lib/net/ob_addr.h"
#include "share/ob_srv_rpc_proxy.h"
#include "rpc/obrpc/ob_net_client.h"
namespace oceanbase {
namespace share {
class ObKmsClient;
}
namespace clog {
struct ObLogStat {
ObLogStat()
: primary_table_id_(OB_INVALID_ID),
data_block_header_size_(0),
log_header_size_(0),
log_size_(0),
trans_log_size_(0),
mutator_size_(0),
padding_size_(0),
new_row_size_(0),
old_row_size_(0),
new_primary_row_size_(0),
primary_row_count_(0),
total_row_count_(0),
total_log_count_(0),
dist_trans_count_(0),
sp_trans_count_(0),
non_compressed_log_cnt_(0),
compressed_log_cnt_(0),
compressed_log_size_(0),
original_log_size_(0),
compressed_tenant_ids_()
{}
~ObLogStat()
{}
int init();
uint64_t primary_table_id_;
int64_t data_block_header_size_;
int64_t log_header_size_;
int64_t log_size_;
int64_t trans_log_size_;
int64_t mutator_size_;
int64_t padding_size_;
int64_t new_row_size_;
int64_t old_row_size_;
int64_t new_primary_row_size_;
int64_t primary_row_count_;
int64_t total_row_count_;
int64_t total_log_count_;
int64_t dist_trans_count_;
int64_t sp_trans_count_;
// compressed info
int64_t non_compressed_log_cnt_; // number of uncompressed entries`
int64_t compressed_log_cnt_; // number of compressed entries
int64_t compressed_log_size_; // data size of compressed entries
int64_t original_log_size_; // data size of compressed entries before compression
hash::ObHashSet<uint64_t> compressed_tenant_ids_;
TO_STRING_KV(K_(data_block_header_size), K_(log_header_size), K_(log_size), K_(trans_log_size), K_(mutator_size),
K_(padding_size), K_(new_row_size), K_(old_row_size), K_(new_primary_row_size), K_(primary_row_count),
K_(total_row_count), K_(total_log_count), K_(dist_trans_count), K_(sp_trans_count), K_(non_compressed_log_cnt),
K_(compressed_log_cnt), K_(compressed_log_size), K_(original_log_size), K_(compressed_tenant_ids));
private:
DISALLOW_COPY_AND_ASSIGN(ObLogStat);
};
class ObInfoEntryDumpFunctor {
public:
ObInfoEntryDumpFunctor(const uint64_t file_id) : file_id_(file_id)
{}
virtual ~ObInfoEntryDumpFunctor()
{}
bool operator()(const common::ObPartitionKey& partition_key, const uint64_t min_log_id);
private:
uint64_t file_id_;
};
enum ObLogFileType {
OB_UNKNOWN_FILE_TYPE,
OB_CLOG_FILE_TYPE,
OB_ILOG_FILE_TYPE,
OB_MAX_FILE_TYPE,
};
class ObLogEntryParserImpl {
public:
ObLogEntryParserImpl()
: is_inited_(false),
dump_hex_(false),
file_id_(-1),
cur_offset_(OB_INVALID_OFFSET),
print_buf_(NULL),
allocator_(ObModIds::OB_LOG_TOOL)
{}
virtual ~ObLogEntryParserImpl()
{}
int init(const int64_t file_id, const ObLogEntryFilter& filter, const common::ObString& host, const int32_t port,
const char* config_file);
bool is_inited() const
{
return is_inited_;
};
int dump_clog_entry(ObLogEntry& entry, int64_t pos);
protected:
// parse mutator data
int dump_memtable_mutator(const char* buf, int64_t len);
// parse trans log entry
int dump_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_sp_trans_abort_log(const char* data, int64_t len);
int dump_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_trans_prepare_log(const char* data, int64_t len);
int dump_trans_commit_log(const char* data, int64_t len);
int dump_trans_abort_log(const char* data, int64_t len);
int dump_trans_clear_log(const char* data, int64_t len);
int dump_trans_prepare_commit_log(const char* data, int64_t len);
int dump_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_trans_redo_prepare_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_trans_prepare_commit_clear_log(const char* data, int64_t len);
int dump_trans_redo_prepare_commit_clear_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_trans_mutator_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_trans_mutator_state_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int dump_trans_state_log(const char* data, int64_t len);
int dump_trans_mutator_abort_log(const char* data, int64_t len);
int dump_part_split_src_log(const char* data, int64_t len);
int dump_part_split_dest_log(const char* data, int64_t len);
int dump_trans_checkpoint_log(const char* data, int64_t len);
int dump_new_offline_partition_log(const char* data, int64_t len);
int dump_add_partition_to_pg_log(const char* data, int64_t len);
int dump_remove_partition_from_pg_log(const char* data, int64_t len);
int dump_trans_log(const ObStorageLogType log_type, const int64_t trans_inc, const uint64_t real_tenant_id,
const char* buf, const int64_t buf_len, int64_t& pos);
int dump_partition_schema_version_change_log(const char* data, int64_t len);
// parse freeze log
int dump_freeze_log(const char* buf, const int64_t buf_len, ObStorageLogType& log_type, ObFreezeType& freeze_type,
ObPartitionKey& pkey, int64_t& frozen_version, ObSavedStorageInfo& info);
int dump_obj(const common::ObObj& obj, uint64_t column_id);
int format_dump_clog_entry(ObLogEntry& entry);
int format_dump_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int format_dump_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int format_dump_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int format_dump_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int format_dump_trans_redo_prepare_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int format_dump_trans_redo_prepare_commit_clear_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int format_dump_memtable_mutator(const char* buf, int64_t len);
int format_dump_obj(const common::ObObj& obj, uint64_t column_id);
int check_filter(const ObLogEntry& entry, bool& need_print);
private:
int dump_clog_entry_(ObLogEntry& entry, int64_t pos);
protected:
static const int64_t MAGIC_NUM_LEN = 2L;
static const int64_t SKIP_STEP = 4 * 1024L;
static const int64_t PRINT_BUF_SIZE = 5 * 1024 * 1024;
bool is_inited_;
bool dump_hex_;
uint64_t file_id_;
int64_t cur_offset_;
ObTransID cur_trans_id_;
char* print_buf_;
ObLogEntryFilter filter_;
ObArenaAllocator allocator_;
common::ObAddr host_addr_;
obrpc::ObNetClient client_;
obrpc::ObSrvRpcProxy rpc_proxy_;
};
class ObLogEntryParser : public ObLogEntryParserImpl {
public:
ObLogEntryParser()
: ObLogEntryParserImpl(), log_file_type_(OB_UNKNOWN_FILE_TYPE), buf_cur_(NULL), buf_end_(NULL), is_ofs_(false)
{}
virtual ~ObLogEntryParser()
{}
int init(uint64_t file_id, char* buf, int64_t buf_len, const ObLogEntryFilter& filter, const common::ObString& host,
const int32_t port, const char* config_file, const bool is_ofs);
int dump_all_entry(bool is_hex);
int format_dump_entry();
int stat_log();
const ObLogStat& get_log_stat() const
{
return log_stat_;
}
protected:
int get_type_(ObCLogItemType& item_type);
int dump_block_(const ObLogBlockMetaV2& meta);
inline void advance_(const int64_t step);
int advance_to_next_align_();
int parse_next_entry();
int format_dump_next_entry();
int stat_next_entry();
// for format_dump
int skip_block_(const ObLogBlockMetaV2& meta);
int dump_ilog_entry(ObIndexEntry& entry);
// for stat clog file
int stat_block_(const ObLogBlockMetaV2& meta);
int stat_clog_entry(const ObLogEntry& entry, const int64_t pos, const bool is_compressed);
int stat_sp_trans_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int stat_sp_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int stat_trans_redo_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int stat_trans_redo_prepare_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int stat_trans_redo_prepare_commit_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int stat_trans_redo_prepare_commit_clear_log(const char* data, int64_t len, const uint64_t real_tenant_id);
int stat_memtable_mutator(const char* buf, int64_t len);
protected:
static const int64_t MAGIC_NUM_LEN = 2L;
static const int64_t SKIP_STEP = 4 * 1024L;
static const int64_t PRINT_BUF_SIZE = 5 * 1024 * 1024;
ObLogFileType log_file_type_;
char* buf_cur_;
char* buf_end_;
bool is_ofs_;
ObLogStat log_stat_;
ObReadBuf compress_rbuf_;
DISALLOW_COPY_AND_ASSIGN(ObLogEntryParser);
};
} // namespace clog
} // end namespace oceanbase
#endif // OCEANBASE_CLOG_OB_RAW_ENTRY_ITERATOR_

60
tools/ob_admin/main.cpp Normal file
View File

@ -0,0 +1,60 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <iostream>
#include <sstream>
#include <iterator>
#include <string.h>
#include "share/ob_define.h"
#include "ob_admin_executor.h"
#include "clog_tool/ob_admin_clog_v2_executor.h"
using namespace oceanbase::common;
using namespace oceanbase::tools;
void print_usage()
{
fprintf(stderr, "\nUsage: ob_admin clog_tool\n");
}
int main(int argc, char* argv[])
{
int ret = 0;
OB_LOGGER.set_log_level("INFO");
OB_LOGGER.set_file_name("ob_admin.log", true, false);
const char* log_level = getenv("OB_ADMIN_LOG_LEVEL");
if (NULL != log_level) {
OB_LOGGER.set_log_level(log_level);
}
std::ostringstream ss;
copy(argv, argv + argc, std::ostream_iterator<char*>(ss, " "));
_OB_LOG(INFO, "cmd: [%s]", ss.str().c_str());
ObAdminExecutor* executor = NULL;
if (argc < 2) {
print_usage();
} else {
if (0 == strcmp("clog_tool", argv[1])) {
executor = new ObAdminClogV2Executor();
} else {
print_usage();
}
if (NULL != executor) {
if (OB_FAIL(executor->execute(argc, argv))) {
COMMON_LOG(WARN, "Fail to executor cmd, ", K(ret));
}
delete executor;
}
}
return ret;
}

57
tools/ob_admin/ob_admin_executor.cpp Normal file
View File

@ -0,0 +1,57 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_admin_executor.h"
namespace oceanbase {
using namespace common;
namespace tools {
int ObAdminExecutor::parse_options(int argc, char* argv[])
{
int ret = OB_SUCCESS;
int option_index = 0;
struct option long_options[] = {{"host", 1, NULL, 'h'},
{"port", 1, NULL, 'p'},
{"config_file", 1, NULL, 'f'},
{"tenant_id", 1, NULL, 't'},
{"wallet", 1, NULL, 'w'},
{NULL, 0, NULL, 0}};
int c;
while (-1 != (c = getopt_long(argc, argv, "h:p:f:t:w:", long_options, &option_index))) {
switch (c) {
case 'h':
DB_host_.assign_ptr(optarg, strlen(optarg));
break;
case 'p':
DB_port_ = static_cast<int32_t>(strtol(optarg, NULL, 10));
break;
case 'f':
config_file_ = optarg;
break;
case 't':
tenant_id_ = static_cast<uint64_t>(strtol(optarg, NULL, 10));
break;
case 'w':
wallet_file_ = optarg;
break;
case '?':
case ':':
ret = OB_ERR_UNEXPECTED;
break;
default:
break;
}
}
return ret;
}
} // namespace tools
} // namespace oceanbase

43
tools/ob_admin/ob_admin_executor.h Normal file
View File

@ -0,0 +1,43 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_ADMIN_EXECUTOR_H_
#define OB_ADMIN_EXECUTOR_H_
#include <stdlib.h>
#include <stdio.h>
#include <getopt.h>
#include "share/ob_define.h"
namespace oceanbase {
namespace tools {
class ObAdminExecutor {
public:
ObAdminExecutor() : DB_port_(-1), tenant_id_(0), config_file_(NULL), wallet_file_(NULL)
{}
virtual ~ObAdminExecutor()
{}
virtual int execute(int argc, char* argv[]) = 0;
protected:
int parse_options(int argc, char* argv[]);
protected:
common::ObString DB_host_;
int32_t DB_port_;
uint64_t tenant_id_;
const char* config_file_;
const char* wallet_file_;
};
} // namespace tools
} // namespace oceanbase
#endif /* OB_ADMIN_EXECUTOR_H_ */