hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Dump shared macro block

Browse Source
This commit is contained in:
JiahuaChen 2022-12-27 08:42:22 +00:00 committed by ob-robot
parent 59c443225d
commit b2ce07cc06
2 changed files with 71 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

196
tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp
View File

@ -34,9 +34,7 @@ namespace tools
ObAdminDumpsstExecutor::ObAdminDumpsstExecutor()
:ObAdminExecutor(),
is_quiet_(false),
in_csv_(false),
cmd_(DUMP_MAX),
skip_log_replay_(false),
hex_print_(false),
dump_macro_context_(),
key_hex_str_(NULL),
@ -57,10 +55,7 @@ int ObAdminDumpsstExecutor::execute(int argc, char *argv[])
lib::set_memory_limit(96 * 1024 * 1024 * 1024LL);
lib::set_tenant_memory_limit(500, 96 * 1024 * 1024 * 1024LL);
if (skip_log_replay_ && DUMP_MACRO_DATA != cmd_) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(ERROR, "Only dump macro block can skip slog replay, ", K(ret));
} else if (OB_FAIL(ObKVGlobalCache::get_instance().init(
if (OB_FAIL(ObKVGlobalCache::get_instance().init(
&ObTenantMemLimitGetter::get_instance(), 1024L, 512 * 1024 * 1024, 64 * 1024))) {
STORAGE_LOG(ERROR, "Fail to init kv cache, ", K(ret));
} else if (OB_FAIL(OB_STORE_CACHE.init(
@ -95,15 +90,9 @@ int ObAdminDumpsstExecutor::execute(int argc, char *argv[])
case DUMP_MACRO_DATA:
dump_macro_block(dump_macro_context_);
break;
case DUMP_SSTABLE:
dump_sstable();
break;
case PRINT_MACRO_BLOCK:
print_macro_block();
break;
case DUMP_SSTABLE_META:
dump_sstable_meta();
break;
default:
print_usage();
exit(1);
@ -148,7 +137,7 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
int ret = OB_SUCCESS;
int opt = 0;
const char* opt_string = "d:hf:a:i:n:qt:sxk:m:";
const char* opt_string = "d:hf:a:i:n:qxk:m:";
struct option longopts[] = {
// commands
@ -159,10 +148,7 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
{ "macro-id", 1, NULL, 'a' },
{ "micro-id", 1, NULL, 'i' },
{ "macro-size", 1, NULL, 'n' },
{ "csv", 0, NULL, 0 },
{ "table_key", 1, NULL, 't'},
{ "quiet", 0, NULL, 'q' },
{ "skip_replay", 0, NULL, 's' },
{ "hex-print", 0, NULL, 'x' },
{ "master_key", 1, NULL, 'k'},
{ "master_key_id", 1, NULL, 'm'}
@ -180,10 +166,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
cmd_ = DUMP_SUPER_BLOCK;
} else if (0 == strcmp(optarg, "mb") || 0 == strcmp(optarg, "macro_block")) {
cmd_ = DUMP_MACRO_DATA;
} else if (0 == strcmp(optarg, "sst") || 0 == strcmp(optarg, "sstable")) {
cmd_ = DUMP_SSTABLE;
} else if (0 == strcmp(optarg, "sstm") || 0 == strcmp(optarg, "sstable_meta")) {
cmd_ = DUMP_SSTABLE_META;
} else if (0 == strcmp(optarg, "pm") || 0 == strcmp(optarg, "print_macro")) {
cmd_ = PRINT_MACRO_BLOCK;
} else {
@ -215,10 +197,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
is_quiet_ = true;
break;
}
case 's': {
skip_log_replay_ = true;
break;
}
case 'x': {
hex_print_ = true;
break;
@ -227,23 +205,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
master_key_id_ = strtoll(optarg, NULL, 10);
break;
}
case 0: {
std::string name = longopts[index].name;
if ("csv" == name) {
in_csv_ = true;
}
break;
}
case 't': {
// if (OB_FAIL(parse_table_key(optarg, table_key_))) {
// printf("failed to parse table key\n");
// print_usage();
// exit(1);
// }
// break;
print_usage();
exit(1);
}
case 'k': {
key_hex_str_ = optarg;
break;
@ -268,11 +229,64 @@ void ObAdminDumpsstExecutor::print_macro_block()
STORAGE_LOG(ERROR, "not supported command", K(ret));
}
int ObAdminDumpsstExecutor::dump_single_macro_block(const char* buf, const int64_t size)
{
int ret = OB_SUCCESS;
ObSSTableDataBlockReader macro_reader;
if (OB_ISNULL(buf) || size <= 0) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(ERROR, "invalid argument", K(ret), KP(buf), K(size));
} else if (OB_FAIL(macro_reader.init(buf, size, hex_print_))) {
STORAGE_LOG(ERROR, "failed to init macro reader", K(ret), KP(buf), K(size));
} else if (OB_FAIL(macro_reader.dump())) {
STORAGE_LOG(ERROR, "failed dump macro block", K(ret), KP(buf), K(size));
}
return ret;
}
int ObAdminDumpsstExecutor::dump_shared_macro_block(const char* buf, const int64_t size)
{
int ret = OB_SUCCESS;
const int64_t aligned_size = 4096;
int64_t current_page_offset = aligned_size;
if (OB_ISNULL(buf) || size <= 0) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(ERROR, "invalid argument", K(ret), KP(buf), K(size));
} else {
while (OB_SUCC(ret) && current_page_offset < size) {
ObMacroBlockCommonHeader common_header;
int64_t pos = 0;
const char* cur_buf = buf + current_page_offset;
const int64_t cur_size = size - current_page_offset;
if (OB_FAIL(common_header.deserialize(cur_buf, cur_size, pos))) {
if (OB_DESERIALIZE_ERROR != ret) {
STORAGE_LOG(ERROR, "deserialize common header fail", K(ret), K(pos));
} else {
ret = OB_SUCCESS;
break;
}
} else if (OB_FAIL(common_header.check_integrity())) {
STORAGE_LOG(ERROR, "invalid common header", K(ret), K(common_header));
} else if OB_FAIL(dump_single_macro_block(cur_buf,
common_header.get_header_size() + common_header.get_payload_size())) {
STORAGE_LOG(ERROR, "dump single block fail", K(ret), K(common_header));
} else {
current_page_offset = upper_align(
current_page_offset + common_header.get_header_size() + common_header.get_payload_size(),
aligned_size);
}
}
}
STORAGE_LOG(INFO, "dump shared block finish", K(ret), K(current_page_offset));
return ret;
}
int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macro_block_context)
{
int ret = OB_SUCCESS;
ObMacroBlockHandle macro_handle;
ObSSTableDataBlockReader macro_reader;
ObMacroBlockCommonHeader common_header;
int64_t pos = 0;
ObMacroBlockReadInfo read_info;
read_info.macro_block_id_.set_block_index(macro_block_context.second_id_);
@ -280,7 +294,6 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macr
read_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_READ);
read_info.offset_ = 0;
read_info.size_ = OB_DEFAULT_MACRO_BLOCK_SIZE;
STORAGE_LOG(INFO, "begin dump macro block", K(macro_block_context));
if (OB_UNLIKELY(!macro_block_context.is_valid())) {
@ -288,10 +301,18 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macr
STORAGE_LOG(ERROR, "invalid macro block id", K(macro_block_context), K(ret));
} else if (OB_FAIL(ObBlockManager::read_block(read_info, macro_handle))) {
STORAGE_LOG(ERROR, "Fail to read macro block, ", K(ret), K(read_info));
} else if (OB_FAIL(macro_reader.init(macro_handle.get_buffer(), macro_handle.get_data_size(), hex_print_))) {
STORAGE_LOG(ERROR, "failed to init macro reader", K(read_info), K(ret));
} else if (OB_FAIL(macro_reader.dump())) {
STORAGE_LOG(ERROR, "failed dump macro block", K(ret));
} else if (OB_FAIL(common_header.deserialize(macro_handle.get_buffer(), macro_handle.get_data_size(), pos))) {
STORAGE_LOG(ERROR, "deserialize common header fail", K(ret), K(pos));
} else if (OB_FAIL(common_header.check_integrity())) {
STORAGE_LOG(ERROR, "invalid common header", K(ret), K(common_header));
} else if (ObMacroBlockCommonHeader::SharedSSTableData == common_header.get_type()) {
if (OB_FAIL(dump_shared_macro_block(macro_handle.get_buffer(), macro_handle.get_data_size()))) {
STORAGE_LOG(ERROR, "dump shared block fail", K(ret));
}
} else {
if (OB_FAIL(dump_single_macro_block(macro_handle.get_buffer(), macro_handle.get_data_size()))) {
STORAGE_LOG(ERROR, "dump single block fail", K(ret));
}
}
macro_handle.reset();
@ -300,89 +321,12 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macr
}
void ObAdminDumpsstExecutor::dump_sstable()
{
int ret = OB_NOT_IMPLEMENT;
STORAGE_LOG(ERROR, "not supported command", K(ret));
// adapt later if needed
/*
ObSSTable *sstable = NULL;
if (OB_FAIL(replay_slog_to_get_sstable(sstable))) {
STORAGE_LOG(ERROR, "failed to acquire table", K_(table_key), K(ret));
} else if (OB_ISNULL(sstable)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "sstable is null", K(ret));
} else {
const ObIArray<blocksstable::MacroBlockId> &macro_array = sstable->get_macro_block_ids();
ObDumpMacroBlockContext context;
for (int64_t i = 0; OB_SUCC(ret) && i < macro_array.count(); ++i) {
context.second_id_ = macro_array.at(i).block_index();
if (OB_FAIL(dump_macro_block(context))) {
STORAGE_LOG(ERROR, "failed to dump macro block", K(context), K(ret));
}
}
if (OB_SUCC(ret) && sstable->has_lob_macro_blocks()) {
for (int64_t i = 0; OB_SUCC(ret) && i < sstable->get_lob_macro_block_ids().count(); ++i) {
context.second_id_ = sstable->get_lob_macro_block_ids().at(i).block_index();
if (OB_FAIL(dump_macro_block(context))) {
STORAGE_LOG(ERROR, "Failed to dump lob macro block", K(context), K(ret));
}
}
}
}
if (OB_FAIL(ret)) {
printf("failed to dump sstable, ret = %d\n", ret);
}
*/
}
void ObAdminDumpsstExecutor::dump_sstable_meta()
{
int ret = OB_NOT_IMPLEMENT;
STORAGE_LOG(ERROR, "not supported command", K(ret));
/*
int ret = OB_SUCCESS;
ObTableHandle handle;
ObSSTable *sstable = NULL;
if (OB_FAIL(ObPartitionService::get_instance().acquire_sstable(table_key_, handle))) {
STORAGE_LOG(ERROR, "fail to acquire table", K(ret), K(table_key_));
} else if (OB_FAIL(handle.get_sstable(sstable))) {
STORAGE_LOG(ERROR, "fail to get sstable", K(ret));
} else if (OB_ISNULL(sstable)) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "error unexpected, sstable must not be NULL", K(ret));
} else {
char buf[1024];
const ObSSTableMeta &meta = sstable->get_meta();
snprintf(buf, 1024, "table_id=%ld, tablet_id=%ld", table_key_.table_id_, table_key_.tablet_id_.id());
PrintHelper::print_dump_title(buf);
PrintHelper::print_dump_line("index_id", meta.index_id_);
PrintHelper::print_dump_line("row_count", meta.row_count_);
PrintHelper::print_dump_line("occupy_size", meta.occupy_size_);
PrintHelper::print_dump_line("data_checksum", meta.data_checksum_);
PrintHelper::print_dump_line("row_checksum", meta.row_checksum_);
PrintHelper::print_dump_line("macro_block_count", meta.macro_block_count_);
PrintHelper::print_dump_line("use_old_macro_block_count", meta.use_old_macro_block_count_);
PrintHelper::print_dump_line("column_count", meta.column_cnt_);
PrintHelper::print_dump_line("schema_version", meta.schema_version_);
PrintHelper::print_dump_line("progressive_merge_start_version", meta.progressive_merge_start_version_);
PrintHelper::print_dump_line("progressive_merge_end_version", meta.progressive_merge_end_version_);
PrintHelper::print_dump_line("checksum_method", meta.checksum_method_);
meta.column_metas_.to_string(buf, 1024);
PrintHelper::print_dump_line("column_metas", buf);
meta.new_column_metas_.to_string(buf, 1024);
PrintHelper::print_dump_line("new_column_metas", buf);
}
*/
}
void ObAdminDumpsstExecutor::print_usage()
{
printf("\n");
printf("Usage: dumpsst command [command args] [options]\n");
printf("commands:\n");
printf(HELP_FMT, "-d,--dump", "dump, args: [super_block|print_macro|macro_block|macro_meta|sstable|sstable_meta]");
printf(HELP_FMT, "-d,--dump", "dump, args: [super_block|print_macro|macro_block|macro_meta]");
printf(HELP_FMT, "-h,--help", "display this message.");
printf("options:\n");
@ -391,8 +335,6 @@ void ObAdminDumpsstExecutor::print_usage()
printf(HELP_FMT, "-i,--micro-id", "micro block id, -1 means all micro blocks");
printf(HELP_FMT, "-n,--macro-size", "macro block size, in bytes");
printf(HELP_FMT, "-q,--quiet", "log level: ERROR");
printf(HELP_FMT, "-t,--table-key", "table key: table_type,table_id:partition_id,index_id,base_version:multi_version_start:snapshot_version,start_log_ts:end_log_ts:max_log_ts,major_version");
printf(HELP_FMT, "-s,--skip_replay", "skip slog replay, only work for macro_block mode");
printf(HELP_FMT, "-x,--hex-print", "print obj value in hex mode");
printf(HELP_FMT, "-k,--master_key", "master key, hex str");
printf(HELP_FMT, "-m,--master_key_id", "master key id");

7
tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h
View File

@ -31,8 +31,6 @@ enum ObAdminDumpsstCmd
DUMP_SUPER_BLOCK,
DUMP_MACRO_DATA,
PRINT_MACRO_BLOCK,
DUMP_SSTABLE,
DUMP_SSTABLE_META,
DUMP_MAX,
};
@ -64,14 +62,13 @@ private:
void print_macro_meta();
void print_super_block();
int dump_macro_block(const ObDumpMacroBlockContext &context);
int dump_single_macro_block(const char* buf, const int64_t size);
int dump_shared_macro_block(const char* buf, const int64_t size);
void dump_sstable();
void dump_sstable_meta();
bool is_quiet_;
bool in_csv_;
ObAdminDumpsstCmd cmd_;
storage::ObITable::TableKey table_key_;
bool skip_log_replay_;
bool hex_print_;
ObDumpMacroBlockContext dump_macro_context_;
char *key_hex_str_;