From b2ce07cc06f92a6b0f7bb05dcb149b25fc1215d1 Mon Sep 17 00:00:00 2001 From: JiahuaChen Date: Tue, 27 Dec 2022 08:42:22 +0000 Subject: [PATCH] Dump shared macro block --- .../dumpsst/ob_admin_dumpsst_executor.cpp | 196 ++++++------------ .../dumpsst/ob_admin_dumpsst_executor.h | 7 +- 2 files changed, 71 insertions(+), 132 deletions(-) diff --git a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp index 79c0c5640..2154a581b 100644 --- a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp +++ b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp @@ -34,9 +34,7 @@ namespace tools ObAdminDumpsstExecutor::ObAdminDumpsstExecutor() :ObAdminExecutor(), is_quiet_(false), - in_csv_(false), cmd_(DUMP_MAX), - skip_log_replay_(false), hex_print_(false), dump_macro_context_(), key_hex_str_(NULL), @@ -57,10 +55,7 @@ int ObAdminDumpsstExecutor::execute(int argc, char *argv[]) lib::set_memory_limit(96 * 1024 * 1024 * 1024LL); lib::set_tenant_memory_limit(500, 96 * 1024 * 1024 * 1024LL); - if (skip_log_replay_ && DUMP_MACRO_DATA != cmd_) { - ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(ERROR, "Only dump macro block can skip slog replay, ", K(ret)); - } else if (OB_FAIL(ObKVGlobalCache::get_instance().init( + if (OB_FAIL(ObKVGlobalCache::get_instance().init( &ObTenantMemLimitGetter::get_instance(), 1024L, 512 * 1024 * 1024, 64 * 1024))) { STORAGE_LOG(ERROR, "Fail to init kv cache, ", K(ret)); } else if (OB_FAIL(OB_STORE_CACHE.init( @@ -95,15 +90,9 @@ int ObAdminDumpsstExecutor::execute(int argc, char *argv[]) case DUMP_MACRO_DATA: dump_macro_block(dump_macro_context_); break; - case DUMP_SSTABLE: - dump_sstable(); - break; case PRINT_MACRO_BLOCK: print_macro_block(); break; - case DUMP_SSTABLE_META: - dump_sstable_meta(); - break; default: print_usage(); exit(1); @@ -148,7 +137,7 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[]) int ret = OB_SUCCESS; int opt = 0; - const char* opt_string = "d:hf:a:i:n:qt:sxk:m:"; + const char* opt_string = "d:hf:a:i:n:qxk:m:"; struct option longopts[] = { // commands @@ -159,10 +148,7 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[]) { "macro-id", 1, NULL, 'a' }, { "micro-id", 1, NULL, 'i' }, { "macro-size", 1, NULL, 'n' }, - { "csv", 0, NULL, 0 }, - { "table_key", 1, NULL, 't'}, { "quiet", 0, NULL, 'q' }, - { "skip_replay", 0, NULL, 's' }, { "hex-print", 0, NULL, 'x' }, { "master_key", 1, NULL, 'k'}, { "master_key_id", 1, NULL, 'm'} @@ -180,10 +166,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[]) cmd_ = DUMP_SUPER_BLOCK; } else if (0 == strcmp(optarg, "mb") || 0 == strcmp(optarg, "macro_block")) { cmd_ = DUMP_MACRO_DATA; - } else if (0 == strcmp(optarg, "sst") || 0 == strcmp(optarg, "sstable")) { - cmd_ = DUMP_SSTABLE; - } else if (0 == strcmp(optarg, "sstm") || 0 == strcmp(optarg, "sstable_meta")) { - cmd_ = DUMP_SSTABLE_META; } else if (0 == strcmp(optarg, "pm") || 0 == strcmp(optarg, "print_macro")) { cmd_ = PRINT_MACRO_BLOCK; } else { @@ -215,10 +197,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[]) is_quiet_ = true; break; } - case 's': { - skip_log_replay_ = true; - break; - } case 'x': { hex_print_ = true; break; @@ -227,23 +205,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[]) master_key_id_ = strtoll(optarg, NULL, 10); break; } - case 0: { - std::string name = longopts[index].name; - if ("csv" == name) { - in_csv_ = true; - } - break; - } - case 't': { - // if (OB_FAIL(parse_table_key(optarg, table_key_))) { - // printf("failed to parse table key\n"); - // print_usage(); - // exit(1); - // } - // break; - print_usage(); - exit(1); - } case 'k': { key_hex_str_ = optarg; break; @@ -268,11 +229,64 @@ void ObAdminDumpsstExecutor::print_macro_block() STORAGE_LOG(ERROR, "not supported command", K(ret)); } +int ObAdminDumpsstExecutor::dump_single_macro_block(const char* buf, const int64_t size) +{ + int ret = OB_SUCCESS; + ObSSTableDataBlockReader macro_reader; + if (OB_ISNULL(buf) || size <= 0) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(ERROR, "invalid argument", K(ret), KP(buf), K(size)); + } else if (OB_FAIL(macro_reader.init(buf, size, hex_print_))) { + STORAGE_LOG(ERROR, "failed to init macro reader", K(ret), KP(buf), K(size)); + } else if (OB_FAIL(macro_reader.dump())) { + STORAGE_LOG(ERROR, "failed dump macro block", K(ret), KP(buf), K(size)); + } + return ret; +} + +int ObAdminDumpsstExecutor::dump_shared_macro_block(const char* buf, const int64_t size) +{ + int ret = OB_SUCCESS; + const int64_t aligned_size = 4096; + int64_t current_page_offset = aligned_size; + if (OB_ISNULL(buf) || size <= 0) { + ret = OB_INVALID_ARGUMENT; + STORAGE_LOG(ERROR, "invalid argument", K(ret), KP(buf), K(size)); + } else { + while (OB_SUCC(ret) && current_page_offset < size) { + ObMacroBlockCommonHeader common_header; + int64_t pos = 0; + const char* cur_buf = buf + current_page_offset; + const int64_t cur_size = size - current_page_offset; + if (OB_FAIL(common_header.deserialize(cur_buf, cur_size, pos))) { + if (OB_DESERIALIZE_ERROR != ret) { + STORAGE_LOG(ERROR, "deserialize common header fail", K(ret), K(pos)); + } else { + ret = OB_SUCCESS; + break; + } + } else if (OB_FAIL(common_header.check_integrity())) { + STORAGE_LOG(ERROR, "invalid common header", K(ret), K(common_header)); + } else if OB_FAIL(dump_single_macro_block(cur_buf, + common_header.get_header_size() + common_header.get_payload_size())) { + STORAGE_LOG(ERROR, "dump single block fail", K(ret), K(common_header)); + } else { + current_page_offset = upper_align( + current_page_offset + common_header.get_header_size() + common_header.get_payload_size(), + aligned_size); + } + } + } + STORAGE_LOG(INFO, "dump shared block finish", K(ret), K(current_page_offset)); + return ret; +} + int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext ¯o_block_context) { int ret = OB_SUCCESS; ObMacroBlockHandle macro_handle; - ObSSTableDataBlockReader macro_reader; + ObMacroBlockCommonHeader common_header; + int64_t pos = 0; ObMacroBlockReadInfo read_info; read_info.macro_block_id_.set_block_index(macro_block_context.second_id_); @@ -280,7 +294,6 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext ¯ read_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_READ); read_info.offset_ = 0; read_info.size_ = OB_DEFAULT_MACRO_BLOCK_SIZE; - STORAGE_LOG(INFO, "begin dump macro block", K(macro_block_context)); if (OB_UNLIKELY(!macro_block_context.is_valid())) { @@ -288,10 +301,18 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext ¯ STORAGE_LOG(ERROR, "invalid macro block id", K(macro_block_context), K(ret)); } else if (OB_FAIL(ObBlockManager::read_block(read_info, macro_handle))) { STORAGE_LOG(ERROR, "Fail to read macro block, ", K(ret), K(read_info)); - } else if (OB_FAIL(macro_reader.init(macro_handle.get_buffer(), macro_handle.get_data_size(), hex_print_))) { - STORAGE_LOG(ERROR, "failed to init macro reader", K(read_info), K(ret)); - } else if (OB_FAIL(macro_reader.dump())) { - STORAGE_LOG(ERROR, "failed dump macro block", K(ret)); + } else if (OB_FAIL(common_header.deserialize(macro_handle.get_buffer(), macro_handle.get_data_size(), pos))) { + STORAGE_LOG(ERROR, "deserialize common header fail", K(ret), K(pos)); + } else if (OB_FAIL(common_header.check_integrity())) { + STORAGE_LOG(ERROR, "invalid common header", K(ret), K(common_header)); + } else if (ObMacroBlockCommonHeader::SharedSSTableData == common_header.get_type()) { + if (OB_FAIL(dump_shared_macro_block(macro_handle.get_buffer(), macro_handle.get_data_size()))) { + STORAGE_LOG(ERROR, "dump shared block fail", K(ret)); + } + } else { + if (OB_FAIL(dump_single_macro_block(macro_handle.get_buffer(), macro_handle.get_data_size()))) { + STORAGE_LOG(ERROR, "dump single block fail", K(ret)); + } } macro_handle.reset(); @@ -300,89 +321,12 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext ¯ } -void ObAdminDumpsstExecutor::dump_sstable() -{ - int ret = OB_NOT_IMPLEMENT; - STORAGE_LOG(ERROR, "not supported command", K(ret)); - // adapt later if needed - /* - ObSSTable *sstable = NULL; - - if (OB_FAIL(replay_slog_to_get_sstable(sstable))) { - STORAGE_LOG(ERROR, "failed to acquire table", K_(table_key), K(ret)); - } else if (OB_ISNULL(sstable)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(ERROR, "sstable is null", K(ret)); - } else { - const ObIArray ¯o_array = sstable->get_macro_block_ids(); - ObDumpMacroBlockContext context; - for (int64_t i = 0; OB_SUCC(ret) && i < macro_array.count(); ++i) { - context.second_id_ = macro_array.at(i).block_index(); - if (OB_FAIL(dump_macro_block(context))) { - STORAGE_LOG(ERROR, "failed to dump macro block", K(context), K(ret)); - } - } - if (OB_SUCC(ret) && sstable->has_lob_macro_blocks()) { - for (int64_t i = 0; OB_SUCC(ret) && i < sstable->get_lob_macro_block_ids().count(); ++i) { - context.second_id_ = sstable->get_lob_macro_block_ids().at(i).block_index(); - if (OB_FAIL(dump_macro_block(context))) { - STORAGE_LOG(ERROR, "Failed to dump lob macro block", K(context), K(ret)); - } - } - } - } - if (OB_FAIL(ret)) { - printf("failed to dump sstable, ret = %d\n", ret); - } - */ -} - -void ObAdminDumpsstExecutor::dump_sstable_meta() -{ - int ret = OB_NOT_IMPLEMENT; - STORAGE_LOG(ERROR, "not supported command", K(ret)); - /* - int ret = OB_SUCCESS; - ObTableHandle handle; - ObSSTable *sstable = NULL; - if (OB_FAIL(ObPartitionService::get_instance().acquire_sstable(table_key_, handle))) { - STORAGE_LOG(ERROR, "fail to acquire table", K(ret), K(table_key_)); - } else if (OB_FAIL(handle.get_sstable(sstable))) { - STORAGE_LOG(ERROR, "fail to get sstable", K(ret)); - } else if (OB_ISNULL(sstable)) { - ret = OB_ERR_UNEXPECTED; - STORAGE_LOG(WARN, "error unexpected, sstable must not be NULL", K(ret)); - } else { - char buf[1024]; - const ObSSTableMeta &meta = sstable->get_meta(); - snprintf(buf, 1024, "table_id=%ld, tablet_id=%ld", table_key_.table_id_, table_key_.tablet_id_.id()); - PrintHelper::print_dump_title(buf); - PrintHelper::print_dump_line("index_id", meta.index_id_); - PrintHelper::print_dump_line("row_count", meta.row_count_); - PrintHelper::print_dump_line("occupy_size", meta.occupy_size_); - PrintHelper::print_dump_line("data_checksum", meta.data_checksum_); - PrintHelper::print_dump_line("row_checksum", meta.row_checksum_); - PrintHelper::print_dump_line("macro_block_count", meta.macro_block_count_); - PrintHelper::print_dump_line("use_old_macro_block_count", meta.use_old_macro_block_count_); - PrintHelper::print_dump_line("column_count", meta.column_cnt_); - PrintHelper::print_dump_line("schema_version", meta.schema_version_); - PrintHelper::print_dump_line("progressive_merge_start_version", meta.progressive_merge_start_version_); - PrintHelper::print_dump_line("progressive_merge_end_version", meta.progressive_merge_end_version_); - PrintHelper::print_dump_line("checksum_method", meta.checksum_method_); - meta.column_metas_.to_string(buf, 1024); - PrintHelper::print_dump_line("column_metas", buf); - meta.new_column_metas_.to_string(buf, 1024); - PrintHelper::print_dump_line("new_column_metas", buf); - } - */ -} - void ObAdminDumpsstExecutor::print_usage() { printf("\n"); printf("Usage: dumpsst command [command args] [options]\n"); printf("commands:\n"); - printf(HELP_FMT, "-d,--dump", "dump, args: [super_block|print_macro|macro_block|macro_meta|sstable|sstable_meta]"); + printf(HELP_FMT, "-d,--dump", "dump, args: [super_block|print_macro|macro_block|macro_meta]"); printf(HELP_FMT, "-h,--help", "display this message."); printf("options:\n"); @@ -391,8 +335,6 @@ void ObAdminDumpsstExecutor::print_usage() printf(HELP_FMT, "-i,--micro-id", "micro block id, -1 means all micro blocks"); printf(HELP_FMT, "-n,--macro-size", "macro block size, in bytes"); printf(HELP_FMT, "-q,--quiet", "log level: ERROR"); - printf(HELP_FMT, "-t,--table-key", "table key: table_type,table_id:partition_id,index_id,base_version:multi_version_start:snapshot_version,start_log_ts:end_log_ts:max_log_ts,major_version"); - printf(HELP_FMT, "-s,--skip_replay", "skip slog replay, only work for macro_block mode"); printf(HELP_FMT, "-x,--hex-print", "print obj value in hex mode"); printf(HELP_FMT, "-k,--master_key", "master key, hex str"); printf(HELP_FMT, "-m,--master_key_id", "master key id"); diff --git a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h index d5af7674a..efb225f43 100644 --- a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h +++ b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h @@ -31,8 +31,6 @@ enum ObAdminDumpsstCmd DUMP_SUPER_BLOCK, DUMP_MACRO_DATA, PRINT_MACRO_BLOCK, - DUMP_SSTABLE, - DUMP_SSTABLE_META, DUMP_MAX, }; @@ -64,14 +62,13 @@ private: void print_macro_meta(); void print_super_block(); int dump_macro_block(const ObDumpMacroBlockContext &context); + int dump_single_macro_block(const char* buf, const int64_t size); + int dump_shared_macro_block(const char* buf, const int64_t size); void dump_sstable(); void dump_sstable_meta(); bool is_quiet_; - bool in_csv_; ObAdminDumpsstCmd cmd_; - storage::ObITable::TableKey table_key_; - bool skip_log_replay_; bool hex_print_; ObDumpMacroBlockContext dump_macro_context_; char *key_hex_str_;