Dump shared macro block

tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp

@@ -34,9 +34,7 @@ namespace tools
 ObAdminDumpsstExecutor::ObAdminDumpsstExecutor()
     :ObAdminExecutor(),
      is_quiet_(false),
-     in_csv_(false),
      cmd_(DUMP_MAX),
-     skip_log_replay_(false),
      hex_print_(false),
      dump_macro_context_(),
      key_hex_str_(NULL),
@@ -57,10 +55,7 @@ int ObAdminDumpsstExecutor::execute(int argc, char *argv[])
     lib::set_memory_limit(96 * 1024 * 1024 * 1024LL);
     lib::set_tenant_memory_limit(500, 96 * 1024 * 1024 * 1024LL);
 
-    if (skip_log_replay_ && DUMP_MACRO_DATA != cmd_) {
+    if (OB_FAIL(ObKVGlobalCache::get_instance().init(
-      ret = OB_INVALID_ARGUMENT;
-      STORAGE_LOG(ERROR, "Only dump macro block can skip slog replay, ", K(ret));
-    } else if (OB_FAIL(ObKVGlobalCache::get_instance().init(
         &ObTenantMemLimitGetter::get_instance(), 1024L, 512 * 1024 * 1024, 64 * 1024))) {
       STORAGE_LOG(ERROR, "Fail to init kv cache, ", K(ret));
     } else if (OB_FAIL(OB_STORE_CACHE.init(
@@ -95,15 +90,9 @@ int ObAdminDumpsstExecutor::execute(int argc, char *argv[])
         case DUMP_MACRO_DATA:
           dump_macro_block(dump_macro_context_);
           break;
-        case DUMP_SSTABLE:
-          dump_sstable();
-          break;
         case PRINT_MACRO_BLOCK:
           print_macro_block();
           break;
-        case DUMP_SSTABLE_META:
-          dump_sstable_meta();
-          break;
         default:
           print_usage();
           exit(1);
@@ -148,7 +137,7 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
   int ret = OB_SUCCESS;
 
   int opt = 0;
-  const char* opt_string = "d:hf:a:i:n:qt:sxk:m:";
+  const char* opt_string = "d:hf:a:i:n:qxk:m:";
 
   struct option longopts[] = {
     // commands
@@ -159,10 +148,7 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
     { "macro-id", 1, NULL, 'a' },
     { "micro-id", 1, NULL, 'i' },
     { "macro-size", 1, NULL, 'n' },
-    { "csv", 0, NULL, 0 },
-    { "table_key", 1, NULL, 't'},
     { "quiet", 0, NULL, 'q' },
-    { "skip_replay", 0, NULL, 's' },
     { "hex-print", 0, NULL, 'x' },
     { "master_key", 1, NULL, 'k'},
     { "master_key_id", 1, NULL, 'm'}
@@ -180,10 +166,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
         cmd_ = DUMP_SUPER_BLOCK;
       } else if (0 == strcmp(optarg, "mb") || 0 == strcmp(optarg, "macro_block")) {
         cmd_ = DUMP_MACRO_DATA;
-      } else if (0 == strcmp(optarg, "sst") || 0 == strcmp(optarg, "sstable")) {
-        cmd_ = DUMP_SSTABLE;
-      } else if (0 == strcmp(optarg, "sstm") || 0 == strcmp(optarg, "sstable_meta")) {
-        cmd_ = DUMP_SSTABLE_META;
       } else if (0 == strcmp(optarg, "pm") || 0 == strcmp(optarg, "print_macro")) {
         cmd_ = PRINT_MACRO_BLOCK;
       } else {
@@ -215,10 +197,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
       is_quiet_ = true;
       break;
     }
-    case 's': {
-      skip_log_replay_ = true;
-      break;
-    }
     case 'x': {
       hex_print_ = true;
       break;
@@ -227,23 +205,6 @@ int ObAdminDumpsstExecutor::parse_cmd(int argc, char *argv[])
       master_key_id_ = strtoll(optarg, NULL, 10);
       break;
     }
-    case 0: {
-      std::string name = longopts[index].name;
-      if ("csv" == name) {
-        in_csv_ = true;
-      }
-      break;
-    }
-    case 't': {
-      // if (OB_FAIL(parse_table_key(optarg, table_key_))) {
-      //   printf("failed to parse table key\n");
-      //   print_usage();
-      //   exit(1);
-      // }
-      // break;
-      print_usage();
-      exit(1);
-    }
     case 'k': {
       key_hex_str_ = optarg;
       break;
@@ -268,11 +229,64 @@ void ObAdminDumpsstExecutor::print_macro_block()
   STORAGE_LOG(ERROR, "not supported command", K(ret));
 }
 
+int ObAdminDumpsstExecutor::dump_single_macro_block(const char* buf, const int64_t size)
+{
+  int ret = OB_SUCCESS;
+  ObSSTableDataBlockReader macro_reader;
+  if (OB_ISNULL(buf) || size <= 0) {
+    ret = OB_INVALID_ARGUMENT;
+    STORAGE_LOG(ERROR, "invalid argument", K(ret), KP(buf), K(size));
+  } else if (OB_FAIL(macro_reader.init(buf, size, hex_print_))) {
+    STORAGE_LOG(ERROR, "failed to init macro reader", K(ret), KP(buf), K(size));
+  } else if (OB_FAIL(macro_reader.dump())) {
+    STORAGE_LOG(ERROR, "failed dump macro block", K(ret), KP(buf), K(size));
+  }
+  return ret;
+}
+
+int ObAdminDumpsstExecutor::dump_shared_macro_block(const char* buf, const int64_t size)
+{
+  int ret = OB_SUCCESS;
+  const int64_t aligned_size = 4096;
+  int64_t current_page_offset = aligned_size;
+  if (OB_ISNULL(buf) || size <= 0) {
+    ret = OB_INVALID_ARGUMENT;
+    STORAGE_LOG(ERROR, "invalid argument", K(ret), KP(buf), K(size));
+  } else {
+    while (OB_SUCC(ret) && current_page_offset < size) {
+      ObMacroBlockCommonHeader common_header;
+      int64_t pos = 0;
+      const char* cur_buf = buf + current_page_offset;
+      const int64_t cur_size = size - current_page_offset;
+      if (OB_FAIL(common_header.deserialize(cur_buf, cur_size, pos))) {
+        if (OB_DESERIALIZE_ERROR != ret) {
+          STORAGE_LOG(ERROR, "deserialize common header fail", K(ret), K(pos));
+        } else {
+          ret = OB_SUCCESS;
+          break;
+        }
+      } else if (OB_FAIL(common_header.check_integrity())) {
+        STORAGE_LOG(ERROR, "invalid common header", K(ret), K(common_header));
+      } else if OB_FAIL(dump_single_macro_block(cur_buf,
+          common_header.get_header_size() + common_header.get_payload_size())) {
+        STORAGE_LOG(ERROR, "dump single block fail", K(ret), K(common_header));
+      } else {
+        current_page_offset = upper_align(
+            current_page_offset + common_header.get_header_size() + common_header.get_payload_size(),
+            aligned_size);
+      }
+    }
+  }
+  STORAGE_LOG(INFO, "dump shared block finish", K(ret), K(current_page_offset));
+  return ret;
+}
+
 int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macro_block_context)
 {
   int ret = OB_SUCCESS;
   ObMacroBlockHandle macro_handle;
-  ObSSTableDataBlockReader macro_reader;
+  ObMacroBlockCommonHeader common_header;
+  int64_t pos = 0;
 
   ObMacroBlockReadInfo read_info;
   read_info.macro_block_id_.set_block_index(macro_block_context.second_id_);
@@ -280,7 +294,6 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macr
   read_info.io_desc_.set_wait_event(ObWaitEventIds::DB_FILE_COMPACT_READ);
   read_info.offset_ = 0;
   read_info.size_ = OB_DEFAULT_MACRO_BLOCK_SIZE;
-  
 
   STORAGE_LOG(INFO, "begin dump macro block", K(macro_block_context));
   if (OB_UNLIKELY(!macro_block_context.is_valid())) {
@@ -288,10 +301,18 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macr
     STORAGE_LOG(ERROR, "invalid macro block id", K(macro_block_context), K(ret));
   } else if (OB_FAIL(ObBlockManager::read_block(read_info, macro_handle))) {
     STORAGE_LOG(ERROR, "Fail to read macro block, ", K(ret), K(read_info));
-  } else if (OB_FAIL(macro_reader.init(macro_handle.get_buffer(), macro_handle.get_data_size(), hex_print_))) {
+  } else if (OB_FAIL(common_header.deserialize(macro_handle.get_buffer(), macro_handle.get_data_size(), pos))) {
-    STORAGE_LOG(ERROR, "failed to init macro reader", K(read_info), K(ret));
+    STORAGE_LOG(ERROR, "deserialize common header fail", K(ret), K(pos));
-  } else if (OB_FAIL(macro_reader.dump())) {
+  } else if (OB_FAIL(common_header.check_integrity())) {
-    STORAGE_LOG(ERROR, "failed dump macro block", K(ret));
+    STORAGE_LOG(ERROR, "invalid common header", K(ret), K(common_header));
+  } else if (ObMacroBlockCommonHeader::SharedSSTableData == common_header.get_type()) {
+    if (OB_FAIL(dump_shared_macro_block(macro_handle.get_buffer(), macro_handle.get_data_size()))) {
+      STORAGE_LOG(ERROR, "dump shared block fail", K(ret));
+    }
+  } else {
+    if (OB_FAIL(dump_single_macro_block(macro_handle.get_buffer(), macro_handle.get_data_size()))) {
+      STORAGE_LOG(ERROR, "dump single block fail", K(ret));
+    }
   }
 
   macro_handle.reset();
@@ -300,89 +321,12 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext &macr
 }
 
 
-void ObAdminDumpsstExecutor::dump_sstable()
-{
-  int ret = OB_NOT_IMPLEMENT;
-  STORAGE_LOG(ERROR, "not supported command", K(ret));
-  // adapt later if needed
-  /*
-  ObSSTable *sstable = NULL;
-
-  if (OB_FAIL(replay_slog_to_get_sstable(sstable))) {
-    STORAGE_LOG(ERROR, "failed to acquire table", K_(table_key), K(ret));
-  } else if (OB_ISNULL(sstable)) {
-    ret = OB_ERR_UNEXPECTED;
-    STORAGE_LOG(ERROR, "sstable is null", K(ret));
-  } else {
-    const ObIArray<blocksstable::MacroBlockId> &macro_array = sstable->get_macro_block_ids();
-    ObDumpMacroBlockContext context;
-    for (int64_t i = 0; OB_SUCC(ret) && i < macro_array.count(); ++i) {
-      context.second_id_ = macro_array.at(i).block_index();
-      if (OB_FAIL(dump_macro_block(context))) {
-        STORAGE_LOG(ERROR, "failed to dump macro block", K(context), K(ret));
-      }
-    }
-    if (OB_SUCC(ret) && sstable->has_lob_macro_blocks()) {
-      for (int64_t i = 0; OB_SUCC(ret) && i < sstable->get_lob_macro_block_ids().count(); ++i) {
-        context.second_id_ = sstable->get_lob_macro_block_ids().at(i).block_index();
-        if (OB_FAIL(dump_macro_block(context))) {
-          STORAGE_LOG(ERROR, "Failed to dump lob macro block", K(context), K(ret));
-        }
-      }
-    }
-  }
-  if (OB_FAIL(ret)) {
-    printf("failed to dump sstable, ret = %d\n", ret);
-  }
-  */
-}
-
-void ObAdminDumpsstExecutor::dump_sstable_meta()
-{
-  int ret = OB_NOT_IMPLEMENT;
-  STORAGE_LOG(ERROR, "not supported command", K(ret));
-  /*
-  int ret = OB_SUCCESS;
-  ObTableHandle handle;
-  ObSSTable *sstable = NULL;
-  if (OB_FAIL(ObPartitionService::get_instance().acquire_sstable(table_key_, handle))) {
-    STORAGE_LOG(ERROR, "fail to acquire table", K(ret), K(table_key_));
-  } else if (OB_FAIL(handle.get_sstable(sstable))) {
-    STORAGE_LOG(ERROR, "fail to get sstable", K(ret));
-  } else if (OB_ISNULL(sstable)) {
-    ret = OB_ERR_UNEXPECTED;
-    STORAGE_LOG(WARN, "error unexpected, sstable must not be NULL", K(ret));
-  } else {
-    char buf[1024];
-    const ObSSTableMeta &meta = sstable->get_meta();
-    snprintf(buf, 1024, "table_id=%ld, tablet_id=%ld", table_key_.table_id_, table_key_.tablet_id_.id());
-    PrintHelper::print_dump_title(buf);
-    PrintHelper::print_dump_line("index_id", meta.index_id_);
-    PrintHelper::print_dump_line("row_count", meta.row_count_);
-    PrintHelper::print_dump_line("occupy_size", meta.occupy_size_);
-    PrintHelper::print_dump_line("data_checksum", meta.data_checksum_);
-    PrintHelper::print_dump_line("row_checksum", meta.row_checksum_);
-    PrintHelper::print_dump_line("macro_block_count", meta.macro_block_count_);
-    PrintHelper::print_dump_line("use_old_macro_block_count", meta.use_old_macro_block_count_);
-    PrintHelper::print_dump_line("column_count", meta.column_cnt_);
-    PrintHelper::print_dump_line("schema_version", meta.schema_version_);
-    PrintHelper::print_dump_line("progressive_merge_start_version", meta.progressive_merge_start_version_);
-    PrintHelper::print_dump_line("progressive_merge_end_version", meta.progressive_merge_end_version_);
-    PrintHelper::print_dump_line("checksum_method", meta.checksum_method_);
-    meta.column_metas_.to_string(buf, 1024);
-    PrintHelper::print_dump_line("column_metas", buf);
-    meta.new_column_metas_.to_string(buf, 1024);
-    PrintHelper::print_dump_line("new_column_metas", buf);
-  }
-  */
-}
-
 void ObAdminDumpsstExecutor::print_usage()
 {
   printf("\n");
   printf("Usage: dumpsst command [command args] [options]\n");
   printf("commands:\n");
-  printf(HELP_FMT, "-d,--dump", "dump, args: [super_block|print_macro|macro_block|macro_meta|sstable|sstable_meta]");
+  printf(HELP_FMT, "-d,--dump", "dump, args: [super_block|print_macro|macro_block|macro_meta]");
   printf(HELP_FMT, "-h,--help", "display this message.");
 
   printf("options:\n");
@@ -391,8 +335,6 @@ void ObAdminDumpsstExecutor::print_usage()
   printf(HELP_FMT, "-i,--micro-id", "micro block id, -1 means all micro blocks");
   printf(HELP_FMT, "-n,--macro-size", "macro block size, in bytes");
   printf(HELP_FMT, "-q,--quiet", "log level: ERROR");
-  printf(HELP_FMT, "-t,--table-key", "table key: table_type,table_id:partition_id,index_id,base_version:multi_version_start:snapshot_version,start_log_ts:end_log_ts:max_log_ts,major_version");
-  printf(HELP_FMT, "-s,--skip_replay", "skip slog replay, only work for macro_block mode");
   printf(HELP_FMT, "-x,--hex-print", "print obj value in hex mode");
   printf(HELP_FMT, "-k,--master_key", "master key, hex str");
   printf(HELP_FMT, "-m,--master_key_id", "master key id");

tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h

@@ -31,8 +31,6 @@ enum ObAdminDumpsstCmd
   DUMP_SUPER_BLOCK,
   DUMP_MACRO_DATA,
   PRINT_MACRO_BLOCK,
-  DUMP_SSTABLE,
-  DUMP_SSTABLE_META,
   DUMP_MAX,
 };
 
@@ -64,14 +62,13 @@ private:
   void print_macro_meta();
   void print_super_block();
   int dump_macro_block(const ObDumpMacroBlockContext &context);
+  int dump_single_macro_block(const char* buf, const int64_t size);
+  int dump_shared_macro_block(const char* buf, const int64_t size);
   void dump_sstable();
   void dump_sstable_meta();
 
   bool is_quiet_;
-  bool in_csv_;
   ObAdminDumpsstCmd cmd_;
-  storage::ObITable::TableKey table_key_;
-  bool skip_log_replay_;
   bool hex_print_;
   ObDumpMacroBlockContext dump_macro_context_;
   char *key_hex_str_;