virtual table adaptation

src/observer/virtual_table/ob_all_virtual_table_mgr.cpp

@@ -205,6 +205,8 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row)
     ret = OB_ERR_UNEXPECTED;
     SERVER_LOG(WARN, "table shouldn't NULL here", K(ret), K(table));
   } else {
+    const int64_t nested_offset = table->is_sstable() ? static_cast<ObSSTable *>(table)->get_macro_offset() : 0;
+    const int64_t nested_size = table->is_sstable() ? static_cast<ObSSTable *>(table)->get_macro_read_size() : 0;
     const ObITable::TableKey &table_key = table->get_key();
     const int64_t col_count = output_column_ids_.count();
     for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) {
@@ -301,6 +303,12 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row)
           cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
           break;
         }
+        case NESTED_OFFSET:
+          cur_row_.cells_[i].set_int(nested_offset);
+          break;
+        case NESTED_SIZE:
+          cur_row_.cells_[i].set_int(nested_size);
+          break;
         default:
           ret = OB_ERR_UNEXPECTED;
           SERVER_LOG(WARN, "invalid col_id", K(ret), K(col_id));

src/observer/virtual_table/ob_all_virtual_table_mgr.h

@@ -52,7 +52,9 @@ class ObAllVirtualTableMgr : public common::ObVirtualTableScannerIterator,
     LINKED_BLOCK_CNT,
     REF,
     IS_ACTIVE,
-    CONTAIN_UNCOMMITTED_ROW
+    CONTAIN_UNCOMMITTED_ROW,
+    NESTED_OFFSET,
+    NESTED_SIZE
   };
 public:
   ObAllVirtualTableMgr();

src/share/inner_table/ob_inner_table_schema.12001_12050.cpp

@@ -2865,6 +2865,36 @@ int ObInnerTableSchema::all_virtual_table_mgr_schema(ObTableSchema &table_schema
       false, //is_nullable
       false); //is_autoincrement
   }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("nested_offset", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("nested_size", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObIntType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      sizeof(int64_t), //column_length
+      -1, //column_precision
+      -1, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
   if (OB_SUCC(ret)) {
     table_schema.get_part_option().set_part_num(1);
     table_schema.set_part_level(PARTITION_LEVEL_ONE);

src/share/inner_table/ob_inner_table_schema.15051_15100.cpp

@@ -3312,6 +3312,36 @@ int ObInnerTableSchema::all_virtual_table_mgr_ora_schema(ObTableSchema &table_sc
       false, //is_nullable
       false); //is_autoincrement
   }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("NESTED_OFFSET", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
+
+  if (OB_SUCC(ret)) {
+    ADD_COLUMN_SCHEMA("NESTED_SIZE", //column_name
+      ++column_id, //column_id
+      0, //rowkey_id
+      0, //index_id
+      0, //part_key_pos
+      ObNumberType, //column_type
+      CS_TYPE_INVALID, //column_collation_type
+      38, //column_length
+      38, //column_precision
+      0, //column_scale
+      false, //is_nullable
+      false); //is_autoincrement
+  }
   if (OB_SUCC(ret)) {
     table_schema.get_part_option().set_part_num(1);
     table_schema.set_part_level(PARTITION_LEVEL_ONE);

src/share/inner_table/ob_inner_table_schema_def.py

@@ -8435,6 +8435,8 @@ def_table_schema(
       ('ref', 'int'),
       ('is_active', 'varchar:MAX_COLUMN_YES_NO_LENGTH'),
       ('contain_uncommitted_row', 'varchar:MAX_COLUMN_YES_NO_LENGTH'),
+      ('nested_offset', 'int'),
+      ('nested_size', 'int'),
     ],
   partition_columns = ['svr_ip', 'svr_port'],
   vtable_route_policy = 'distributed',

src/storage/blocksstable/ob_shared_macro_block_manager.cpp

@@ -296,6 +296,16 @@ int ObSharedMacroBlockMgr::try_switch_macro_block()
   return ret;
 }
 
+int64_t ObSharedMacroBlockMgr::get_shared_block_cnt()
+{
+  int64_t count = 0;
+  {
+    lib::ObMutexGuard guard(blocks_mutex_);
+    count = block_used_size_.count();
+  }
+  return count;
+}
+
 int ObSharedMacroBlockMgr::add_block(const MacroBlockId &block_id, const int64_t block_size)
 {
   int ret = OB_SUCCESS;
@@ -407,7 +417,9 @@ int ObSharedMacroBlockMgr::defragment()
       tablet_handle.reset();
       iter_allocator.reuse();
       if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) {
+        if (OB_UNLIKELY(OB_ITER_END != ret)) {
           LOG_WARN("fail to get tablet", K(ret), K(tablet_handle));
+        }
       } else if (OB_UNLIKELY(!tablet_handle.is_valid())) {
         ret = OB_ERR_UNEXPECTED;
         LOG_WARN("invalid tablet handle", K(ret), K(tablet_handle));

src/storage/blocksstable/ob_shared_macro_block_manager.h

@@ -62,6 +62,7 @@ public:
   int start();
   void stop();
   void wait();
+  int64_t get_shared_block_cnt();
   int write_block(const char* buf, const int64_t size, ObBlockInfo &block_info, ObMacroBlocksWriteCtx &write_ctx);
   int add_block(const MacroBlockId &block_id, const int64_t block_size);
   int free_block(const MacroBlockId &block_id, const int64_t block_size);

src/storage/ob_disk_usage_reporter.cpp

@@ -25,6 +25,7 @@
 #include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
 #include "lib/function/ob_function.h"
 #include "logservice/ob_server_log_block_mgr.h"
+#include "storage/blocksstable/ob_shared_macro_block_manager.h"
 
 namespace oceanbase
 {
@@ -219,7 +220,7 @@ int ObDiskUsageReportTask::count_tenant_data(const uint64_t tenant_id)
       if (OB_UNLIKELY(!tablet_handle.is_valid())) {
         ret = OB_ERR_UNEXPECTED;
         STORAGE_LOG(WARN, "unexpected invalid tablet", K(ret), K(tablet_handle));
-      } else if (OB_FAIL(tablet_handle.get_obj()->get_sstables_size(sstable_size))) {
+      } else if (OB_FAIL(tablet_handle.get_obj()->get_sstables_size(sstable_size, true /*ignore shared block*/))) {
         STORAGE_LOG(WARN, "failed to get new tablet's disk usage", K(ret), K(sstable_size));
       } else {
         data_size += sstable_size;
@@ -228,6 +229,7 @@ int ObDiskUsageReportTask::count_tenant_data(const uint64_t tenant_id)
     }
     if (OB_ITER_END == ret || OB_SUCCESS == ret) {
       ret = OB_SUCCESS;
+      data_size += MTL(ObSharedMacroBlockMgr*)->get_shared_block_cnt() * OB_DEFAULT_MACRO_BLOCK_SIZE;
       report_key.tenant_id_ = tenant_id;
       report_key.file_type_ = ObDiskReportFileType::OB_DISK_REPORT_TENANT_DATA;
       if (OB_FAIL(result_map_.set_refactored(report_key, data_size, 1))) {

src/storage/slog/ob_storage_log_writer.cpp

@@ -565,7 +565,7 @@ int ObStorageLogWriter::advance_file_id()
     write_offset_ = 0;
     cursor_.offset_ = write_offset_;
     batch_write_buf_.reuse();
-    STORAGE_REDO_LOG(WARN, "Successfully open slog file", K(cursor_.file_id_));
+    STORAGE_REDO_LOG(INFO, "Successfully open slog file", K(cursor_.file_id_));
   }
 
   return ret;

src/storage/tablet/ob_tablet.cpp

@@ -1207,13 +1207,11 @@ int ObTablet::inner_get_all_sstables(common::ObIArray<ObITable *> &sstables) con
   return ret;
 }
 
-int ObTablet::get_sstables_size(int64_t &used_size) const
+int ObTablet::get_sstables_size(int64_t &used_size, const bool ignore_shared_block) const
 {
   int ret = OB_SUCCESS;
   common::ObSArray<ObITable *> sstables;
   bool multi_version = false;
-  int64_t mem_block_cnt = 0;
-
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
     LOG_WARN("not inited", K(ret), K_(is_inited));
@@ -1227,23 +1225,21 @@ int ObTablet::get_sstables_size(int64_t &used_size) const
         ret = OB_ERR_UNEXPECTED;
         LOG_WARN("sstable is null", K(ret), K(*this), K(i));
       } else if (FALSE_IT(sstable = static_cast<ObSSTable *> (table))) {
+      } else if (sstable->is_small_sstable() && ignore_shared_block) {
+        // skip small sstables
       } else {
         const ObSSTableBasicMeta &basic_meta = sstable->get_meta().get_basic_meta();
         if (multi_version && sstable->is_major_sstable()) {
-          mem_block_cnt -= basic_meta.get_total_use_old_macro_block_count();
+          used_size -= basic_meta.get_total_use_old_macro_block_count() * sstable->get_macro_read_size();
         } else if (sstable->is_major_sstable()) {
           multi_version = true;
         }
-        mem_block_cnt += basic_meta.get_total_macro_block_count();
+        used_size += basic_meta.get_total_macro_block_count() * sstable->get_macro_read_size();
       }
     }
-    if (OB_FAIL(ret)) {
+    if (OB_SUCC(ret) && tablet_meta_.has_next_tablet_ && OB_FAIL(
-      // do nothing
+        next_tablet_guard_.get_obj()->get_sstables_size(used_size, true /*ignore shared block*/))) {
-    } else if (tablet_meta_.has_next_tablet_ && OB_FAIL(
-        next_tablet_guard_.get_obj()->get_sstables_size(used_size))) {
       LOG_WARN("failed to get size of tablets on the list", K(ret), K(used_size));
-    } else {
-      used_size += mem_block_cnt * common::OB_DEFAULT_MACRO_BLOCK_SIZE;
     }
   }
   return ret;

src/storage/tablet/ob_tablet.h

@@ -221,7 +221,7 @@ public:
       const int64_t &major_snapshot_version,
       ObTabletTableIterator &iter);
   int get_all_sstables(common::ObIArray<ObITable *> &sstables) const;
-  int get_sstables_size(int64_t &used_size) const;
+  int get_sstables_size(int64_t &used_size, const bool ignore_shared_block = false) const;
   int get_memtables(common::ObIArray<storage::ObITable *> &memtables, const bool need_active = false) const;
   int get_ddl_memtables(common::ObIArray<ObITable *> &ddl_memtables) const;
   int check_need_remove_old_table(const int64_t multi_version_start, bool &need_remove) const;