hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

virtual table adaptation

Browse Source
This commit is contained in:
obdev
2023-02-21 10:12:17 +00:00
committed by ob-robot
parent 279bed6adf
commit c28b303f46
25 changed files with 170 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/observer/virtual_table/ob_all_virtual_table_mgr.cpp
View File

@ -205,6 +205,8 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row)
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "table shouldn't NULL here", K(ret), K(table));
} else {
const int64_t nested_offset = table->is_sstable() ? static_cast<ObSSTable *>(table)->get_macro_offset() : 0;
const int64_t nested_size = table->is_sstable() ? static_cast<ObSSTable *>(table)->get_macro_read_size() : 0;
const ObITable::TableKey &table_key = table->get_key();
const int64_t col_count = output_column_ids_.count();
for (int64_t i = 0; OB_SUCC(ret) && i < col_count; ++i) {
@ -301,6 +303,12 @@ int ObAllVirtualTableMgr::process_curr_tenant(common::ObNewRow *&row)
cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
break;
}
case NESTED_OFFSET:
cur_row_.cells_[i].set_int(nested_offset);
break;
case NESTED_SIZE:
cur_row_.cells_[i].set_int(nested_size);
break;
default:
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "invalid col_id", K(ret), K(col_id));

4
src/observer/virtual_table/ob_all_virtual_table_mgr.h
View File

@ -52,7 +52,9 @@ class ObAllVirtualTableMgr : public common::ObVirtualTableScannerIterator,
LINKED_BLOCK_CNT,
REF,
IS_ACTIVE,
CONTAIN_UNCOMMITTED_ROW
CONTAIN_UNCOMMITTED_ROW,
NESTED_OFFSET,
NESTED_SIZE
};
public:
ObAllVirtualTableMgr();

30
src/share/inner_table/ob_inner_table_schema.12001_12050.cpp
View File

@ -2865,6 +2865,36 @@ int ObInnerTableSchema::all_virtual_table_mgr_schema(ObTableSchema &table_schema
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("nested_offset", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("nested_size", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
table_schema.get_part_option().set_part_num(1);
table_schema.set_part_level(PARTITION_LEVEL_ONE);

30
src/share/inner_table/ob_inner_table_schema.15051_15100.cpp
View File

@ -3312,6 +3312,36 @@ int ObInnerTableSchema::all_virtual_table_mgr_ora_schema(ObTableSchema &table_sc
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("NESTED_OFFSET", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("NESTED_SIZE", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
table_schema.get_part_option().set_part_num(1);
table_schema.set_part_level(PARTITION_LEVEL_ONE);

2
src/share/inner_table/ob_inner_table_schema_def.py
View File

@ -8435,6 +8435,8 @@ def_table_schema(
('ref', 'int'),
('is_active', 'varchar:MAX_COLUMN_YES_NO_LENGTH'),
('contain_uncommitted_row', 'varchar:MAX_COLUMN_YES_NO_LENGTH'),
('nested_offset', 'int'),
('nested_size', 'int'),
],
partition_columns = ['svr_ip', 'svr_port'],
vtable_route_policy = 'distributed',

12
src/storage/blocksstable/ob_shared_macro_block_manager.cpp
View File

@ -296,6 +296,16 @@ int ObSharedMacroBlockMgr::try_switch_macro_block()
return ret;
}
int64_t ObSharedMacroBlockMgr::get_shared_block_cnt()
{
int64_t count = 0;
{
lib::ObMutexGuard guard(blocks_mutex_);
count = block_used_size_.count();
}
return count;
}
int ObSharedMacroBlockMgr::add_block(const MacroBlockId &block_id, const int64_t block_size)
{
int ret = OB_SUCCESS;
@ -407,7 +417,9 @@ int ObSharedMacroBlockMgr::defragment()
tablet_handle.reset();
iter_allocator.reuse();
if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("fail to get tablet", K(ret), K(tablet_handle));
}
} else if (OB_UNLIKELY(!tablet_handle.is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid tablet handle", K(ret), K(tablet_handle));

1
src/storage/blocksstable/ob_shared_macro_block_manager.h
View File

@ -62,6 +62,7 @@ public:
int start();
void stop();
void wait();
int64_t get_shared_block_cnt();
int write_block(const char* buf, const int64_t size, ObBlockInfo &block_info, ObMacroBlocksWriteCtx &write_ctx);
int add_block(const MacroBlockId &block_id, const int64_t block_size);
int free_block(const MacroBlockId &block_id, const int64_t block_size);

4
src/storage/ob_disk_usage_reporter.cpp
View File

@ -25,6 +25,7 @@
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
#include "lib/function/ob_function.h"
#include "logservice/ob_server_log_block_mgr.h"
#include "storage/blocksstable/ob_shared_macro_block_manager.h"
namespace oceanbase
{
@ -219,7 +220,7 @@ int ObDiskUsageReportTask::count_tenant_data(const uint64_t tenant_id)
if (OB_UNLIKELY(!tablet_handle.is_valid())) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "unexpected invalid tablet", K(ret), K(tablet_handle));
} else if (OB_FAIL(tablet_handle.get_obj()->get_sstables_size(sstable_size))) {
} else if (OB_FAIL(tablet_handle.get_obj()->get_sstables_size(sstable_size, true /*ignore shared block*/))) {
STORAGE_LOG(WARN, "failed to get new tablet's disk usage", K(ret), K(sstable_size));
} else {
data_size += sstable_size;
@ -228,6 +229,7 @@ int ObDiskUsageReportTask::count_tenant_data(const uint64_t tenant_id)
}
if (OB_ITER_END == ret || OB_SUCCESS == ret) {
ret = OB_SUCCESS;
data_size += MTL(ObSharedMacroBlockMgr*)->get_shared_block_cnt() * OB_DEFAULT_MACRO_BLOCK_SIZE;
report_key.tenant_id_ = tenant_id;
report_key.file_type_ = ObDiskReportFileType::OB_DISK_REPORT_TENANT_DATA;
if (OB_FAIL(result_map_.set_refactored(report_key, data_size, 1))) {

2
src/storage/slog/ob_storage_log_writer.cpp
View File

@ -565,7 +565,7 @@ int ObStorageLogWriter::advance_file_id()
write_offset_ = 0;
cursor_.offset_ = write_offset_;
batch_write_buf_.reuse();
STORAGE_REDO_LOG(WARN, "Successfully open slog file", K(cursor_.file_id_));
STORAGE_REDO_LOG(INFO, "Successfully open slog file", K(cursor_.file_id_));
}
return ret;

18
src/storage/tablet/ob_tablet.cpp
View File

@ -1207,13 +1207,11 @@ int ObTablet::inner_get_all_sstables(common::ObIArray<ObITable *> &sstables) con
return ret;
}
int ObTablet::get_sstables_size(int64_t &used_size) const
int ObTablet::get_sstables_size(int64_t &used_size, const bool ignore_shared_block) const
{
int ret = OB_SUCCESS;
common::ObSArray<ObITable *> sstables;
bool multi_version = false;
int64_t mem_block_cnt = 0;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not inited", K(ret), K_(is_inited));
@ -1227,23 +1225,21 @@ int ObTablet::get_sstables_size(int64_t &used_size) const
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sstable is null", K(ret), K(*this), K(i));
} else if (FALSE_IT(sstable = static_cast<ObSSTable *> (table))) {
} else if (sstable->is_small_sstable() && ignore_shared_block) {
// skip small sstables
} else {
const ObSSTableBasicMeta &basic_meta = sstable->get_meta().get_basic_meta();
if (multi_version && sstable->is_major_sstable()) {
mem_block_cnt -= basic_meta.get_total_use_old_macro_block_count();
used_size -= basic_meta.get_total_use_old_macro_block_count() * sstable->get_macro_read_size();
} else if (sstable->is_major_sstable()) {
multi_version = true;
}
mem_block_cnt += basic_meta.get_total_macro_block_count();
used_size += basic_meta.get_total_macro_block_count() * sstable->get_macro_read_size();
}
}
if (OB_FAIL(ret)) {
// do nothing
} else if (tablet_meta_.has_next_tablet_ && OB_FAIL(
next_tablet_guard_.get_obj()->get_sstables_size(used_size))) {
if (OB_SUCC(ret) && tablet_meta_.has_next_tablet_ && OB_FAIL(
next_tablet_guard_.get_obj()->get_sstables_size(used_size, true /*ignore shared block*/))) {
LOG_WARN("failed to get size of tablets on the list", K(ret), K(used_size));
} else {
used_size += mem_block_cnt * common::OB_DEFAULT_MACRO_BLOCK_SIZE;
}
}
return ret;

2
src/storage/tablet/ob_tablet.h
View File

@ -221,7 +221,7 @@ public:
const int64_t &major_snapshot_version,
ObTabletTableIterator &iter);
int get_all_sstables(common::ObIArray<ObITable *> &sstables) const;
int get_sstables_size(int64_t &used_size) const;
int get_sstables_size(int64_t &used_size, const bool ignore_shared_block = false) const;
int get_memtables(common::ObIArray<storage::ObITable *> &memtables, const bool need_active = false) const;
int get_ddl_memtables(common::ObIArray<ObITable *> &ddl_memtables) const;
int check_need_remove_old_table(const int64_t multi_version_start, bool &need_remove) const;