hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add stat column in virtual table for tmp file writing and reading

Browse Source
This commit is contained in:
wanyue-wy 2024-12-06 09:15:25 +00:00 committed by ob-robot
parent 5358c920cf
commit f904ff0ede
15 changed files with 870 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/objit/src/ob_llvm_di_helper.cpp
View File

@ -323,9 +323,9 @@ int ObLLVMDIHelper::create_struct_type(
SmallVector<Metadata *, 8> element_types;
for (int i = 0; OB_SUCC(ret) && i < member_types.count(); i++) {
if (OB_ISNULL(member_types.at(i).get_v())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("member type is NULL", K(ret), K(i), K(member_types.count()));
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("member type is NULL", K(ret), K(i), K(member_types.count()));
} else {
element_types.push_back(static_cast<ObDIType *>(member_types.at(i).get_v()));
}
}

77
src/observer/virtual_table/ob_all_virtual_tmp_file.cpp
View File

@ -202,6 +202,53 @@ int ObAllVirtualTmpFileInfo::fill_columns_(tmp_file::ObTmpFileInfo *tmp_file_inf
cur_row_.cells_[i].set_varchar(file_label_buffer_);
cur_row_.cells_[i].set_default_collation_type();
break;
case TYPE:
break;
case COMPRESSIBLE_FD:
break;
case PERSISTED_TAIL_PAGE_WRITES:
cur_row_.cells_[i].set_int(tmp_file_info->write_persisted_tail_page_cnt_);
break;
case LACK_PAGE_CNT:
cur_row_.cells_[i].set_int(tmp_file_info->lack_page_cnt_);
break;
case TOTAL_TRUNCATED_PAGE_READ_CNT:
cur_row_.cells_[i].set_int(tmp_file_info->total_truncated_page_read_cnt_);
break;
case TRUNCATED_PAGE_HITS:
cur_row_.cells_[i].set_int(tmp_file_info->truncated_page_read_hits_);
break;
case TOTAL_KV_CACHE_PAGE_READ_CNT:
cur_row_.cells_[i].set_int(tmp_file_info->total_kv_cache_page_read_cnt_);
break;
case KV_CACHE_PAGE_HITS:
cur_row_.cells_[i].set_int(tmp_file_info->kv_cache_page_read_hits_);
break;
case TOTAL_UNCACHED_PAGE_READ_CNT:
cur_row_.cells_[i].set_int(tmp_file_info->total_uncached_page_read_cnt_);
break;
case UNCACHED_PAGE_HITS:
cur_row_.cells_[i].set_int(tmp_file_info->uncached_page_read_hits_);
break;
case TOTAL_WBP_PAGE_READ_CNT:
cur_row_.cells_[i].set_int(tmp_file_info->total_wbp_page_read_cnt_);
break;
case WBP_PAGE_HITS:
cur_row_.cells_[i].set_int(tmp_file_info->wbp_page_read_hits_);
break;
#ifdef OB_BUILD_SHARED_STORAGE
/* columns in ss modes begin */
case AGGREGATE_READ_IO_CNT:
if (GCTX.is_shared_storage_mode()) {
tmp_file::ObSSTmpFileInfo *ss_tmp_file_info = static_cast<tmp_file::ObSSTmpFileInfo *>(tmp_file_info);
if (OB_FAIL(fill_ss_column_(i, ss_tmp_file_info))) {
SERVER_LOG(WARN, "fail to fill ss column", KR(ret), K(i), KPC(ss_tmp_file_info));
}
}
break;
/* columns in ss modes end */
#endif
/* columns in sn modes begin */
case META_TREE_EPOCH:
case META_TREE_LEVELS:
case META_BYTES:
@ -215,10 +262,7 @@ int ObAllVirtualTmpFileInfo::fill_columns_(tmp_file::ObTmpFileInfo *tmp_file_inf
}
}
break;
case TYPE:
break;
case COMPRESSIBLE_FD:
break;
/* columns in sn modes end */
default:
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "invalid column_id", KR(ret), K(col_id));
@ -265,6 +309,29 @@ int ObAllVirtualTmpFileInfo::fill_sn_column_(const uint64_t col_index, tmp_file:
return ret;
}
#ifdef OB_BUILD_SHARED_STORAGE
int ObAllVirtualTmpFileInfo::fill_ss_column_(const uint64_t col_index, tmp_file::ObSSTmpFileInfo *tmp_file_info)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(tmp_file_info)) {
ret = OB_INVALID_ARGUMENT;
SERVER_LOG(WARN, "invalid argument", KR(ret), KP(tmp_file_info));
} else {
uint64_t col_id = output_column_ids_.at(col_index);
switch (col_id) {
case AGGREGATE_READ_IO_CNT:
cur_row_.cells_[col_index].set_int(tmp_file_info->aggregate_read_io_cnt_);
break;
default:
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "invalid column_id", KR(ret), K(col_id));
break;
}
}
return ret;
}
#endif
int ObAllVirtualTmpFileInfo::process_curr_tenant(common::ObNewRow *&row)
{
int ret = OB_SUCCESS;
@ -296,7 +363,7 @@ int ObAllVirtualTmpFileInfo::process_curr_tenant(common::ObNewRow *&row)
tmp_file_info = OB_NEW(tmp_file::ObSNTmpFileInfo, attr);
#ifdef OB_BUILD_SHARED_STORAGE
} else {
tmp_file_info = OB_NEW(tmp_file::ObTmpFileInfo, attr);
tmp_file_info = OB_NEW(tmp_file::ObSSTmpFileInfo, attr);
#endif
}
if (OB_FAIL(get_next_tmp_file_info_(tmp_file_info))) {

14
src/observer/virtual_table/ob_all_virtual_tmp_file.h
View File

@ -45,6 +45,9 @@ private:
int get_next_tmp_file_info_(tmp_file::ObTmpFileInfo *tmp_file_info);
int fill_columns_(tmp_file::ObTmpFileInfo *tmp_file_info);
int fill_sn_column_(const uint64_t col_index, tmp_file::ObSNTmpFileInfo *tmp_file_info);
#ifdef OB_BUILD_SHARED_STORAGE
int fill_ss_column_(const uint64_t col_index, tmp_file::ObSSTmpFileInfo *tmp_file_info);
#endif
private:
enum
@ -80,6 +83,17 @@ private:
PAGE_FLUSH_CNT,
TYPE,
COMPRESSIBLE_FD,
PERSISTED_TAIL_PAGE_WRITES,
LACK_PAGE_CNT,
TOTAL_TRUNCATED_PAGE_READ_CNT,
TRUNCATED_PAGE_HITS,
TOTAL_KV_CACHE_PAGE_READ_CNT,
KV_CACHE_PAGE_HITS,
TOTAL_UNCACHED_PAGE_READ_CNT,
UNCACHED_PAGE_HITS,
AGGREGATE_READ_IO_CNT,
TOTAL_WBP_PAGE_READ_CNT,
WBP_PAGE_HITS,
};
static const int64_t OB_MAX_FILE_LABEL_SIZE = tmp_file::ObTmpFileGlobal::TMP_FILE_MAX_LABEL_SIZE + 1;
char ip_buffer_[common::OB_IP_STR_BUFF];

165
src/share/inner_table/ob_inner_table_schema.12501_12550.cpp
View File

@ -1656,6 +1656,171 @@ int ObInnerTableSchema::all_virtual_temp_file_schema(ObTableSchema &table_schema
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("persisted_tail_page_writes", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("lack_page_cnt", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("total_truncated_page_read_cnt", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("truncated_page_hits", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("total_kv_cache_page_read_cnt", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("kv_cache_page_read_hits", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("total_uncached_page_read_cnt", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("uncached_page_hits", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("aggregate_read_io_cnt", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("total_wbp_page_read_cnt", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("wbp_page_hits", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObIntType, //column_type
CS_TYPE_INVALID, //column_collation_type
sizeof(int64_t), //column_length
-1, //column_precision
-1, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
table_schema.get_part_option().set_part_num(1);
table_schema.set_part_level(PARTITION_LEVEL_ONE);

165
src/share/inner_table/ob_inner_table_schema.15451_15500.cpp
View File

@ -6998,6 +6998,171 @@ int ObInnerTableSchema::all_virtual_temp_file_ora_schema(ObTableSchema &table_sc
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("PERSISTED_TAIL_PAGE_WRITES", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("LACK_PAGE_CNT", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("TOTAL_TRUNCATED_PAGE_READ_CNT", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("TRUNCATED_PAGE_HITS", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("TOTAL_KV_CACHE_PAGE_READ_CNT", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("KV_CACHE_PAGE_READ_HITS", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("TOTAL_UNCACHED_PAGE_READ_CNT", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("UNCACHED_PAGE_HITS", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("AGGREGATE_READ_IO_CNT", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("TOTAL_WBP_PAGE_READ_CNT", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
ADD_COLUMN_SCHEMA("WBP_PAGE_HITS", //column_name
++column_id, //column_id
0, //rowkey_id
0, //index_id
0, //part_key_pos
ObNumberType, //column_type
CS_TYPE_INVALID, //column_collation_type
38, //column_length
38, //column_precision
0, //column_scale
false, //is_nullable
false); //is_autoincrement
}
if (OB_SUCC(ret)) {
table_schema.get_part_option().set_part_num(1);
table_schema.set_part_level(PARTITION_LEVEL_ONE);

11
src/share/inner_table/ob_inner_table_schema_def.py
View File

@ -15698,6 +15698,17 @@ def_table_schema(
('page_flush_cnt', 'int'),
('type', 'int'),
('compressible_fd', 'int'),
('persisted_tail_page_writes', 'int'),
('lack_page_cnt', 'int'),
('total_truncated_page_read_cnt', 'int'),
('truncated_page_hits', 'int'),
('total_kv_cache_page_read_cnt', 'int'),
('kv_cache_page_read_hits', 'int'),
('total_uncached_page_read_cnt', 'int'),
('uncached_page_hits', 'int'),
('aggregate_read_io_cnt', 'int'),
('total_wbp_page_read_cnt', 'int'),
('wbp_page_hits', 'int'),
],
partition_columns = ['svr_ip', 'svr_port'],
vtable_route_policy = 'distributed',

184
src/storage/tmp_file/ob_i_tmp_file.cpp
View File

@ -36,18 +36,29 @@ int ObTmpFileInfo::init(
const int64_t write_back_data_page_num,
const int64_t flushed_data_page_num,
const int64_t ref_cnt,
const int64_t write_req_cnt,
const int64_t unaligned_write_req_cnt,
const int64_t read_req_cnt,
const int64_t unaligned_read_req_cnt,
const int64_t total_read_size,
const int64_t last_access_ts,
const int64_t last_modify_ts,
const int64_t birth_ts,
const void* const tmp_file_ptr,
const char* const label)
const char* const label,
const int64_t write_req_cnt,
const int64_t unaligned_write_req_cnt,
const int64_t write_persisted_tail_page_cnt,
const int64_t lack_page_cnt,
const int64_t last_modify_ts,
const int64_t read_req_cnt,
const int64_t unaligned_read_req_cnt,
const int64_t total_truncated_page_read_cnt,
const int64_t total_kv_cache_page_read_cnt,
const int64_t total_uncached_page_read_cnt,
const int64_t total_wbp_page_read_cnt,
const int64_t truncated_page_read_hits,
const int64_t kv_cache_page_read_hits,
const int64_t uncached_page_read_hits,
const int64_t wbp_page_read_hits,
const int64_t total_read_size,
const int64_t last_access_ts)
{
int ret = OB_SUCCESS;
// common info
trace_id_ = trace_id;
tenant_id_ = tenant_id;
dir_id_ = dir_id;
@ -59,23 +70,38 @@ int ObTmpFileInfo::init(
write_back_data_page_num_ = write_back_data_page_num;
flushed_data_page_num_ = flushed_data_page_num;
ref_cnt_ = ref_cnt;
write_req_cnt_ = write_req_cnt;
unaligned_write_req_cnt_ = unaligned_write_req_cnt;
read_req_cnt_ = read_req_cnt;
unaligned_read_req_cnt_ = unaligned_read_req_cnt;
total_read_size_ = total_read_size;
last_access_ts_ = last_access_ts;
last_modify_ts_ = last_modify_ts;
birth_ts_ = birth_ts;
tmp_file_ptr_ = tmp_file_ptr;
if (NULL != label) {
label_.assign_strive(label);
}
// write info
write_req_cnt_ = write_req_cnt;
unaligned_write_req_cnt_ = unaligned_write_req_cnt;
write_persisted_tail_page_cnt_ = write_persisted_tail_page_cnt;
lack_page_cnt_ = lack_page_cnt;
last_modify_ts_ = last_modify_ts;
// read info
read_req_cnt_ = read_req_cnt;
unaligned_read_req_cnt_ = unaligned_read_req_cnt;
total_truncated_page_read_cnt_ = total_truncated_page_read_cnt;
total_kv_cache_page_read_cnt_ = total_kv_cache_page_read_cnt;
total_uncached_page_read_cnt_ = total_uncached_page_read_cnt;
total_wbp_page_read_cnt_ = total_wbp_page_read_cnt;
truncated_page_read_hits_ = truncated_page_read_hits;
kv_cache_page_read_hits_ = kv_cache_page_read_hits;
uncached_page_read_hits_ = uncached_page_read_hits;
wbp_page_read_hits_ = wbp_page_read_hits;
total_read_size_ = total_read_size;
last_access_ts_ = last_access_ts;
return ret;
}
void ObTmpFileInfo::reset()
{
// common info
trace_id_.reset();
tenant_id_ = OB_INVALID_TENANT_ID;
dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID;
@ -87,16 +113,28 @@ void ObTmpFileInfo::reset()
write_back_data_page_num_ = 0;
flushed_data_page_num_ = 0;
ref_cnt_ = 0;
write_req_cnt_ = 0;
unaligned_write_req_cnt_ = 0;
read_req_cnt_ = 0;
unaligned_read_req_cnt_ = 0;
total_read_size_ = 0;
last_access_ts_ = -1;
last_modify_ts_ = -1;
birth_ts_ = -1;
tmp_file_ptr_ = nullptr;
label_.reset();
// write info
write_req_cnt_ = 0;
unaligned_write_req_cnt_ = 0;
write_persisted_tail_page_cnt_ = 0;
lack_page_cnt_ = 0;
last_modify_ts_ = -1;
// read info
read_req_cnt_ = 0;
unaligned_read_req_cnt_ = 0;
total_truncated_page_read_cnt_ = 0;
total_kv_cache_page_read_cnt_ = 0;
total_uncached_page_read_cnt_ = 0;
total_wbp_page_read_cnt_ = 0;
truncated_page_read_hits_ = 0;
kv_cache_page_read_hits_ = 0;
uncached_page_read_hits_ = 0;
wbp_page_read_hits_ = 0;
total_read_size_ = 0;
last_access_ts_ = -1;
}
ObITmpFileHandle::ObITmpFileHandle(ObITmpFile *tmp_file)
@ -181,15 +219,25 @@ ObITmpFile::ObITmpFile()
page_idx_cache_(),
callback_allocator_(nullptr),
trace_id_(),
birth_ts_(-1),
label_(),
write_req_cnt_(0),
unaligned_write_req_cnt_(0),
write_persisted_tail_page_cnt_(0),
lack_page_cnt_(0),
last_modify_ts_(-1),
read_req_cnt_(0),
unaligned_read_req_cnt_(0),
total_truncated_page_read_cnt_(0),
total_kv_cache_page_read_cnt_(0),
total_uncached_page_read_cnt_(0),
total_wbp_page_read_cnt_(0),
truncated_page_read_hits_(0),
kv_cache_page_read_hits_(0),
uncached_page_read_hits_(0),
wbp_page_read_hits_(0),
total_read_size_(0),
last_access_ts_(-1),
last_modify_ts_(-1),
birth_ts_(-1),
label_()
last_access_ts_(-1)
{
}
@ -278,16 +326,29 @@ void ObITmpFile::reset()
page_idx_cache_.destroy();
callback_allocator_ = nullptr;
/******for virtual table begin******/
// common info
trace_id_.reset();
write_req_cnt_ = 0;
unaligned_write_req_cnt_ = 0;
read_req_cnt_ = 0;
unaligned_read_req_cnt_ = 0;
total_read_size_ = 0;
last_access_ts_ = -1;
last_modify_ts_ = -1;
birth_ts_ = -1;
label_.reset();
// write info
write_req_cnt_ = 0;
unaligned_write_req_cnt_ = 0;
write_persisted_tail_page_cnt_ = 0;
lack_page_cnt_ = 0;
last_modify_ts_ = -1;
// read info
read_req_cnt_ = 0;
unaligned_read_req_cnt_ = 0;
total_truncated_page_read_cnt_ = 0;
total_kv_cache_page_read_cnt_ = 0;
total_uncached_page_read_cnt_ = 0;
total_wbp_page_read_cnt_ = 0;
truncated_page_read_hits_ = 0;
kv_cache_page_read_hits_ = 0;
uncached_page_read_hits_ = 0;
wbp_page_read_hits_ = 0;
total_read_size_ = 0;
last_access_ts_ = -1;
/******for virtual table end******/
}
}
@ -352,7 +413,7 @@ int ObITmpFile::aio_pread(ObTmpFileIOCtx &io_ctx)
io_ctx.set_is_unaligned_read(true);
}
LOG_DEBUG("start to inner read tmp file", K(fd_), KPC(this));
LOG_DEBUG("start to inner read tmp file", K(fd_), K(io_ctx), KPC(this));
if (OB_UNLIKELY(!io_ctx.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", KR(ret), K(fd_), K(io_ctx), K(read_offset_));
@ -392,12 +453,16 @@ int ObITmpFile::aio_pread(ObTmpFileIOCtx &io_ctx)
// Iterate to read memory data (in write buffer pool).
if (OB_SUCC(ret) && io_ctx.get_todo_size() > 0) {
const int64_t aligned_begin_offset = get_page_begin_offset_(io_ctx.get_read_offset_in_file());
if (OB_UNLIKELY(0 == cached_page_nums_)) {
ret = OB_ITER_END;
LOG_WARN("iter end", KR(ret), K(fd_), K(io_ctx));
} else if (OB_FAIL(inner_read_from_wbp_(io_ctx))) {
LOG_WARN("fail to read tmp file from wbp", KR(ret), K(fd_), K(io_ctx), KPC(this));
} else {
const int64_t aligned_end_offset = get_page_end_offset_(io_ctx.get_read_offset_in_file());
const int64_t total_wbp_page_read_cnt = (aligned_end_offset - aligned_begin_offset) / ObTmpFileGlobal::PAGE_SIZE;
io_ctx.update_read_wbp_page_stat(total_wbp_page_read_cnt);
LOG_DEBUG("finish wbp read", K(fd_), K(io_ctx.get_read_offset_in_file()),
K(io_ctx.get_todo_size()),
K(io_ctx.get_done_size()), K(wbp_begin_offset), KPC(this));
@ -421,6 +486,8 @@ int ObITmpFile::inner_read_truncated_part_(ObTmpFileIOCtx &io_ctx)
} else if (OB_UNLIKELY(io_ctx.get_todo_size() == 0)) {
// do nothing
} else {
int64_t total_truncated_page_read_cnt = 0;
const int64_t origin_read_offset = io_ctx.get_read_offset_in_file();
int64_t read_size = MIN(truncated_offset_ - io_ctx.get_read_offset_in_file(),
io_ctx.get_todo_size());
char *read_buf = io_ctx.get_todo_buffer();
@ -430,6 +497,10 @@ int ObITmpFile::inner_read_truncated_part_(ObTmpFileIOCtx &io_ctx)
} else if (FALSE_IT(MEMSET(read_buf, 0, read_size))) {
} else if (OB_FAIL(io_ctx.update_data_size(read_size))) {
LOG_WARN("fail to update data size", KR(ret), K(fd_), K(read_size));
} else if (FALSE_IT(total_truncated_page_read_cnt = (get_page_end_offset_(io_ctx.get_read_offset_in_file()) -
get_page_begin_offset_(origin_read_offset)) /
ObTmpFileGlobal::PAGE_SIZE)) {
} else if (FALSE_IT(io_ctx.update_read_truncated_stat(total_truncated_page_read_cnt))) {
} else if (OB_UNLIKELY(io_ctx.get_todo_size() > 0 &&
truncated_offset_ == file_size_)) {
ret = OB_ITER_END;
@ -532,9 +603,11 @@ int ObITmpFile::aio_write(ObTmpFileIOCtx &io_ctx)
} else {
bool is_unaligned_write = 0 != file_size_ % ObTmpFileGlobal::PAGE_SIZE ||
0 != io_ctx.get_todo_size() % ObTmpFileGlobal::PAGE_SIZE;
io_ctx.set_is_unaligned_write(is_unaligned_write);
while (OB_SUCC(ret) && io_ctx.get_todo_size() > 0) {
if (OB_FAIL(inner_write_(io_ctx))) {
if (OB_ALLOCATE_TMP_FILE_PAGE_FAILED == ret) {
io_ctx.add_lack_page_cnt();
ret = OB_SUCCESS;
if (TC_REACH_COUNT_INTERVAL(10)) {
LOG_INFO("alloc mem failed, try to evict pages", K(fd_), K(file_size_), K(io_ctx), KPC(this));
@ -547,13 +620,6 @@ int ObITmpFile::aio_write(ObTmpFileIOCtx &io_ctx)
}
}
} // end while
if (OB_SUCC(ret)) {
write_req_cnt_++;
if (is_unaligned_write) {
unaligned_write_req_cnt_++;
}
last_modify_ts_ = ObTimeUtility::current_time();
}
}
if (OB_SUCC(ret)) {
@ -602,6 +668,8 @@ int ObITmpFile::inner_fill_tail_page_(ObTmpFileIOCtx &io_ctx)
} else if (is_in_disk) {
if (OB_FAIL(load_disk_tail_page_and_rewrite_(io_ctx))) {
LOG_WARN("fail to load disk tail page and rewrite", KR(ret), K(fd_), K(io_ctx));
} else {
io_ctx.add_write_persisted_tail_page_cnt();
}
} else {
if (OB_FAIL(append_write_memory_tail_page_(io_ctx))) {
@ -1075,15 +1143,45 @@ int ObITmpFile::insert_or_update_data_flush_node_()
return ret;
}
void ObITmpFile::set_read_stats_vars(const bool is_unaligned_read, const int64_t read_size)
void ObITmpFile::set_read_stats_vars(const ObTmpFileIOCtx &ctx, const int64_t read_size)
{
common::TCRWLock::WLockGuard guard(meta_lock_);
inner_set_read_stats_vars_(ctx, read_size);
}
void ObITmpFile::inner_set_read_stats_vars_(const ObTmpFileIOCtx &ctx, const int64_t read_size)
{
read_req_cnt_++;
if (is_unaligned_read) {
if (ctx.is_unaligned_read()) {
unaligned_read_req_cnt_++;
}
total_read_size_ += read_size;
last_access_ts_ = ObTimeUtility::current_time();
total_truncated_page_read_cnt_ += ctx.get_total_truncated_page_read_cnt();
total_kv_cache_page_read_cnt_ += ctx.get_total_kv_cache_page_read_cnt();
total_uncached_page_read_cnt_ += ctx.get_total_uncached_page_read_cnt();
total_wbp_page_read_cnt_ += ctx.get_total_wbp_page_read_cnt();
truncated_page_read_hits_ += ctx.get_truncated_page_read_hits();
kv_cache_page_read_hits_ += ctx.get_kv_cache_page_read_hits();
uncached_page_read_hits_ += ctx.get_uncached_page_read_hits();
wbp_page_read_hits_ += ctx.get_wbp_page_read_hits();
}
void ObITmpFile::set_write_stats_vars(const ObTmpFileIOCtx &ctx)
{
common::TCRWLock::WLockGuard guard(meta_lock_);
inner_set_write_stats_vars_(ctx);
}
void ObITmpFile::inner_set_write_stats_vars_(const ObTmpFileIOCtx &ctx)
{
write_req_cnt_++;
if (ctx.is_unaligned_write()) {
unaligned_write_req_cnt_++;
}
write_persisted_tail_page_cnt_ += ctx.get_write_persisted_tail_page_cnt();
lack_page_cnt_ += ctx.get_lack_page_cnt();
last_modify_ts_ = ObTimeUtility::current_time();
}
} // end namespace tmp_file

140
src/storage/tmp_file/ob_i_tmp_file.h
View File

@ -42,16 +42,26 @@ public:
write_back_data_page_num_(0),
flushed_data_page_num_(0),
ref_cnt_(0),
write_req_cnt_(0),
unaligned_write_req_cnt_(0),
read_req_cnt_(0),
unaligned_read_req_cnt_(0),
total_read_size_(0),
last_access_ts_(-1),
last_modify_ts_(-1),
birth_ts_(-1),
tmp_file_ptr_(nullptr),
label_() {}
label_(),
write_req_cnt_(0),
unaligned_write_req_cnt_(0),
write_persisted_tail_page_cnt_(0),
lack_page_cnt_(0),
last_modify_ts_(-1),
read_req_cnt_(0),
unaligned_read_req_cnt_(0),
total_truncated_page_read_cnt_(0),
total_kv_cache_page_read_cnt_(0),
total_uncached_page_read_cnt_(0),
total_wbp_page_read_cnt_(0),
truncated_page_read_hits_(0),
kv_cache_page_read_hits_(0),
uncached_page_read_hits_(0),
wbp_page_read_hits_(0),
total_read_size_(0),
last_access_ts_(-1) {}
virtual ~ObTmpFileInfo() { reset(); }
virtual int init(const ObCurTraceId::TraceId &trace_id,
const uint64_t tenant_id,
@ -64,18 +74,29 @@ public:
const int64_t write_back_data_page_num,
const int64_t flushed_data_page_num,
const int64_t ref_cnt,
const int64_t write_req_cnt,
const int64_t unaligned_write_req_cnt,
const int64_t read_req_cnt,
const int64_t unaligned_read_req_cnt,
const int64_t total_read_size,
const int64_t last_access_ts,
const int64_t last_modify_ts,
const int64_t birth_ts,
const void* const tmp_file_ptr,
const char* const label);
const char* const label,
const int64_t write_req_cnt,
const int64_t unaligned_write_req_cnt,
const int64_t write_persisted_tail_page_cnt,
const int64_t lack_page_cnt,
const int64_t last_modify_ts,
const int64_t read_req_cnt,
const int64_t unaligned_read_req_cnt,
const int64_t total_truncated_page_read_cnt,
const int64_t total_kv_cache_page_read_cnt,
const int64_t total_uncached_page_read_cnt,
const int64_t total_wbp_page_read_cnt,
const int64_t truncated_page_read_hits,
const int64_t kv_cache_page_read_hits,
const int64_t uncached_page_read_hits,
const int64_t wbp_page_read_hits,
const int64_t total_read_size,
const int64_t last_access_ts);
virtual void reset();
public:
// common info
common::ObCurTraceId::TraceId trace_id_;
uint64_t tenant_id_;
int64_t dir_id_;
@ -87,24 +108,41 @@ public:
int64_t write_back_data_page_num_;
int64_t flushed_data_page_num_;
int64_t ref_cnt_;
int64_t write_req_cnt_;
int64_t unaligned_write_req_cnt_;
int64_t read_req_cnt_;
int64_t unaligned_read_req_cnt_;
int64_t total_read_size_;
int64_t last_access_ts_;
int64_t last_modify_ts_;
int64_t birth_ts_;
const void *tmp_file_ptr_;
ObFixedLengthString<ObTmpFileGlobal::TMP_FILE_MAX_LABEL_SIZE + 1> label_;
// write info
int64_t write_req_cnt_;
int64_t unaligned_write_req_cnt_;
int64_t write_persisted_tail_page_cnt_;
int64_t lack_page_cnt_;
int64_t last_modify_ts_;
// read info
int64_t read_req_cnt_;
int64_t unaligned_read_req_cnt_;
int64_t total_truncated_page_read_cnt_; // the total read count of truncated pages
int64_t total_kv_cache_page_read_cnt_; // the total read count of pages in kv_cache
int64_t total_uncached_page_read_cnt_; // the total read count of pages with io
int64_t total_wbp_page_read_cnt_; // the total read count of pages in wbp
int64_t truncated_page_read_hits_; // the hit count of truncated pages when read
int64_t kv_cache_page_read_hits_; // the hit count of pages in kv_cache when read
int64_t uncached_page_read_hits_; // the hit count of persisted pages when read
int64_t wbp_page_read_hits_; // the hit count of pages in wbp when read
int64_t total_read_size_;
int64_t last_access_ts_;
TO_STRING_KV(K(trace_id_), K(tenant_id_), K(dir_id_), K(fd_), K(file_size_),
K(truncated_offset_), K(is_deleting_), K(cached_data_page_num_),
K(write_back_data_page_num_), K(flushed_data_page_num_),
K(ref_cnt_), K(write_req_cnt_), K(unaligned_write_req_cnt_),
K(read_req_cnt_), K(unaligned_read_req_cnt_), K(total_read_size_),
K(last_access_ts_), K(last_modify_ts_), K(birth_ts_),
KP(tmp_file_ptr_), K(label_));
K(ref_cnt_), K(birth_ts_), KP(tmp_file_ptr_), K(label_),
K(write_req_cnt_), K(unaligned_write_req_cnt_),
K(write_persisted_tail_page_cnt_), K(lack_page_cnt_), K(last_modify_ts_),
K(read_req_cnt_), K(unaligned_read_req_cnt_),
K(total_truncated_page_read_cnt_), K(total_kv_cache_page_read_cnt_),
K(total_uncached_page_read_cnt_), K(total_wbp_page_read_cnt_),
K(truncated_page_read_hits_), K(kv_cache_page_read_hits_),
K(uncached_page_read_hits_), K(wbp_page_read_hits_),
K(total_read_size_), K(last_access_ts_));
};
class ObITmpFile
@ -202,16 +240,27 @@ public:
K(data_page_flush_level_),
KP(data_flush_node_.get_next()),
KP(wbp_), KP(flush_prio_mgr_), KP(&page_idx_cache_), KP(callback_allocator_),
K(trace_id_),
K(write_req_cnt_), K(unaligned_write_req_cnt_), K(read_req_cnt_),
K(unaligned_read_req_cnt_), K(total_read_size_),
K(last_access_ts_), K(last_modify_ts_), K(birth_ts_), K(label_));
K(trace_id_), K(birth_ts_), K(label_),
K(write_req_cnt_), K(unaligned_write_req_cnt_),
K(write_persisted_tail_page_cnt_), K(lack_page_cnt_), K(last_modify_ts_),
K(read_req_cnt_), K(unaligned_read_req_cnt_),
K(total_truncated_page_read_cnt_), K(total_uncached_page_read_cnt_),
K(total_kv_cache_page_read_cnt_), K(total_wbp_page_read_cnt_),
K(truncated_page_read_hits_), K(uncached_page_read_hits_),
K(kv_cache_page_read_hits_), K(wbp_page_read_hits_),
K(total_read_size_), K(last_access_ts_));
public:
// for virtual table
void set_read_stats_vars(const bool is_unaligned_read, const int64_t read_size);
void set_read_stats_vars(const ObTmpFileIOCtx &ctx, const int64_t read_size);
void set_write_stats_vars(const ObTmpFileIOCtx &ctx);
virtual int copy_info_for_virtual_table(ObTmpFileInfo &tmp_file_info) = 0;
protected:
// for virtual table
virtual void inner_set_read_stats_vars_(const ObTmpFileIOCtx &ctx, const int64_t read_size);
virtual void inner_set_write_stats_vars_(const ObTmpFileIOCtx &ctx);
protected:
int64_t cal_wbp_begin_offset_() const;
virtual bool is_flushing_() = 0;
@ -293,16 +342,29 @@ protected:
ObTmpFileWBPIndexCache page_idx_cache_;
ObIAllocator *callback_allocator_;
/********for virtual table begin********/
// common info
common::ObCurTraceId::TraceId trace_id_;
int64_t write_req_cnt_;
int64_t unaligned_write_req_cnt_;
int64_t read_req_cnt_;
int64_t unaligned_read_req_cnt_;
int64_t total_read_size_;
int64_t last_access_ts_;
int64_t last_modify_ts_;
int64_t birth_ts_;
ObFixedLengthString<ObTmpFileGlobal::TMP_FILE_MAX_LABEL_SIZE + 1> label_;
// write info
int64_t write_req_cnt_;
int64_t unaligned_write_req_cnt_;
int64_t write_persisted_tail_page_cnt_;
int64_t lack_page_cnt_;
int64_t last_modify_ts_;
// read info
int64_t read_req_cnt_;
int64_t unaligned_read_req_cnt_;
int64_t total_truncated_page_read_cnt_;
int64_t total_kv_cache_page_read_cnt_;
int64_t total_uncached_page_read_cnt_;
int64_t total_wbp_page_read_cnt_;
int64_t truncated_page_read_hits_;
int64_t kv_cache_page_read_hits_;
int64_t uncached_page_read_hits_;
int64_t wbp_page_read_hits_;
int64_t total_read_size_;
int64_t last_access_ts_;
/********for virtual table end********/
};

14
src/storage/tmp_file/ob_i_tmp_file_manager.cpp
View File

@ -241,8 +241,7 @@ int ObITenantTmpFileManager::aio_read(const uint64_t tenant_id,
} else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) {
LOG_WARN("fail to aio pread", KR(ret), K(io_info), KPC(tmp_file_handle.get()));
} else {
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(),
io_info.size_);
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx(), io_info.size_);
}
LOG_DEBUG("aio_read a tmp file over", KR(ret), K(io_info), K(io_handle), KPC(tmp_file_handle.get()));
@ -277,8 +276,7 @@ int ObITenantTmpFileManager::aio_pread(const uint64_t tenant_id,
} else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) {
LOG_WARN("fail to aio pread", KR(ret), K(io_info), KPC(tmp_file_handle.get()));
} else {
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(),
io_info.size_);
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx(), io_info.size_);
}
LOG_DEBUG("aio_pread a tmp file over", KR(ret), K(io_info), K(offset), K(io_handle), KPC(tmp_file_handle.get()));
@ -312,8 +310,7 @@ int ObITenantTmpFileManager::read(const uint64_t tenant_id,
} else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) {
LOG_WARN("fail to aio pread", KR(ret), K(io_info), KPC(tmp_file_handle.get()));
} else {
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(),
io_info.size_);
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx(), io_info.size_);
}
if (OB_SUCC(ret) || OB_ITER_END == ret) {
@ -356,8 +353,7 @@ int ObITenantTmpFileManager::pread(const uint64_t tenant_id,
} else if (OB_FAIL(tmp_file_handle.get()->aio_pread(io_handle.get_io_ctx()))) {
LOG_WARN("fail to aio pread", KR(ret), K(io_info), KPC(tmp_file_handle.get()));
} else {
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx().is_unaligned_read(),
io_info.size_);
tmp_file_handle.get()->set_read_stats_vars(io_handle.get_io_ctx(), io_info.size_);
}
if (OB_SUCC(ret) || OB_ITER_END == ret) {
@ -395,6 +391,8 @@ int ObITenantTmpFileManager::aio_write(const uint64_t tenant_id,
LOG_WARN("fail to init io handle", KR(ret), K(tenant_id), K(io_info));
} else if (OB_FAIL(tmp_file_handle.get()->aio_write(io_handle.get_io_ctx()))) {
LOG_WARN("fail to aio write", KR(ret), K(io_info), KPC(tmp_file_handle.get()));
} else {
tmp_file_handle.get()->set_write_stats_vars(io_handle.get_io_ctx());
}
LOG_DEBUG("aio_write a tmp file over", KR(ret), K(io_info), K(io_handle), KPC(tmp_file_handle.get()));

108
src/storage/tmp_file/ob_shared_nothing_tmp_file.cpp
View File

@ -27,43 +27,6 @@ namespace oceanbase
{
namespace tmp_file
{
int ObSNTmpFileInfo::init(
const ObCurTraceId::TraceId &trace_id,
const uint64_t tenant_id,
const int64_t dir_id,
const int64_t fd,
const int64_t file_size,
const int64_t truncated_offset,
const bool is_deleting,
const int64_t cached_page_num,
const int64_t write_back_data_page_num,
const int64_t flushed_data_page_num,
const int64_t ref_cnt,
const int64_t write_req_cnt,
const int64_t unaligned_write_req_cnt,
const int64_t read_req_cnt,
const int64_t unaligned_read_req_cnt,
const int64_t total_read_size,
const int64_t last_access_ts,
const int64_t last_modify_ts,
const int64_t birth_ts,
const void* const tmp_file_ptr,
const char* const label)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObTmpFileInfo::init(trace_id, tenant_id, dir_id, fd, file_size,
truncated_offset, is_deleting, cached_page_num,
write_back_data_page_num, flushed_data_page_num,
ref_cnt, write_req_cnt, unaligned_write_req_cnt,
read_req_cnt, unaligned_read_req_cnt,
total_read_size, last_access_ts, last_modify_ts,
birth_ts, tmp_file_ptr, label))) {
LOG_WARN("failed to init ObTmpFileInfo", KR(ret));
}
return ret;
}
void ObSNTmpFileInfo::reset()
{
meta_tree_epoch_ = 0;
@ -304,6 +267,10 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di
ObTmpFileIOCtx &io_ctx)
{
int ret = OB_SUCCESS;
int64_t total_kv_cache_page_read_cnt = 0;
int64_t total_uncached_page_read_cnt = 0;
int64_t kv_cache_page_read_hits = 0;
int64_t uncached_page_read_hits = 0;
common::ObArray<ObSharedNothingTmpFileDataItem> data_items;
if (OB_FAIL(meta_tree_.search_data_items(io_ctx.get_read_offset_in_file(),
expected_read_disk_size, data_items))) {
@ -346,6 +313,10 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di
} else if (OB_FAIL(io_ctx.update_data_size(read_size))) {
LOG_WARN("fail to update data size", KR(ret), K(read_size));
} else {
kv_cache_page_read_hits++;
total_kv_cache_page_read_cnt += (get_page_end_offset_(io_ctx.get_read_offset_in_file()) -
get_page_begin_offset_(io_ctx.get_read_offset_in_file() - read_size)) /
ObTmpFileGlobal::PAGE_SIZE;
remain_read_size -= read_size;
LOG_DEBUG("succ to read data from cached block",
KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block),
@ -357,6 +328,7 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di
LOG_WARN("fail to get block", KR(ret), K(fd_), K(block_index));
} else { // not hit block cache, read page from disk.
ret = OB_SUCCESS;
const int64_t read_size = end_read_offset_in_block - begin_read_offset_in_block;
if (io_ctx.is_disable_page_cache()) {
if (OB_FAIL(inner_direct_read_from_block_(block_index,
begin_read_offset_in_block,
@ -367,12 +339,21 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di
K(begin_read_offset_in_block), K(end_read_offset_in_block),
K(remain_read_size), K(expected_read_disk_size),
K(data_items[i]), K(io_ctx), KPC(this));
} else {
uncached_page_read_hits++;
total_uncached_page_read_cnt += (get_page_end_offset_(io_ctx.get_read_offset_in_file()) -
get_page_begin_offset_(io_ctx.get_read_offset_in_file() - read_size)) /
ObTmpFileGlobal::PAGE_SIZE;
}
} else {
if (OB_FAIL(inner_cached_read_from_block_(block_index,
begin_read_offset_in_block,
end_read_offset_in_block,
io_ctx))) {
io_ctx,
total_kv_cache_page_read_cnt,
total_uncached_page_read_cnt,
kv_cache_page_read_hits,
uncached_page_read_hits))) {
LOG_WARN("fail to cached read from block",
KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block),
K(begin_read_offset_in_block), K(end_read_offset_in_block),
@ -381,7 +362,6 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di
}
}
if (OB_SUCC(ret)) {
const int64_t read_size = end_read_offset_in_block - begin_read_offset_in_block;
remain_read_size -= read_size;
LOG_DEBUG("succ to read from block",
KR(ret), K(fd_), K(block_index), K(begin_offset_in_block), K(end_offset_in_block),
@ -390,8 +370,12 @@ int ObSharedNothingTmpFile::inner_read_from_disk_(const int64_t expected_read_di
K(data_items[i]), K(io_ctx), KPC(this));
}
}
}
} // end for
if (OB_SUCC(ret)) {
io_ctx.update_read_kv_cache_page_stat(total_kv_cache_page_read_cnt, kv_cache_page_read_hits);
io_ctx.update_sn_read_uncached_page_stat(total_uncached_page_read_cnt, uncached_page_read_hits);
}
return ret;
}
@ -446,7 +430,11 @@ int ObSharedNothingTmpFile::inner_direct_read_from_block_(const int64_t block_in
int ObSharedNothingTmpFile::inner_cached_read_from_block_(const int64_t block_index,
const int64_t begin_read_offset_in_block,
const int64_t end_read_offset_in_block,
ObTmpFileIOCtx &io_ctx)
ObTmpFileIOCtx &io_ctx,
int64_t &total_kv_cache_page_read_cnt,
int64_t &total_uncached_page_read_cnt,
int64_t &kv_cache_page_read_hits,
int64_t &uncached_page_read_hits)
{
int ret = OB_SUCCESS;
const int64_t begin_page_idx_in_block = get_page_virtual_id_(begin_read_offset_in_block, false);
@ -464,7 +452,7 @@ int ObSharedNothingTmpFile::inner_cached_read_from_block_(const int64_t block_in
LOG_WARN("fail to init iterator", KR(ret), K(fd_), K(block_index),
K(begin_page_idx_in_block), K(end_page_idx_in_block));
} else {
int64_t has_read_cached_page_num = 0;
int64_t already_read_cached_page_num = 0;
while (OB_SUCC(ret) && iterator.has_next()) {
bool is_in_cache = false;
int64_t begin_page_id = ObTmpFileGlobal::INVALID_PAGE_ID;
@ -488,12 +476,14 @@ int ObSharedNothingTmpFile::inner_cached_read_from_block_(const int64_t block_in
if (OB_FAIL(ret)) {
} else if (is_in_cache) {
if (OB_FAIL(inner_read_continuous_cached_pages_(begin_read_offset, end_read_offset,
page_value_handles, has_read_cached_page_num,
page_value_handles, already_read_cached_page_num,
io_ctx))) {
LOG_WARN("fail to inner read continuous cached pages", KR(ret), K(fd_), K(begin_read_offset),
K(end_read_offset), K(io_ctx));
} else {
has_read_cached_page_num += (end_page_id - begin_page_id + 1);
total_kv_cache_page_read_cnt += end_page_id - begin_page_id + 1;
already_read_cached_page_num += (end_page_id - begin_page_id + 1);
kv_cache_page_read_hits++;
}
} else {
if (OB_FAIL(inner_read_continuous_uncached_pages_(block_index, begin_read_offset,
@ -502,6 +492,11 @@ int ObSharedNothingTmpFile::inner_cached_read_from_block_(const int64_t block_in
K(begin_read_offset),
K(end_read_offset),
K(io_ctx));
} else {
total_uncached_page_read_cnt += (get_page_end_offset_(end_read_offset) -
get_page_begin_offset_(begin_read_offset)) /
ObTmpFileGlobal::PAGE_SIZE;
uncached_page_read_hits++;
}
}
}
@ -1235,16 +1230,21 @@ int ObSharedNothingTmpFile::copy_info_for_virtual_table(ObTmpFileInfo &tmp_file_
common::TCRWLock::RLockGuardWithTimeout lock_guard(meta_lock_, 100 * 1000L, ret);
ObSNTmpFileInfo &sn_tmp_file_info = static_cast<ObSNTmpFileInfo&>(tmp_file_info);
if (OB_FAIL(sn_tmp_file_info.init(trace_id_, tenant_id_,
dir_id_, fd_, file_size_,
truncated_offset_, is_deleting_,
cached_page_nums_, write_back_data_page_num_,
flushed_data_page_num_, ref_cnt_,
write_req_cnt_, unaligned_write_req_cnt_,
read_req_cnt_, unaligned_read_req_cnt_,
total_read_size_, last_access_ts_,
last_modify_ts_, birth_ts_,
this, label_.ptr()))) {
if (OB_FAIL(ret)) {
LOG_WARN("fail to get lock for reading in virtual table", KR(ret), KPC(this));
} else if (OB_FAIL(sn_tmp_file_info.init(trace_id_, tenant_id_, dir_id_, fd_,
file_size_, truncated_offset_, is_deleting_,
cached_page_nums_, write_back_data_page_num_,
flushed_data_page_num_, ref_cnt_,
birth_ts_, this, label_.ptr(),
write_req_cnt_, unaligned_write_req_cnt_,
write_persisted_tail_page_cnt_, lack_page_cnt_, last_modify_ts_,
read_req_cnt_, unaligned_read_req_cnt_,
total_truncated_page_read_cnt_, total_kv_cache_page_read_cnt_,
total_uncached_page_read_cnt_, total_wbp_page_read_cnt_,
truncated_page_read_hits_, kv_cache_page_read_hits_,
uncached_page_read_hits_, wbp_page_read_hits_,
total_read_size_, last_access_ts_))) {
LOG_WARN("fail to init tmp_file_info", KR(ret), KPC(this));
} else if (OB_FAIL(meta_tree_.copy_info(sn_tmp_file_info))) {
LOG_WARN("fail to copy tree info", KR(ret), KPC(this));

27
src/storage/tmp_file/ob_shared_nothing_tmp_file.h
View File

@ -50,27 +50,6 @@ public:
write_back_meta_page_num_(0),
all_type_page_flush_cnt_(0) {}
virtual ~ObSNTmpFileInfo() { reset(); }
virtual int init(const ObCurTraceId::TraceId &trace_id,
const uint64_t tenant_id,
const int64_t dir_id,
const int64_t fd,
const int64_t file_size,
const int64_t truncated_offset,
const bool is_deleting,
const int64_t cached_page_num,
const int64_t write_back_data_page_num,
const int64_t flushed_data_page_num,
const int64_t ref_cnt,
const int64_t write_req_cnt,
const int64_t unaligned_write_req_cnt,
const int64_t read_req_cnt,
const int64_t unaligned_read_req_cnt,
const int64_t total_read_size,
const int64_t last_access_ts,
const int64_t last_modify_ts,
const int64_t birth_ts,
const void* const tmp_file_ptr,
const char* const label) override;
virtual void reset() override;
public:
int64_t meta_tree_epoch_;
@ -215,7 +194,11 @@ private:
ObTmpFileIOCtx &io_ctx);
int inner_cached_read_from_block_(const int64_t block_index,
const int64_t begin_read_offset_in_block, const int64_t end_read_offset_in_block,
ObTmpFileIOCtx &io_ctx);
ObTmpFileIOCtx &io_ctx,
int64_t &total_kv_cache_page_read_cnt,
int64_t &total_uncached_page_read_cnt,
int64_t &kv_cache_page_read_hits,
int64_t &uncached_page_read_hits);
int collect_pages_in_block_(const int64_t block_index,
const int64_t begin_page_idx_in_block,
const int64_t end_page_idx_in_block,

32
src/storage/tmp_file/ob_tmp_file_io_ctx.cpp
View File

@ -34,12 +34,24 @@ ObTmpFileIOCtx::ObTmpFileIOCtx():
read_offset_in_file_(-1),
disable_page_cache_(false),
disable_block_cache_(false),
is_unaligned_read_(false),
io_flag_(),
io_timeout_ms_(DEFAULT_IO_WAIT_TIME_MS),
io_handles_(),
page_cache_handles_(),
block_cache_handles_()
block_cache_handles_(),
is_unaligned_write_(false),
write_persisted_tail_page_cnt_(0),
lack_page_cnt_(0),
is_unaligned_read_(false),
total_truncated_page_read_cnt_(0),
total_kv_cache_page_read_cnt_(0),
total_uncached_page_read_cnt_(0),
total_wbp_page_read_cnt_(0),
truncated_page_read_hits_(0),
kv_cache_page_read_hits_(0),
uncached_page_read_hits_(0),
aggregate_read_io_cnt_(0),
wbp_page_read_hits_(0)
{
io_handles_.set_attr(ObMemAttr(MTL_ID(), "TMP_IO_HDL"));
page_cache_handles_.set_attr(ObMemAttr(MTL_ID(), "TMP_PCACHE_HDL"));
@ -115,9 +127,23 @@ void ObTmpFileIOCtx::reset()
dir_id_ = ObTmpFileGlobal::INVALID_TMP_FILE_DIR_ID;
disable_page_cache_ = false;
disable_block_cache_ = false;
is_unaligned_read_ = false;
io_flag_.reset();
io_timeout_ms_ = DEFAULT_IO_WAIT_TIME_MS;
/********for virtual table statistics begin********/
is_unaligned_write_ = false;
write_persisted_tail_page_cnt_ = 0;
lack_page_cnt_ = 0;
is_unaligned_read_ = false;
total_truncated_page_read_cnt_ = 0;
total_kv_cache_page_read_cnt_ = 0;
total_uncached_page_read_cnt_ = 0;
total_wbp_page_read_cnt_ = 0;
truncated_page_read_hits_ = 0;
kv_cache_page_read_hits_ = 0;
uncached_page_read_hits_ = 0;
aggregate_read_io_cnt_ = 0;
wbp_page_read_hits_ = 0;
/********for virtual table statistics end ********/
}
bool ObTmpFileIOCtx::is_valid() const

89
src/storage/tmp_file/ob_tmp_file_io_ctx.h
View File

@ -62,16 +62,75 @@ public:
OB_INLINE bool is_disable_block_cache() const { return disable_block_cache_; }
OB_INLINE common::ObIOFlag get_io_flag() const { return io_flag_; }
OB_INLINE int64_t get_io_timeout_ms() const { return io_timeout_ms_; }
OB_INLINE void set_is_unaligned_read(const bool is_unaligned_read) { is_unaligned_read_ = is_unaligned_read; }
OB_INLINE bool is_unaligned_read() { return is_unaligned_read_; }
TO_STRING_KV(K(is_inited_), K(is_read_),
K(fd_), K(dir_id_), KP(buf_),
K(buf_size_), K(done_size_), K(todo_size_),
K(read_offset_in_file_),
K(disable_page_cache_),
K(disable_block_cache_),
K(io_flag_), K(io_timeout_ms_));
K(disable_page_cache_), K(disable_block_cache_),
K(io_flag_), K(io_timeout_ms_),
K(is_unaligned_write_),
K(write_persisted_tail_page_cnt_), K(lack_page_cnt_),
K(is_unaligned_read_),
K(total_truncated_page_read_cnt_), K(total_kv_cache_page_read_cnt_),
K(total_uncached_page_read_cnt_), K(total_wbp_page_read_cnt_),
K(truncated_page_read_hits_), K(kv_cache_page_read_hits_),
K(uncached_page_read_hits_), K(aggregate_read_io_cnt_), K(wbp_page_read_hits_));
public:
// for virtual table stat info
OB_INLINE void set_is_unaligned_write(const bool is_unaligned_write) { is_unaligned_write_ = is_unaligned_write; }
OB_INLINE bool is_unaligned_write() const { return is_unaligned_write_; }
OB_INLINE void add_write_persisted_tail_page_cnt() { write_persisted_tail_page_cnt_++; }
OB_INLINE int64_t get_write_persisted_tail_page_cnt() const { return write_persisted_tail_page_cnt_; }
OB_INLINE void add_lack_page_cnt() { lack_page_cnt_++; }
OB_INLINE int64_t get_lack_page_cnt() const { return lack_page_cnt_; }
OB_INLINE void set_is_unaligned_read(const bool is_unaligned_read) { is_unaligned_read_ = is_unaligned_read; }
OB_INLINE bool is_unaligned_read() const { return is_unaligned_read_; }
OB_INLINE void update_read_truncated_stat(const int64_t page_num)
{
total_truncated_page_read_cnt_ += page_num;
truncated_page_read_hits_++;
}
OB_INLINE int64_t get_total_truncated_page_read_cnt() const { return total_truncated_page_read_cnt_; }
OB_INLINE int64_t get_truncated_page_read_hits() const { return truncated_page_read_hits_; }
OB_INLINE void update_read_kv_cache_page_stat(const int64_t page_num, const int64_t read_cnt)
{
total_kv_cache_page_read_cnt_ += page_num;
kv_cache_page_read_hits_ += read_cnt;
}
OB_INLINE int64_t get_total_kv_cache_page_read_cnt() const { return total_kv_cache_page_read_cnt_; }
OB_INLINE int64_t get_kv_cache_page_read_hits() const { return kv_cache_page_read_hits_; }
OB_INLINE void update_sn_read_uncached_page_stat(const int64_t page_num, const int64_t read_cnt)
{
total_uncached_page_read_cnt_ += page_num;
uncached_page_read_hits_ += read_cnt;
}
#ifdef OB_BUILD_SHARED_STORAGE
OB_INLINE void update_ss_read_uncached_page_stat(const int64_t page_num,
const int64_t uncached_read_cnt,
const int64_t aggregate_read_cnt)
{
total_uncached_page_read_cnt_ += page_num;
uncached_page_read_hits_ += uncached_read_cnt;
aggregate_read_io_cnt_ += aggregate_read_cnt;
}
OB_INLINE int64_t get_aggregate_read_io_cnt() const { return aggregate_read_io_cnt_; }
#endif
OB_INLINE int64_t get_total_uncached_page_read_cnt() const { return total_uncached_page_read_cnt_; }
OB_INLINE int64_t get_uncached_page_read_hits() const { return uncached_page_read_hits_; }
OB_INLINE void update_read_wbp_page_stat(const int64_t page_num)
{
total_wbp_page_read_cnt_ += page_num;
wbp_page_read_hits_++;
}
OB_INLINE int64_t get_total_wbp_page_read_cnt() const { return total_wbp_page_read_cnt_; }
OB_INLINE int64_t get_wbp_page_read_hits() const { return wbp_page_read_hits_; }
public:
struct ObIReadHandle
@ -167,12 +226,28 @@ private:
// read_offset of file in file's read function
bool disable_page_cache_;
bool disable_block_cache_; // only used in ut, to control whether read data from block cache
bool is_unaligned_read_; // for statistics
common::ObIOFlag io_flag_;
int64_t io_timeout_ms_;
common::ObSEArray<ObIOReadHandle, 1> io_handles_;
common::ObSEArray<ObPageCacheHandle, 1> page_cache_handles_;
common::ObSEArray<ObBlockCacheHandle, 1> block_cache_handles_;
/********for virtual table statistics begin********/
// for write
bool is_unaligned_write_;
int64_t write_persisted_tail_page_cnt_;
int64_t lack_page_cnt_;
// for read
bool is_unaligned_read_;
int64_t total_truncated_page_read_cnt_;
int64_t total_kv_cache_page_read_cnt_;
int64_t total_uncached_page_read_cnt_;
int64_t total_wbp_page_read_cnt_;
int64_t truncated_page_read_hits_;
int64_t kv_cache_page_read_hits_;
int64_t uncached_page_read_hits_;
int64_t aggregate_read_io_cnt_;
int64_t wbp_page_read_hits_;
/********for virtual table statistics end********/
DISALLOW_COPY_AND_ASSIGN(ObTmpFileIOCtx);
};

11
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_mysql.result
View File

@ -5620,6 +5620,17 @@ write_back_meta_page_num bigint(20) NO NULL
page_flush_cnt bigint(20) NO NULL
type bigint(20) NO NULL
compressible_fd bigint(20) NO NULL
persisted_tail_page_writes bigint(20) NO NULL
lack_page_cnt bigint(20) NO NULL
total_truncated_page_read_cnt bigint(20) NO NULL
truncated_page_hits bigint(20) NO NULL
total_kv_cache_page_read_cnt bigint(20) NO NULL
kv_cache_page_read_hits bigint(20) NO NULL
total_uncached_page_read_cnt bigint(20) NO NULL
uncached_page_hits bigint(20) NO NULL
aggregate_read_io_cnt bigint(20) NO NULL
total_wbp_page_read_cnt bigint(20) NO NULL
wbp_page_hits bigint(20) NO NULL
select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_temp_file;
IF(count(*) >= 0, 1, 0)
1

11
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result
View File

@ -10901,6 +10901,17 @@ write_back_meta_page_num bigint(20) NO NULL
page_flush_cnt bigint(20) NO NULL
type bigint(20) NO NULL
compressible_fd bigint(20) NO NULL
persisted_tail_page_writes bigint(20) NO NULL
lack_page_cnt bigint(20) NO NULL
total_truncated_page_read_cnt bigint(20) NO NULL
truncated_page_hits bigint(20) NO NULL
total_kv_cache_page_read_cnt bigint(20) NO NULL
kv_cache_page_read_hits bigint(20) NO NULL
total_uncached_page_read_cnt bigint(20) NO NULL
uncached_page_hits bigint(20) NO NULL
aggregate_read_io_cnt bigint(20) NO NULL
total_wbp_page_read_cnt bigint(20) NO NULL
wbp_page_hits bigint(20) NO NULL
select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_temp_file;
IF(count(*) >= 0, 1, 0)
1