hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add defence for too large ObMigrationTabletParam

Browse Source
This commit is contained in:
wxhwang 2024-01-23 13:12:35 +00:00 committed by ob-robot
parent c0aa2e6f1c
commit 854ac60415
5 changed files with 29 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/share/backup/ob_backup_struct.h
View File

@ -109,6 +109,9 @@ static constexpr const int64_t OB_COMMENT_LENGTH = 1024;
static constexpr const int64_t DEFAULT_ARCHIVE_FILE_SIZE = 64 << 20; // 64MB
static constexpr const int64_t DEFAULT_BACKUP_DATA_FILE_SIZE = 4 * 1024LL * 1024LL * 1024LL; // 4GB
// max ObMigrationTabletParam serialize size during backup.
static constexpr const int64_t MAX_BACKUP_TABLET_META_SERIALIZE_SIZE = 2 * 1024LL * 1024LL; // 2MB
//add by physical backup and restore
const char *const OB_STR_INCARNATION = "incarnation";
const char *const OB_STRING_BACKUP_DATA = "data";

10
src/storage/backup/ob_backup_extern_info_mgr.cpp
View File

@ -613,6 +613,9 @@ int ObExternTabletMetaReader::get_next(storage::ObMigrationTabletParam &tablet_m
LOG_WARN("tablet meta reader not init", K(ret));
} else if (end_() && OB_FAIL(read_next_batch_())) {
LOG_WARN("failed to update inner array", K(ret));
} else if (end_()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet_meta_array_ is empty", K(ret));
} else if (OB_FAIL(tablet_meta.assign(tablet_meta_array_.at(cur_tablet_idx_)))) {
LOG_WARN("failed to assign tablet meta", K(ret), K(cur_tablet_idx_), K(tablet_meta_array_));
} else if (OB_FALSE_IT(cur_tablet_idx_++)) {
@ -651,7 +654,7 @@ int ObExternTabletMetaReader::read_next_range_tablet_metas_()
int ret = OB_SUCCESS;
share::ObBackupPath path;
char *buf = nullptr;
const int64_t DEFAULT_BUF_LEN = 2 * 1024 * 1024; // 2M
const int64_t DEFAULT_BUF_LEN = 2 * MAX_BACKUP_TABLET_META_SERIALIZE_SIZE;
const int64_t buf_len = tablet_info_trailer_array_.at(cur_trailer_idx_).length_ - cur_buf_offset_ < DEFAULT_BUF_LEN ?
(tablet_info_trailer_array_.at(cur_trailer_idx_).length_ - cur_buf_offset_) : DEFAULT_BUF_LEN;
int64_t cur_total_len = 0;
@ -686,7 +689,7 @@ int ObExternTabletMetaReader::read_next_range_tablet_metas_()
LOG_WARN("common_header is not valid", K(ret), K(path), K(buffer_reader));
} else if (common_header->data_zlength_ > buffer_reader.remain()) {
cur_total_len = buffer_reader.pos() - sizeof(ObBackupCommonHeader);
LOG_INFO("buf not enough, wait later", K(cur_total_len), K(buffer_reader));
LOG_INFO("buf not enough, wait later", K(cur_total_len), K(buffer_reader), KPC(common_header));
break;
} else if (OB_FAIL(common_header->check_data_checksum(buffer_reader.current(), common_header->data_zlength_))) {
LOG_WARN("failed to check data checksum", K(ret), K(*common_header), K(path), K(buffer_reader));
@ -702,6 +705,9 @@ int ObExternTabletMetaReader::read_next_range_tablet_metas_()
}
if (OB_FAIL(ret)) {
} else if (cur_tablet_meta_array.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet meta is too large", K(ret));
} else {
tablet_meta_array_.reset();
if (OB_FAIL(tablet_meta_array_.assign(cur_tablet_meta_array))) {

5
src/storage/backup/ob_backup_task.cpp
View File

@ -3690,9 +3690,14 @@ int ObLSBackupMetaTask::backup_ls_meta_and_tablet_metas_(const uint64_t tenant_i
blocksstable::ObBufferReader buffer_reader;
int64_t macro_block_count = 0;
int64_t start_time = 0;
const int64_t serialize_size = tablet_info.param_.get_serialize_size();
if (!tablet_info.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("tablet meta is invalid", K(ret), K(tablet_info));
} else if (MAX_BACKUP_TABLET_META_SERIALIZE_SIZE < serialize_size) {
// In case of the tablet meta is too large.
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tablet meta is too large.", K(ret), K(serialize_size), K(tablet_info));
} else if (OB_FAIL(buffer_writer.ensure_space(backup::OB_BACKUP_READ_BLOCK_SIZE))) {
LOG_WARN("failed to ensure space");
} else if (OB_FAIL(buffer_writer.write_serialize(tablet_info.param_))) {

5
src/storage/ob_storage_rpc.cpp
View File

@ -1183,7 +1183,10 @@ int ObStorageStreamRpcP<RPC_CODE>::fill_data(const Data &data)
|| (curr_ts - last_send_time_ >= FLUSH_TIME_INTERVAL
&& this->result_.get_capacity() != this->result_.get_remain())) {
LOG_INFO("flush", K(this->result_));
if (OB_FAIL(flush_and_wait())) {
if (0 == this->result_.get_position()) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(ERROR, "data is too large", K(ret));
} else if (OB_FAIL(flush_and_wait())) {
STORAGE_LOG(WARN, "failed to flush_and_wait", K(ret));
}
}

11
tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp
View File

@ -1557,7 +1557,7 @@ int ObAdminDumpBackupDataExecutor::print_ls_tablet_meta_tablets_()
STORAGE_LOG(WARN, "failed to read tablet metas file trailer", K(ret));
} else {
common::ObArenaAllocator allocator;
const int64_t DEFAULT_BUF_LEN = 2 * 1024 * 1024; // 2M
const int64_t DEFAULT_BUF_LEN = 2 * MAX_BACKUP_TABLET_META_SERIALIZE_SIZE;
int64_t cur_buf_offset = tablet_meta_trailer.offset_;
int64_t cur_total_len = 0;
char *buf = nullptr;
@ -1570,6 +1570,7 @@ int ObAdminDumpBackupDataExecutor::print_ls_tablet_meta_tablets_()
} else if (OB_FAIL(ObAdminDumpBackupDataUtil::pread_file(backup_path_, storage_info_, cur_buf_offset, buf_len, buf))) {
STORAGE_LOG(WARN, "failed to pread file", K(ret), K(backup_path_), K(storage_info_), K(cur_buf_offset), K(buf_len));
} else {
int64_t tablet_cnt = 0;
backup::ObBackupTabletMeta tablet_meta;
blocksstable::ObBufferReader buffer_reader(buf, buf_len);
while (OB_SUCC(ret)) {
@ -1589,7 +1590,12 @@ int ObAdminDumpBackupDataExecutor::print_ls_tablet_meta_tablets_()
STORAGE_LOG(WARN, "common_header is not valid", K(ret), K(backup_path_), K(buffer_reader));
} else if (common_header->data_zlength_ > buffer_reader.remain()) {
cur_total_len = buffer_reader.pos() - sizeof(ObBackupCommonHeader);
STORAGE_LOG(INFO, "buf not enough, wait later", K(cur_total_len), K(buffer_reader));
if (0 == tablet_cnt) {
ret = OB_ERR_UNEXPECTED;
STORAGE_LOG(WARN, "tablet meta is too large", KPC(common_header), K(cur_total_len), K(buffer_reader));
} else {
STORAGE_LOG(INFO, "buf not enough, wait later", KPC(common_header), K(cur_total_len), K(buffer_reader), K(tablet_cnt));
}
break;
} else if (OB_FAIL(common_header->check_data_checksum(buffer_reader.current(), common_header->data_zlength_))) {
STORAGE_LOG(WARN, "failed to check data checksum", K(ret), K(*common_header), K(backup_path_), K(buffer_reader));
@ -1598,6 +1604,7 @@ int ObAdminDumpBackupDataExecutor::print_ls_tablet_meta_tablets_()
} else if (OB_FAIL(buffer_reader.advance(common_header->data_length_ + common_header->align_length_))) {
STORAGE_LOG(WARN, "failed to advance buffer", K(ret));
} else {
++tablet_cnt;
tablet_meta.tablet_id_ = tablet_meta.tablet_meta_.tablet_id_;
if (OB_FAIL(dump_common_header_(*common_header))) {
STORAGE_LOG(WARN, "failed to dump common header", K(ret));