hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

The backup module module memory is divided into each tenant to prevent pressure on the system tenant's memory usage

Browse Source
This commit is contained in:
oceanoverflow
2023-07-18 08:12:23 +00:00
committed by ob-robot
parent 40fee59e6c
commit 4dc8f35b58
9 changed files with 68 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/storage/backup/ob_backup_ctx.cpp
View File

@ -1405,7 +1405,7 @@ int ObLSBackupCtx::prepare_tablet_id_reader_(ObILSTabletIdReader *&reader)
const share::ObBackupSetDesc &backup_set_desc = param_.backup_set_desc_;
const share::ObLSID &ls_id = param_.ls_id_;
const ObLSTabletIdReaderType type = LS_TABLET_ID_READER;
if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_ls_tablet_id_reader(type))) {
if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_ls_tablet_id_reader(type, tenant_id))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to get ls tablet id reader", K(ret), K(type));
} else if (OB_FAIL(tmp_reader->init(backup_dest, tenant_id, backup_set_desc, ls_id))) {

60
src/storage/backup/ob_backup_factory.cpp
View File

@ -17,99 +17,115 @@
namespace oceanbase {
namespace backup {
ObILSTabletIdReader *ObLSBackupFactory::get_ls_tablet_id_reader(const ObLSTabletIdReaderType &type)
ObILSTabletIdReader *ObLSBackupFactory::get_ls_tablet_id_reader(
const ObLSTabletIdReaderType &type, const uint64_t tenant_id)
{
ObILSTabletIdReader *reader = NULL;
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
if (LS_TABLET_ID_READER == type) {
reader = OB_NEW(ObLSTabletIdReader, ObModIds::BACKUP);
reader = OB_NEW(ObLSTabletIdReader, attr);
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unknown tablet reader type", K(type));
}
return reader;
}
ObITabletLogicMacroIdReader *ObLSBackupFactory::get_tablet_logic_macro_id_reader(const ObTabletLogicIdReaderType &type)
ObITabletLogicMacroIdReader *ObLSBackupFactory::get_tablet_logic_macro_id_reader(
const ObTabletLogicIdReaderType &type, const uint64_t tenant_id)
{
ObITabletLogicMacroIdReader *reader = NULL;
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
if (TABLET_LOGIC_ID_READER == type) {
reader = OB_NEW(ObTabletLogicMacroIdReader, ObModIds::BACKUP);
reader = OB_NEW(ObTabletLogicMacroIdReader, attr);
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unknown reader type", K(type));
}
return reader;
}
ObIMacroBlockBackupReader *ObLSBackupFactory::get_macro_block_backup_reader(const ObMacroBlockReaderType &type)
ObIMacroBlockBackupReader *ObLSBackupFactory::get_macro_block_backup_reader(
const ObMacroBlockReaderType &type, const uint64_t tenant_id)
{
ObIMacroBlockBackupReader *reader = NULL;
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
if (LOCAL_MACRO_BLOCK_READER == type) {
reader = OB_NEW(ObMacroBlockBackupReader, ObModIds::BACKUP);
reader = OB_NEW(ObMacroBlockBackupReader, attr);
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unknown reader type", K(type));
}
return reader;
}
ObMultiMacroBlockBackupReader *ObLSBackupFactory::get_multi_macro_block_backup_reader()
ObMultiMacroBlockBackupReader *ObLSBackupFactory::get_multi_macro_block_backup_reader(const uint64_t tenant_id)
{
return OB_NEW(ObMultiMacroBlockBackupReader, ObModIds::BACKUP);
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
return OB_NEW(ObMultiMacroBlockBackupReader, attr);
}
ObITabletMetaBackupReader *ObLSBackupFactory::get_tablet_meta_backup_reader(const ObTabletMetaReaderType &type)
ObITabletMetaBackupReader *ObLSBackupFactory::get_tablet_meta_backup_reader(
const ObTabletMetaReaderType &type, const uint64_t tenant_id)
{
ObITabletMetaBackupReader *reader = NULL;
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
if (TABLET_META_READER == type) {
reader = OB_NEW(ObTabletMetaBackupReader, ObModIds::BACKUP);
reader = OB_NEW(ObTabletMetaBackupReader, attr);
} else if (SSTABLE_META_READER == type) {
reader = OB_NEW(ObSSTableMetaBackupReader, ObModIds::BACKUP);
reader = OB_NEW(ObSSTableMetaBackupReader, attr);
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unknown reader type", K(type));
}
return reader;
}
ObIBackupIndexIterator *ObLSBackupFactory::get_backup_index_iterator(const ObBackupIndexIteratorType &type)
ObIBackupIndexIterator *ObLSBackupFactory::get_backup_index_iterator(
const ObBackupIndexIteratorType &type, const uint64_t tenant_id)
{
ObIBackupIndexIterator *iterator = NULL;
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
if (BACKUP_MACRO_BLOCK_INDEX_ITERATOR == type) {
iterator = OB_NEW(ObBackupMacroBlockIndexIterator, ObModIds::BACKUP);
iterator = OB_NEW(ObBackupMacroBlockIndexIterator, attr);
} else if (BACKUP_MACRO_RANGE_INDEX_ITERATOR == type) {
iterator = OB_NEW(ObBackupMacroRangeIndexIterator, ObModIds::BACKUP);
iterator = OB_NEW(ObBackupMacroRangeIndexIterator, attr);
} else if (BACKUP_META_INDEX_ITERATOR == type) {
iterator = OB_NEW(ObBackupMetaIndexIterator, ObModIds::BACKUP);
iterator = OB_NEW(ObBackupMetaIndexIterator, attr);
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unknown iterator type", K(type));
}
return iterator;
}
ObIBackupTabletProvider *ObLSBackupFactory::get_backup_tablet_provider(const ObBackupTabletProviderType &type)
ObIBackupTabletProvider *ObLSBackupFactory::get_backup_tablet_provider(
const ObBackupTabletProviderType &type, const uint64_t tenant_id)
{
ObIBackupTabletProvider *provider = NULL;
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
if (BACKUP_TABLET_PROVIDER == type) {
provider = OB_NEW(ObBackupTabletProvider, ObModIds::BACKUP);
provider = OB_NEW(ObBackupTabletProvider, attr);
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unknown provider type", K(type));
}
return provider;
}
ObIBackupMacroBlockIndexFuser *ObLSBackupFactory::get_backup_macro_index_fuser(const ObBackupMacroIndexFuserType &type)
ObIBackupMacroBlockIndexFuser *ObLSBackupFactory::get_backup_macro_index_fuser(
const ObBackupMacroIndexFuserType &type, const uint64_t tenant_id)
{
ObIBackupMacroBlockIndexFuser *fuser = NULL;
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
if (type == BACKUP_MACRO_INDEX_MINOR_FUSER) {
fuser = OB_NEW(ObBackupMacroIndexMinorFuser, ObModIds::BACKUP);
fuser = OB_NEW(ObBackupMacroIndexMinorFuser, attr);
} else if (type == BACKUP_MACRO_INDEX_MAJOR_FUSER) {
fuser = OB_NEW(ObBackupMacroIndexMajorFuser, ObModIds::BACKUP);
fuser = OB_NEW(ObBackupMacroIndexMajorFuser, attr);
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "unknown fuser type", K(type));
}
return fuser;
}
ObBackupTabletCtx *ObLSBackupFactory::get_backup_tablet_ctx()
ObBackupTabletCtx *ObLSBackupFactory::get_backup_tablet_ctx(const uint64_t tenant_id)
{
return OB_NEW(ObBackupTabletCtx, ObModIds::BACKUP);
lib::ObMemAttr attr(tenant_id, ObModIds::BACKUP);
return OB_NEW(ObBackupTabletCtx, attr);
}
void ObLSBackupFactory::free(ObILSTabletIdReader *&reader)

18
src/storage/backup/ob_backup_factory.h
View File

@ -23,15 +23,15 @@ namespace backup {
class ObLSBackupFactory {
public:
static ObILSTabletIdReader *get_ls_tablet_id_reader(const ObLSTabletIdReaderType &type);
static ObITabletLogicMacroIdReader *get_tablet_logic_macro_id_reader(const ObTabletLogicIdReaderType &type);
static ObIMacroBlockBackupReader *get_macro_block_backup_reader(const ObMacroBlockReaderType &type);
static ObMultiMacroBlockBackupReader *get_multi_macro_block_backup_reader();
static ObITabletMetaBackupReader *get_tablet_meta_backup_reader(const ObTabletMetaReaderType &type);
static ObIBackupIndexIterator *get_backup_index_iterator(const ObBackupIndexIteratorType &type);
static ObIBackupTabletProvider *get_backup_tablet_provider(const ObBackupTabletProviderType &type);
static ObIBackupMacroBlockIndexFuser *get_backup_macro_index_fuser(const ObBackupMacroIndexFuserType &type);
static ObBackupTabletCtx *get_backup_tablet_ctx();
static ObILSTabletIdReader *get_ls_tablet_id_reader(const ObLSTabletIdReaderType &type, const uint64_t tenant_id);
static ObITabletLogicMacroIdReader *get_tablet_logic_macro_id_reader(const ObTabletLogicIdReaderType &type, const uint64_t tenant_id);
static ObIMacroBlockBackupReader *get_macro_block_backup_reader(const ObMacroBlockReaderType &type, const uint64_t tenant_id);
static ObMultiMacroBlockBackupReader *get_multi_macro_block_backup_reader(const uint64_t tenant_id);
static ObITabletMetaBackupReader *get_tablet_meta_backup_reader(const ObTabletMetaReaderType &type, const uint64_t tenant_id);
static ObIBackupIndexIterator *get_backup_index_iterator(const ObBackupIndexIteratorType &type, const uint64_t tenant_id);
static ObIBackupTabletProvider *get_backup_tablet_provider(const ObBackupTabletProviderType &type, const uint64_t tenant_id);
static ObIBackupMacroBlockIndexFuser *get_backup_macro_index_fuser(const ObBackupMacroIndexFuserType &type, const uint64_t tenant_id);
static ObBackupTabletCtx *get_backup_tablet_ctx(const uint64_t tenant_id);
static void free(ObILSTabletIdReader *&reader);
static void free(ObITabletLogicMacroIdReader *&reader);

10
src/storage/backup/ob_backup_index_merger.cpp
View File

@ -874,7 +874,7 @@ int ObBackupMacroBlockIndexMerger::prepare_prev_backup_set_index_iter_(
ret = OB_ERR_UNEXPECTED;
LOG_WARN("no need to prepare if not incremental", K(ret));
} else if (OB_ISNULL(tmp_iter = static_cast<ObBackupMacroRangeIndexIterator *>(
ObLSBackupFactory::get_backup_index_iterator(type)))) {
ObLSBackupFactory::get_backup_index_iterator(type, merge_param.tenant_id_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to get backup index iterator", K(ret), K(type));
} else if (OB_FAIL(ObLSBackupOperator::get_prev_backup_set_desc(merge_param.tenant_id_,
@ -940,7 +940,7 @@ int ObBackupMacroBlockIndexMerger::alloc_merge_iter_(const bool tenant_level,
const ObBackupIndexIteratorType type = BACKUP_MACRO_BLOCK_INDEX_ITERATOR;
ObBackupMacroBlockIndexIterator *tmp_iter = NULL;
if (OB_ISNULL(tmp_iter = static_cast<ObBackupMacroBlockIndexIterator *>(
ObLSBackupFactory::get_backup_index_iterator(type)))) {
ObLSBackupFactory::get_backup_index_iterator(type, merge_param.tenant_id_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc iterator", K(ret));
} else if (OB_FAIL(tmp_iter->init(merge_param.task_id_,
@ -959,7 +959,7 @@ int ObBackupMacroBlockIndexMerger::alloc_merge_iter_(const bool tenant_level,
const ObBackupIndexIteratorType type = BACKUP_MACRO_RANGE_INDEX_ITERATOR;
ObBackupMacroRangeIndexIterator *tmp_iter = NULL;
if (OB_ISNULL(tmp_iter = static_cast<ObBackupMacroRangeIndexIterator *>(
ObLSBackupFactory::get_backup_index_iterator(type)))) {
ObLSBackupFactory::get_backup_index_iterator(type, merge_param.tenant_id_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc iterator", K(ret));
} else if (OB_FAIL(tmp_iter->init(merge_param.task_id_,
@ -1041,7 +1041,7 @@ int ObBackupMacroBlockIndexMerger::prepare_merge_fuser_(ObIBackupMacroBlockIndex
} else {
type = BACKUP_MACRO_INDEX_MINOR_FUSER;
}
if (OB_ISNULL(tmp_fuser = ObLSBackupFactory::get_backup_macro_index_fuser(type))) {
if (OB_ISNULL(tmp_fuser = ObLSBackupFactory::get_backup_macro_index_fuser(type, merge_param_.tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate provider", K(ret), K(type));
} else {
@ -1354,7 +1354,7 @@ int ObBackupMetaIndexMerger::alloc_merge_iter_(const ObBackupIndexMergeParam &me
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret));
} else if (OB_ISNULL(tmp_iter = static_cast<ObBackupMetaIndexIterator *>(
ObLSBackupFactory::get_backup_index_iterator(type)))) {
ObLSBackupFactory::get_backup_index_iterator(type, merge_param.tenant_id_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc iterator", K(ret));
} else if (OB_FAIL(tmp_iter->init(merge_param.task_id_,

8
src/storage/backup/ob_backup_reader.cpp
View File

@ -330,7 +330,7 @@ ObMultiMacroBlockBackupReader::~ObMultiMacroBlockBackupReader()
reset();
}
int ObMultiMacroBlockBackupReader::init(
int ObMultiMacroBlockBackupReader::init(const uint64_t tenant_id,
const ObMacroBlockReaderType &reader_type, const common::ObIArray<ObBackupMacroBlockId> &list)
{
int ret = OB_SUCCESS;
@ -346,7 +346,7 @@ int ObMultiMacroBlockBackupReader::init(
readers_.reset();
for (int64_t i = 0; OB_SUCC(ret) && i < list.count(); ++i) {
ObIMacroBlockBackupReader *reader = NULL;
if (OB_FAIL(alloc_macro_block_reader_(reader_type, reader))) {
if (OB_FAIL(alloc_macro_block_reader_(tenant_id, reader_type, reader))) {
LOG_WARN("failed to alloc macro block reader", K(ret), K(reader_type));
} else if (OB_FAIL(readers_.push_back(reader))) {
LOG_WARN("failed to push back reader", K(ret));
@ -402,12 +402,12 @@ void ObMultiMacroBlockBackupReader::reset_prev_macro_block_reader_(const int64_t
}
}
int ObMultiMacroBlockBackupReader::alloc_macro_block_reader_(
int ObMultiMacroBlockBackupReader::alloc_macro_block_reader_(const uint64_t tenant_id,
const ObMacroBlockReaderType &reader_type, ObIMacroBlockBackupReader *&reader)
{
int ret = OB_SUCCESS;
ObIMacroBlockBackupReader *tmp_reader = NULL;
if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_macro_block_backup_reader(reader_type))) {
if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_macro_block_backup_reader(reader_type, tenant_id))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to get macro block backup reader", K(ret));
} else {

4
src/storage/backup/ob_backup_reader.h
View File

@ -189,12 +189,12 @@ class ObMultiMacroBlockBackupReader {
public:
ObMultiMacroBlockBackupReader();
virtual ~ObMultiMacroBlockBackupReader();
int init(const ObMacroBlockReaderType &reader_type, const common::ObIArray<ObBackupMacroBlockId> &list);
int init(const uint64_t tenant_id, const ObMacroBlockReaderType &reader_type, const common::ObIArray<ObBackupMacroBlockId> &list);
int get_next_macro_block(blocksstable::ObBufferReader &data, blocksstable::ObLogicMacroBlockId &logic_id);
void reset();
private:
int alloc_macro_block_reader_(const ObMacroBlockReaderType &reader_type, ObIMacroBlockBackupReader *&reader);
int alloc_macro_block_reader_(const uint64_t tenant_id, const ObMacroBlockReaderType &reader_type, ObIMacroBlockBackupReader *&reader);
int prepare_macro_block_reader_(const int64_t idx);
void reset_prev_macro_block_reader_(const int64_t idx);
int fetch_macro_block_with_retry_(blocksstable::ObBufferReader &data, blocksstable::ObLogicMacroBlockId &logic_id);

12
src/storage/backup/ob_backup_task.cpp
View File

@ -766,7 +766,7 @@ int ObLSBackupDataDagNet::prepare_backup_tablet_provider_(const ObLSBackupParam
if (!param.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(param));
} else if (OB_ISNULL(tmp_provider = static_cast<ObBackupTabletProvider *>(ObLSBackupFactory::get_backup_tablet_provider(type)))) {
} else if (OB_ISNULL(tmp_provider = static_cast<ObBackupTabletProvider *>(ObLSBackupFactory::get_backup_tablet_provider(type, param.tenant_id_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate provider", K(ret), K(param));
} else if (OB_FAIL(tmp_provider->init(param, backup_data_type, ls_backup_ctx, index_kv_cache, sql_proxy))) {
@ -2687,7 +2687,7 @@ int ObLSBackupDataTask::do_backup_macro_block_data_()
LOG_WARN("failed to get macro block id list", K(ret));
} else if (OB_UNLIKELY(macro_list.empty())) {
LOG_INFO("no macro list need to backup");
} else if (OB_FAIL(prepare_macro_block_reader_(macro_list, reader))) {
} else if (OB_FAIL(prepare_macro_block_reader_(param_.tenant_id_, macro_list, reader))) {
LOG_WARN("failed to prepare macro block reader", K(ret), K(macro_list));
} else {
LOG_INFO("start backup macro block data", K(macro_list));
@ -3004,7 +3004,7 @@ int ObLSBackupDataTask::get_meta_item_list_(common::ObIArray<ObBackupProviderIte
return ret;
}
int ObLSBackupDataTask::prepare_macro_block_reader_(
int ObLSBackupDataTask::prepare_macro_block_reader_(const uint64_t tenant_id,
const common::ObIArray<ObBackupMacroBlockId> &list, ObMultiMacroBlockBackupReader *&reader)
{
int ret = OB_SUCCESS;
@ -3013,10 +3013,10 @@ int ObLSBackupDataTask::prepare_macro_block_reader_(
if (OB_UNLIKELY(list.empty())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret));
} else if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_multi_macro_block_backup_reader())) {
} else if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_multi_macro_block_backup_reader(param_.tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc iterator", K(ret));
} else if (OB_FAIL(tmp_reader->init(reader_type, list))) {
} else if (OB_FAIL(tmp_reader->init(tenant_id, reader_type, list))) {
LOG_WARN("failed to init", K(ret), K(list));
} else {
reader = tmp_reader;
@ -3034,7 +3034,7 @@ int ObLSBackupDataTask::prepare_tablet_meta_reader_(const common::ObTabletID &ta
if (!tablet_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(tablet_id));
} else if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_tablet_meta_backup_reader(reader_type))) {
} else if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_tablet_meta_backup_reader(reader_type, param_.tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc iterator", K(ret), K(reader_type));
} else if (OB_FAIL(tmp_reader->init(tablet_id, backup_data_type_, tablet_handle))) {

2
src/storage/backup/ob_backup_task.h
View File

@ -528,7 +528,7 @@ private:
int check_disk_space_();
int get_macro_block_id_list_(common::ObIArray<ObBackupMacroBlockId> &list);
int get_meta_item_list_(common::ObIArray<ObBackupProviderItem> &list);
int prepare_macro_block_reader_(
int prepare_macro_block_reader_(const uint64_t tenant_id,
const common::ObIArray<ObBackupMacroBlockId> &list, ObMultiMacroBlockBackupReader *&reader);
int prepare_tablet_meta_reader_(const common::ObTabletID &tablet_id, const ObTabletMetaReaderType &reader_type,
storage::ObTabletHandle &tablet_handle, ObITabletMetaBackupReader *&reader);

4
src/storage/backup/ob_backup_utils.cpp
View File

@ -973,7 +973,7 @@ int ObBackupTabletStat::alloc_stat_(ObBackupTabletCtx *&stat)
int ret = OB_SUCCESS;
stat = NULL;
ObBackupTabletCtx *tmp_ctx = NULL;
if (OB_ISNULL(tmp_ctx = ObLSBackupFactory::get_backup_tablet_ctx())) {
if (OB_ISNULL(tmp_ctx = ObLSBackupFactory::get_backup_tablet_ctx(tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate ctx", K(ret));
} else {
@ -2042,7 +2042,7 @@ int ObBackupTabletProvider::prepare_tablet_logic_id_reader_(const common::ObTabl
if (!tablet_id.is_valid()) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get invalid args", K(ret), K(tablet_id));
} else if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_tablet_logic_macro_id_reader(type))) {
} else if (OB_ISNULL(tmp_reader = ObLSBackupFactory::get_tablet_logic_macro_id_reader(type, param_.tenant_id_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("faild to alloc memory", K(ret));
} else if (OB_FAIL(tmp_reader->init(tablet_id, tablet_handle, table_key, sstable, BATCH_SIZE))) {