hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[FEAT MERGE] merge transfer

Browse Source 
Co-authored-by: wxhwang <wxhwang@126.com>
Co-authored-by: godyangfight <godyangfight@gmail.com>
Co-authored-by: Tyshawn <tuyunshan@gmail.com>
This commit is contained in:
xuhuleon
2023-06-21 11:42:26 +00:00
committed by ob-robot
parent d06678002e
commit 9dae112952
1280 changed files with 149724 additions and 48813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

299
src/storage/compaction/ob_sstable_merge_info_mgr.cpp Normal file → Executable file
View File

@ -14,6 +14,7 @@
#include "ob_sstable_merge_info_mgr.h"
#include "lib/utility/ob_print_utils.h"
#include "storage/ob_sstable_struct.h"
#include "storage/compaction/ob_compaction_diagnose.h"
#include "share/config/ob_server_config.h"
#include "observer/omt/ob_multi_tenant.h"
#include "observer/ob_server_struct.h"
@ -23,135 +24,13 @@ namespace oceanbase
using namespace common;
namespace storage
{
/**
* ------------------------------------------------------------------ObSSTableMergeInfoIter-------------------------------------------------------------
*/
ObSSTableMergeInfoIterator::ObSSTableMergeInfoIterator()
: all_tenants_(NULL/*allocator*/, ObModIds::OB_TENANT_ID_LIST),
tenant_idx_(0),
cur_tenant_id_(OB_INVALID_TENANT_ID),
major_info_idx_(0),
major_info_cnt_(0),
minor_info_idx_(0),
minor_info_cnt_(0),
is_opened_(false)
{
}
ObSSTableMergeInfoIterator::~ObSSTableMergeInfoIterator()
{
}
int ObSSTableMergeInfoIterator::open(const int64_t tenant_id)
{
int ret = OB_SUCCESS;
MAKE_TENANT_SWITCH_SCOPE_GUARD(unused_tenant_guard);
if (is_opened_) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "The ObSSTableMergeInfoIterator has been opened", K(ret));
} else if (!::is_valid_tenant_id(tenant_id)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), K(tenant_id));
} else if (OB_SYS_TENANT_ID != tenant_id) {
all_tenants_.push_back(tenant_id);
} else {
GCTX.omt_->get_tenant_ids(all_tenants_);
}
if (OB_FAIL(ret)) {
} else if (OB_FAIL(next_tenant(unused_tenant_guard))) {
if (OB_ITER_END != ret) {
STORAGE_LOG(WARN, "Fail to init tenant iterator", K(ret), K_(minor_info_idx));
}
} else {
is_opened_ = true;
major_info_idx_ = 0;
minor_info_idx_ = 0;
}
return ret;
}
int ObSSTableMergeInfoIterator::next_tenant(share::ObTenantSwitchGuard &tenant_guard)
{
int ret = OB_SUCCESS;
while (OB_SUCC(ret)) {
if (tenant_idx_ >= all_tenants_.size()) {
ret = OB_ITER_END;
} else if (FALSE_IT(cur_tenant_id_ = all_tenants_[tenant_idx_++])) {
} else if (is_virtual_tenant_id(cur_tenant_id_)) {
// skip virtual tenant
} else if (OB_FAIL(tenant_guard.switch_to(cur_tenant_id_))) {
if (OB_TENANT_NOT_IN_SERVER != ret) {
STORAGE_LOG(WARN, "switch tenant failed", K(ret), K(cur_tenant_id_));
} else {
ret = OB_SUCCESS;
continue;
}
} else {
major_info_idx_ = 0;
minor_info_idx_ = 0;
major_info_cnt_ = MTL(storage::ObTenantSSTableMergeInfoMgr *)->get_major_info_array_cnt();
minor_info_cnt_ = MTL(storage::ObTenantSSTableMergeInfoMgr *)->get_minor_info_array_cnt();
break;
}
}
return ret;
}
int ObSSTableMergeInfoIterator::get_next_merge_info(ObSSTableMergeInfo &merge_info)
{
int ret = OB_SUCCESS;
MAKE_TENANT_SWITCH_SCOPE_GUARD(tenant_guard);
if (!is_opened_) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "The ObSSTableMergeInfoIterator has not been inited", K(ret));
} else if (OB_FAIL(tenant_guard.switch_to(cur_tenant_id_))) {
STORAGE_LOG(WARN, "switch tenant failed", K(ret), K(cur_tenant_id_));
}
while (OB_SUCC(ret)) {
if (major_info_idx_ < major_info_cnt_) {
if (OB_FAIL(MTL(ObTenantSSTableMergeInfoMgr *)->get_major_info(major_info_idx_, merge_info))) {
STORAGE_LOG(WARN, "Fail to get merge info", K(ret), K_(major_info_idx));
} else {
major_info_idx_++;
break;
}
} else if (minor_info_idx_ < minor_info_cnt_) {
if (OB_FAIL(MTL(ObTenantSSTableMergeInfoMgr *)->get_minor_info(minor_info_idx_, merge_info))) {
STORAGE_LOG(WARN, "Fail to get merge info", K(ret), K_(minor_info_idx));
} else {
minor_info_idx_++;
break;
}
} else if (OB_FAIL(next_tenant(tenant_guard))) {
if (OB_ITER_END != ret) {
STORAGE_LOG(WARN, "Fail to next tenant", K(ret), K_(minor_info_idx));
}
}
} // end of while
return ret;
}
void ObSSTableMergeInfoIterator::reset()
{
all_tenants_.reset();
tenant_idx_ = 0;
cur_tenant_id_ = OB_INVALID_TENANT_ID;
major_info_idx_ = 0;
major_info_cnt_ = 0;
minor_info_idx_ = 0;
minor_info_cnt_ = 0;
is_opened_ = false;
}
/**
* ------------------------------------------------------------------ObTenantSSTableMergeInfoMgr---------------------------------------------------------------
*/
ObTenantSSTableMergeInfoMgr::ObTenantSSTableMergeInfoMgr()
: is_inited_(false),
allocator_(SET_USE_UNEXPECTED_500(ObModIds::OB_SSTABLE_MERGE_INFO), OB_MALLOC_BIG_BLOCK_SIZE),
major_merge_infos_(allocator_),
minor_merge_infos_(allocator_)
major_info_pool_(),
minor_info_pool_()
{
}
@ -166,30 +45,55 @@ int ObTenantSSTableMergeInfoMgr::mtl_init(ObTenantSSTableMergeInfoMgr *&sstable_
return sstable_merge_info->init();
}
int ObTenantSSTableMergeInfoMgr::init(const int64_t memory_limit)
int64_t ObTenantSSTableMergeInfoMgr::cal_max()
{
const uint64_t tenant_id = MTL_ID();
int64_t max_size = std::min(lib::get_tenant_memory_limit(tenant_id) * MEMORY_PERCENTAGE / 100,
static_cast<int64_t>(POOL_MAX_SIZE));
return max_size;
}
int ObTenantSSTableMergeInfoMgr::get_next_info(compaction::ObIDiagnoseInfoMgr::Iterator &major_iter,
compaction::ObIDiagnoseInfoMgr::Iterator &minor_iter,
ObSSTableMergeInfo &info, char *buf, const int64_t buf_len)
{
int ret = OB_SUCCESS;
if (OB_FAIL(major_iter.get_next(&info, buf, buf_len))) {
if (OB_ITER_END == ret) {
if (OB_FAIL(minor_iter.get_next(&info, buf, buf_len))) {
if (OB_ITER_END != ret) {
STORAGE_LOG(WARN, "failed to get next minor sstable merge info", K(ret));
}
}
} else {
STORAGE_LOG(WARN, "failed to get next major sstable merge info", K(ret));
}
}
return ret;
}
int ObTenantSSTableMergeInfoMgr::init(const int64_t page_size)
{
int ret = OB_SUCCESS;
if (IS_INIT) {
ret = OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObSSTableMergeInfo has already been initiated", K(ret));
} else if (OB_UNLIKELY(memory_limit <= 0)) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "Invalid argument", K(ret), K(memory_limit));
STORAGE_LOG(WARN, "ObTenantSSTableMergeInfoMgr has already been initiated", K(ret));
} else {
void *buf = NULL;
int64_t max_info_cnt = min(GMEMCONF.get_server_memory_limit(), memory_limit) / (sizeof(ObSSTableMergeInfo));
if (max_info_cnt < 2) {
max_info_cnt = 2;
}
const int64_t info_max_cnt = MAX(max_info_cnt / 2, 1);
if (OB_FAIL(major_merge_infos_.init(info_max_cnt))) {
STORAGE_LOG(WARN, "Fail to init major merge infos", K(ret), K(info_max_cnt));
} else if (OB_FAIL(minor_merge_infos_.init(info_max_cnt))) {
STORAGE_LOG(WARN, "Fail to alloc minor merge infos", K(ret), K(info_max_cnt));
int64_t max_size = cal_max();
if (OB_FAIL(major_info_pool_.init(false,
MTL_ID(),
"MajorMerge",
page_size,
max_size * (100 - MINOR_MEMORY_PERCENTAGE) / 100))) {
STORAGE_LOG(WARN, "failed to init major info pool", K(ret));
} else if (OB_FAIL(minor_info_pool_.init(false,
MTL_ID(),
"MinorMerge",
page_size,
max_size * MINOR_MEMORY_PERCENTAGE / 100))) {
STORAGE_LOG(WARN, "failed to init minor info pool", K(ret));
} else {
is_inited_ = true;
STORAGE_LOG(INFO, "Success to init ObTenantSSTableMergeInfoMgr", K(info_max_cnt));
}
}
@ -201,60 +105,87 @@ int ObTenantSSTableMergeInfoMgr::init(const int64_t memory_limit)
void ObTenantSSTableMergeInfoMgr::destroy()
{
major_merge_infos_.destroy();
minor_merge_infos_.destroy();
allocator_.reset();
is_inited_ = false;
STORAGE_LOG(INFO, "ObTenantSSTableMergeInfoMgr is destroyed");
if (IS_INIT) {
major_info_pool_.destroy();
minor_info_pool_.destroy();
is_inited_ =false;
STORAGE_LOG(INFO, "ObTenantSSTableMergeInfoMgr destroy finish");
}
}
int ObTenantSSTableMergeInfoMgr::add_sstable_merge_info(ObSSTableMergeInfo &merge_info)
int ObTenantSSTableMergeInfoMgr::open_iter(compaction::ObIDiagnoseInfoMgr::Iterator &major_iter,
compaction::ObIDiagnoseInfoMgr::Iterator &minor_iter)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTenantSSTableMergeInfoMgr is not initialized", K(ret));
} else if (OB_UNLIKELY(!merge_info.is_valid())) {
} else if (OB_FAIL(major_info_pool_.open_iter(major_iter))) {
STORAGE_LOG(WARN, "failed to open major iter", K(ret));
} else if (OB_FAIL(minor_info_pool_.open_iter(minor_iter))) {
STORAGE_LOG(WARN, "failed to open minor iter", K(ret));
}
return ret;
}
int ObTenantSSTableMergeInfoMgr::set_max(int64_t max_size)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTenantSSTableMergeInfoMgr is not init", K(ret));
} else if (OB_FAIL(major_info_pool_.set_max(max_size * (100 - MINOR_MEMORY_PERCENTAGE) / 100))) {
STORAGE_LOG(WARN, "failed to resize major info pool", K(ret), "max_size",
max_size * (100 - MINOR_MEMORY_PERCENTAGE) / 100);
} else if (OB_FAIL(minor_info_pool_.set_max(max_size * MINOR_MEMORY_PERCENTAGE / 100))) {
STORAGE_LOG(WARN, "failed to resize minor info pool", K(ret), "max_size",
max_size * MINOR_MEMORY_PERCENTAGE / 100);
}
return ret;
}
int ObTenantSSTableMergeInfoMgr::gc_info()
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTenantSSTableMergeInfoMgr is not init", K(ret));
} else if (OB_FAIL(major_info_pool_.gc_info())) {
STORAGE_LOG(WARN, "failed to gc major info pool", K(ret));
} else if (OB_FAIL(minor_info_pool_.gc_info())) {
STORAGE_LOG(WARN, "failed to gc minor info pool", K(ret));
}
return ret;
}
int ObTenantSSTableMergeInfoMgr::size()
{
int size = 0;
if (IS_INIT) {
size = minor_info_pool_.size() + major_info_pool_.size();
}
return size;
}
int ObTenantSSTableMergeInfoMgr::add_sstable_merge_info(ObSSTableMergeInfo &input_info)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "ObTenantSSTableMergeInfoMgr is not initialized", K(ret));
} else if (OB_UNLIKELY(!input_info.is_valid())) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "merge info is invalid", K(ret), K(merge_info));
} else if (merge_info.is_major_merge_type()) {
if (OB_FAIL(major_merge_infos_.add(merge_info))) {
STORAGE_LOG(WARN, "Fail to add into major merge info manager", K(ret), K(merge_info));
STORAGE_LOG(WARN, "invalid argument", K(ret), K(input_info));
} else {
compaction::ObIDiagnoseInfoMgr *info_pool = &minor_info_pool_;
if (input_info.is_major_merge_type()) {
info_pool = &major_info_pool_;
}
if (OB_FAIL(info_pool->alloc_and_add(0, &input_info))) {
STORAGE_LOG(WARN, "failed to add sstable merge info", K(ret), K(input_info));
}
} else if (OB_FAIL(minor_merge_infos_.add(merge_info))) {
STORAGE_LOG(WARN, "Fail to add into minor merge info manager", K(ret), K(merge_info));
}
if (OB_SUCC(ret)) {
LOG_INFO("success to add into merge info manager", K(ret), K(merge_info));
}
return ret;
}
int ObTenantSSTableMergeInfoMgr::get_major_info(const int64_t idx, ObSSTableMergeInfo &merge_info)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "The ObTenantSSTableMergeInfoMgr has not been inited", K(ret));
} else if (OB_FAIL(major_merge_infos_.get(idx, merge_info))) {
STORAGE_LOG(WARN, "failed to get info", K(ret), K(idx));
}
return ret;
}
int ObTenantSSTableMergeInfoMgr::get_minor_info(const int64_t idx, ObSSTableMergeInfo &merge_info)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
STORAGE_LOG(WARN, "The ObTenantSSTableMergeInfoMgr has not been inited", K(ret));
} else if (OB_FAIL(minor_merge_infos_.get(idx, merge_info))) {
STORAGE_LOG(WARN, "failed to get info", K(ret), K(idx));
}
return ret;
}
}//storage
}//oceanbase