hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

BUGFIX: limit max ls count of tenant

Browse Source
This commit is contained in:
obdev
2023-09-18 03:10:12 +00:00
committed by ob-robot
parent 8036cece48
commit e0b02386b8
9 changed files with 35 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
deps/oblib/src/lib/ob_define.h vendored
View File

@ -448,14 +448,12 @@ const int64_t OB_MAX_SERVER_SESSION_CNT = 32767;
const int64_t OB_MAX_SERVER_TENANT_CNT = 1000;
const int64_t OB_RECYCLE_MACRO_BLOCK_DURATION = 10 * 60 * 1000 * 1000LL; // 10 minutes
const int64_t OB_MINOR_FREEZE_TEAM_UP_INTERVAL = 2 * 60 * 60 * 1000 * 1000L; // 2 hours
const int64_t OB_MAX_PARTITION_NUM_PER_SERVER = 500000; // Version 3.1 is limited to 50w
const int64_t OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER = 10000;
// for define
const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER = 100;
const int64_t OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER = 10;
const int64_t OB_MAX_PG_NUM_PER_SERVER = 10000; // The total number of PG single servers 2.x version is first limited to 1w
const int64_t OB_MAX_SA_PARTITION_NUM_PER_SERVER = 500000; // stand alone partition single server 3.1 version is limited to 50w
const int64_t OB_MAX_PG_PARTITION_COUNT_PER_PG = 100000; // The maximum number of partitions in a single PG is limited to 10w
const int64_t OB_MAX_SPECIAL_LS_NUM = 1 + 1; // 1 for broadcast ls and 1 for sys ls
const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET = 1024; // the maximum of _max_ls_cnt_per_server
const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER = (10 * (OB_MAX_SPECIAL_LS_NUM + OB_MAX_MEMBER_NUMBER) > OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET ?
OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET : 10 * (OB_MAX_SPECIAL_LS_NUM + OB_MAX_MEMBER_NUMBER)); // magnification is 10x
const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT = 8; // the tenant that smaller than 4G will be limit to 8
const int64_t OB_MAX_TIME = 3020399000000;
// Max add partition member timeout.
// Used to make sure no member added after lease expired + %OB_MAX_ADD_MEMBER_TIMEOUT

15
src/share/ob_thread_define.h
View File

@ -24,8 +24,8 @@ TG_DEF(PartSerMigRetryQt, PartSerMigRetryQt, THREAD_POOL, 1)
// (!lib::is_mini_mode() ? OB_MAX_PARTITION_NUM_PER_SERVER : OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) * 2)
// TG_DEF(PartSerLargeCb, PartSerLargeCb, QUEUE_THREAD, ThreadCountPair(storage::ObCallbackQueueThread::QUEUE_THREAD_NUM, storage::ObCallbackQueueThread::MINI_MODE_QUEUE_THREAD_NUM),
// (!lib::is_mini_mode() ? OB_MAX_PARTITION_NUM_PER_SERVER : OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) * 2)
TG_DEF(ReplayEngine, ReplayEngine, QUEUE_THREAD, ThreadCountPair(sysconf(_SC_NPROCESSORS_ONLN), 2),
!lib::is_mini_mode() ? (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_PARTITION_NUM_PER_SERVER : (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER)
// TG_DEF(ReplayEngine, ReplayEngine, QUEUE_THREAD, ThreadCountPair(sysconf(_SC_NPROCESSORS_ONLN), 2),
// !lib::is_mini_mode() ? (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_PARTITION_NUM_PER_SERVER : (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER)
TG_DEF(TransMigrate, TransMigrate, QUEUE_THREAD, ThreadCountPair(GET_THREAD_NUM_BY_NPROCESSORS(24), 1), 10000)
TG_DEF(StandbyTimestampService, StandbyTimestampService, THREAD_POOL, 1)
TG_DEF(WeakReadService, WeakRdSrv, THREAD_POOL, 1)
@ -94,10 +94,8 @@ TG_DEF(LogIOTaskCbThreadPool, LogIOCb, QUEUE_THREAD,
ThreadCountPair(palf::LogIOTaskCbThreadPool::THREAD_NUM,
palf::LogIOTaskCbThreadPool::MINI_MODE_THREAD_NUM),
palf::LogIOTaskCbThreadPool::MAX_LOG_IO_CB_TASK_NUM)
TG_DEF(ReplayService, ReplaySrv, QUEUE_THREAD, 1,
!lib::is_mini_mode() ? (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER)
TG_DEF(LogRouteService, LogRouteSrv, QUEUE_THREAD, 1,
!lib::is_mini_mode() ? (common::MAX_SERVER_COUNT) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : (common::MAX_SERVER_COUNT) * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER)
TG_DEF(ReplayService, ReplaySrv, QUEUE_THREAD, 1, (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET)
TG_DEF(LogRouteService, LogRouteSrv, QUEUE_THREAD, 1, (common::MAX_SERVER_COUNT) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET)
TG_DEF(LogRouterTimer, LogRouterTimer, TIMER)
TG_DEF(LogFetcherLSWorker, LSWorker, MAP_QUEUE_THREAD, ThreadCountPair(4, 1))
TG_DEF(LogFetcherIdlePool, LSIdlePool, MAP_QUEUE_THREAD, 1)
@ -108,15 +106,14 @@ TG_DEF(LSFreeze, LSFreeze, QUEUE_THREAD, ThreadCountPair(storage::ObLSFreezeThre
storage::ObLSFreezeThread::MAX_FREE_TASK_NUM)
TG_DEF(LSFetchLogEngine, FetchLog, QUEUE_THREAD,
ThreadCountPair(palf::FetchLogEngine::FETCH_LOG_THREAD_COUNT, palf::FetchLogEngine::MINI_MODE_FETCH_LOG_THREAD_COUNT),
!lib::is_mini_mode() ? palf::FetchLogEngine::FETCH_LOG_TASK_MAX_COUNT_PER_LS * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : palf::FetchLogEngine::FETCH_LOG_TASK_MAX_COUNT_PER_LS * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER)
palf::FetchLogEngine::FETCH_LOG_TASK_MAX_COUNT_PER_LS * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET)
TG_DEF(DagScheduler, DagScheduler, THREAD_POOL, 1)
TG_DEF(DagWorker, DagWorker, THREAD_POOL, 1)
TG_DEF(RCService, RCSrv, QUEUE_THREAD,
ThreadCountPair(logservice::ObRoleChangeService::MAX_THREAD_NUM,
logservice::ObRoleChangeService::MAX_THREAD_NUM),
logservice::ObRoleChangeService::MAX_RC_EVENT_TASK)
TG_DEF(ApplyService, ApplySrv, QUEUE_THREAD, 1,
!lib::is_mini_mode() ? (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER)
TG_DEF(ApplyService, ApplySrv, QUEUE_THREAD, 1, (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET)
TG_DEF(GlobalCtxTimer, GlobalCtxTimer, TIMER)
TG_DEF(StorageLogWriter, StorageLogWriter, THREAD_POOL, 1)
TG_DEF(ReplayProcessStat, ReplayProcessStat, TIMER)

6
src/share/parameter/ob_parameter_seed.ipp
View File

@ -961,6 +961,12 @@ DEF_TIME(_cache_wash_interval, OB_CLUSTER_PARAMETER, "200ms", "[1ms, 3s]",
"specify interval of cache background wash",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
DEF_INT(_max_ls_cnt_per_server, OB_TENANT_PARAMETER, "0", "[0, 1024]",
"specify max ls count of one tenant on one observer."
"WARNING: Modifying this can potentially destabilize the cluster. It is strongly advised to avoid making such changes as they are unlikely to be necessary."
"0: the cluster will adapt the max ls number according to the memory size of tenant itself",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));
// TODO bin.lb: to be remove
DEF_CAP(dtl_buffer_size, OB_CLUSTER_PARAMETER, "64K", "[4K,2M]", "to be removed",
ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE));

1
src/storage/ls/ob_ls.h
View File

@ -168,6 +168,7 @@ struct DiagnoseInfo
class ObIComponentFactory;
enum class ObInnerLSStatus;
// sizeof(ObLS): 77248
class ObLS : public common::ObLink
{
public:

6
src/storage/meta_mem/ob_tenant_meta_mem_mgr.h
View File

@ -128,9 +128,9 @@ public:
static const int64_t MAX_TABLET_CNT_IN_OBJ_POOL = 50000;
static const int64_t MAX_MEMTABLE_CNT_IN_OBJ_POOL = 2 * MAX_TABLET_CNT_IN_OBJ_POOL;
static const int64_t MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL = MAX_MEMSTORE_CNT * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER;
static const int64_t MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL = OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER;
static const int64_t MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL = OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER;
static const int64_t MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL = MAX_MEMSTORE_CNT * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT;
static const int64_t MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT;
static const int64_t MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT;
static const int64_t MAX_DDL_KV_IN_OBJ_POOL = 5000;
static const int64_t TABLET_TRANSFORM_INTERVAL_US = 2 * 1000 * 1000L; // 2s

7
src/storage/tx_storage/ob_ls_map.cpp
View File

@ -215,12 +215,7 @@ int ObLSMap::add_ls(
if (OB_ISNULL(curr)) {
int64_t cnt = ATOMIC_AAF(&ls_cnt_, 1);
if (cnt > OB_MAX_LS_NUM_PER_TENANT_PER_SERVER) {
ATOMIC_DEC(&ls_cnt_);
ret = OB_TOO_MANY_TENANT_LS;
LOG_WARN("too many lss of a tenant", K(ret), K(cnt),
LITERAL_K(OB_MAX_LS_NUM_PER_TENANT_PER_SERVER));
} else if (OB_FAIL(ls.get_ref_mgr().inc(ObLSGetMod::TXSTORAGE_MOD))) {
if (OB_FAIL(ls.get_ref_mgr().inc(ObLSGetMod::TXSTORAGE_MOD))) {
ATOMIC_DEC(&ls_cnt_);
LOG_WARN("ls inc ref fail", K(ret), K(ls_id));
} else {

13
src/storage/tx_storage/ob_ls_service.cpp
View File

@ -281,12 +281,19 @@ int ObLSService::start()
int ObLSService::check_tenant_ls_num_()
{
int ret = OB_SUCCESS;
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
const int64_t normal_ls_count = ls_map_.get_ls_count();
const int64_t removeing_ls_count = ATOMIC_LOAD(&safe_ls_destroy_task_cnt_);
const int64_t tenant_memory = lib::get_tenant_memory_limit(MTL_ID());
const int64_t tenant_max_ls_limit = (tenant_memory > SMALL_TENANT_MEMORY_LIMIT ?
OB_MAX_LS_NUM_PER_TENANT_PER_SERVER :
OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT);
int64_t tenant_max_ls_limit = OB_MAX(tenant_memory - SMALL_TENANT_MEMORY_LIMIT, 0) / TENANT_MEMORY_PER_LS_NEED +
OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT;
// the max ls limit should not greater than OB_MAX_LS_NUM_PER_TENANT_PER_SERVER
tenant_max_ls_limit = OB_MIN(tenant_max_ls_limit, OB_MAX_LS_NUM_PER_TENANT_PER_SERVER);
// the config priority is higher than the calculate result.
if (OB_LIKELY(tenant_config.is_valid())) {
tenant_max_ls_limit = (tenant_config->_max_ls_cnt_per_server != 0 ?
tenant_config->_max_ls_cnt_per_server : tenant_max_ls_limit);
}
if (normal_ls_count + removeing_ls_count + 1 > tenant_max_ls_limit) {
ret = OB_TOO_MANY_TENANT_LS;
LOG_WARN("too many ls of a tenant", K(ret), K(normal_ls_count), K(removeing_ls_count),

4
src/storage/tx_storage/ob_ls_service.h
View File

@ -47,8 +47,8 @@ struct ObLSMeta;
class ObLSService
{
static const int64_t DEFAULT_LOCK_TIMEOUT = 60_s;
static const int64_t SMALL_TENANT_MEMORY_LIMIT = 5 * 1024 * 1024 * 1024L; // 5G
static const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT = 8;
static const int64_t SMALL_TENANT_MEMORY_LIMIT = 4 * 1024 * 1024 * 1024L; // 4G
static const int64_t TENANT_MEMORY_PER_LS_NEED = 200 * 1024 * 1024L; // 200MB
public:
ObLSService();
~ObLSService();

1
tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result
View File

@ -311,6 +311,7 @@ _log_writer_parallelism
_ls_gc_wait_readonly_tx_time
_ls_migration_wait_completing_timeout
_max_elr_dependent_trx_count
_max_ls_cnt_per_server
_max_malloc_sample_interval
_max_rpc_packet_size
_max_schema_slot_num