diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index ee0a0ed91a..da6bf58b55 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -448,14 +448,12 @@ const int64_t OB_MAX_SERVER_SESSION_CNT = 32767; const int64_t OB_MAX_SERVER_TENANT_CNT = 1000; const int64_t OB_RECYCLE_MACRO_BLOCK_DURATION = 10 * 60 * 1000 * 1000LL; // 10 minutes const int64_t OB_MINOR_FREEZE_TEAM_UP_INTERVAL = 2 * 60 * 60 * 1000 * 1000L; // 2 hours -const int64_t OB_MAX_PARTITION_NUM_PER_SERVER = 500000; // Version 3.1 is limited to 50w -const int64_t OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER = 10000; // for define -const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER = 100; -const int64_t OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER = 10; -const int64_t OB_MAX_PG_NUM_PER_SERVER = 10000; // The total number of PG single servers 2.x version is first limited to 1w -const int64_t OB_MAX_SA_PARTITION_NUM_PER_SERVER = 500000; // stand alone partition single server 3.1 version is limited to 50w -const int64_t OB_MAX_PG_PARTITION_COUNT_PER_PG = 100000; // The maximum number of partitions in a single PG is limited to 10w +const int64_t OB_MAX_SPECIAL_LS_NUM = 1 + 1; // 1 for broadcast ls and 1 for sys ls +const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET = 1024; // the maximum of _max_ls_cnt_per_server +const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER = (10 * (OB_MAX_SPECIAL_LS_NUM + OB_MAX_MEMBER_NUMBER) > OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET ? + OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET : 10 * (OB_MAX_SPECIAL_LS_NUM + OB_MAX_MEMBER_NUMBER)); // magnification is 10x +const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT = 8; // the tenant that smaller than 4G will be limit to 8 const int64_t OB_MAX_TIME = 3020399000000; // Max add partition member timeout. // Used to make sure no member added after lease expired + %OB_MAX_ADD_MEMBER_TIMEOUT diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h index 2bc493c9ef..c544db7ff5 100755 --- a/src/share/ob_thread_define.h +++ b/src/share/ob_thread_define.h @@ -24,8 +24,8 @@ TG_DEF(PartSerMigRetryQt, PartSerMigRetryQt, THREAD_POOL, 1) // (!lib::is_mini_mode() ? OB_MAX_PARTITION_NUM_PER_SERVER : OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) * 2) // TG_DEF(PartSerLargeCb, PartSerLargeCb, QUEUE_THREAD, ThreadCountPair(storage::ObCallbackQueueThread::QUEUE_THREAD_NUM, storage::ObCallbackQueueThread::MINI_MODE_QUEUE_THREAD_NUM), // (!lib::is_mini_mode() ? OB_MAX_PARTITION_NUM_PER_SERVER : OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) * 2) -TG_DEF(ReplayEngine, ReplayEngine, QUEUE_THREAD, ThreadCountPair(sysconf(_SC_NPROCESSORS_ONLN), 2), - !lib::is_mini_mode() ? (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_PARTITION_NUM_PER_SERVER : (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) +// TG_DEF(ReplayEngine, ReplayEngine, QUEUE_THREAD, ThreadCountPair(sysconf(_SC_NPROCESSORS_ONLN), 2), +// !lib::is_mini_mode() ? (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_PARTITION_NUM_PER_SERVER : (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_PARTITION_NUM_PER_SERVER) TG_DEF(TransMigrate, TransMigrate, QUEUE_THREAD, ThreadCountPair(GET_THREAD_NUM_BY_NPROCESSORS(24), 1), 10000) TG_DEF(StandbyTimestampService, StandbyTimestampService, THREAD_POOL, 1) TG_DEF(WeakReadService, WeakRdSrv, THREAD_POOL, 1) @@ -94,10 +94,8 @@ TG_DEF(LogIOTaskCbThreadPool, LogIOCb, QUEUE_THREAD, ThreadCountPair(palf::LogIOTaskCbThreadPool::THREAD_NUM, palf::LogIOTaskCbThreadPool::MINI_MODE_THREAD_NUM), palf::LogIOTaskCbThreadPool::MAX_LOG_IO_CB_TASK_NUM) -TG_DEF(ReplayService, ReplaySrv, QUEUE_THREAD, 1, - !lib::is_mini_mode() ? (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER) -TG_DEF(LogRouteService, LogRouteSrv, QUEUE_THREAD, 1, - !lib::is_mini_mode() ? (common::MAX_SERVER_COUNT) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : (common::MAX_SERVER_COUNT) * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER) +TG_DEF(ReplayService, ReplaySrv, QUEUE_THREAD, 1, (common::REPLAY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET) +TG_DEF(LogRouteService, LogRouteSrv, QUEUE_THREAD, 1, (common::MAX_SERVER_COUNT) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET) TG_DEF(LogRouterTimer, LogRouterTimer, TIMER) TG_DEF(LogFetcherLSWorker, LSWorker, MAP_QUEUE_THREAD, ThreadCountPair(4, 1)) TG_DEF(LogFetcherIdlePool, LSIdlePool, MAP_QUEUE_THREAD, 1) @@ -108,15 +106,14 @@ TG_DEF(LSFreeze, LSFreeze, QUEUE_THREAD, ThreadCountPair(storage::ObLSFreezeThre storage::ObLSFreezeThread::MAX_FREE_TASK_NUM) TG_DEF(LSFetchLogEngine, FetchLog, QUEUE_THREAD, ThreadCountPair(palf::FetchLogEngine::FETCH_LOG_THREAD_COUNT, palf::FetchLogEngine::MINI_MODE_FETCH_LOG_THREAD_COUNT), - !lib::is_mini_mode() ? palf::FetchLogEngine::FETCH_LOG_TASK_MAX_COUNT_PER_LS * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : palf::FetchLogEngine::FETCH_LOG_TASK_MAX_COUNT_PER_LS * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER) + palf::FetchLogEngine::FETCH_LOG_TASK_MAX_COUNT_PER_LS * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET) TG_DEF(DagScheduler, DagScheduler, THREAD_POOL, 1) TG_DEF(DagWorker, DagWorker, THREAD_POOL, 1) TG_DEF(RCService, RCSrv, QUEUE_THREAD, ThreadCountPair(logservice::ObRoleChangeService::MAX_THREAD_NUM, logservice::ObRoleChangeService::MAX_THREAD_NUM), logservice::ObRoleChangeService::MAX_RC_EVENT_TASK) -TG_DEF(ApplyService, ApplySrv, QUEUE_THREAD, 1, - !lib::is_mini_mode() ? (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER) +TG_DEF(ApplyService, ApplySrv, QUEUE_THREAD, 1, (common::APPLY_TASK_QUEUE_SIZE + 1) * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_CAN_BE_SET) TG_DEF(GlobalCtxTimer, GlobalCtxTimer, TIMER) TG_DEF(StorageLogWriter, StorageLogWriter, THREAD_POOL, 1) TG_DEF(ReplayProcessStat, ReplayProcessStat, TIMER) diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 6c60850d55..7fd93b98de 100755 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -961,6 +961,12 @@ DEF_TIME(_cache_wash_interval, OB_CLUSTER_PARAMETER, "200ms", "[1ms, 3s]", "specify interval of cache background wash", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +DEF_INT(_max_ls_cnt_per_server, OB_TENANT_PARAMETER, "0", "[0, 1024]", + "specify max ls count of one tenant on one observer." + "WARNING: Modifying this can potentially destabilize the cluster. It is strongly advised to avoid making such changes as they are unlikely to be necessary." + "0: the cluster will adapt the max ls number according to the memory size of tenant itself", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + // TODO bin.lb: to be remove DEF_CAP(dtl_buffer_size, OB_CLUSTER_PARAMETER, "64K", "[4K,2M]", "to be removed", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 6b5d405957..de61da6e65 100755 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -168,6 +168,7 @@ struct DiagnoseInfo class ObIComponentFactory; enum class ObInnerLSStatus; +// sizeof(ObLS): 77248 class ObLS : public common::ObLink { public: diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h index f57adda3ca..50f1618ecb 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h @@ -128,9 +128,9 @@ public: static const int64_t MAX_TABLET_CNT_IN_OBJ_POOL = 50000; static const int64_t MAX_MEMTABLE_CNT_IN_OBJ_POOL = 2 * MAX_TABLET_CNT_IN_OBJ_POOL; - static const int64_t MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL = MAX_MEMSTORE_CNT * OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER; - static const int64_t MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL = OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER; - static const int64_t MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL = OB_MINI_MODE_MAX_LS_NUM_PER_TENANT_PER_SERVER; + static const int64_t MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL = MAX_MEMSTORE_CNT * OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; + static const int64_t MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; + static const int64_t MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL = OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; static const int64_t MAX_DDL_KV_IN_OBJ_POOL = 5000; static const int64_t TABLET_TRANSFORM_INTERVAL_US = 2 * 1000 * 1000L; // 2s diff --git a/src/storage/tx_storage/ob_ls_map.cpp b/src/storage/tx_storage/ob_ls_map.cpp index 02697d1d6a..e1c272b51e 100644 --- a/src/storage/tx_storage/ob_ls_map.cpp +++ b/src/storage/tx_storage/ob_ls_map.cpp @@ -215,12 +215,7 @@ int ObLSMap::add_ls( if (OB_ISNULL(curr)) { int64_t cnt = ATOMIC_AAF(&ls_cnt_, 1); - if (cnt > OB_MAX_LS_NUM_PER_TENANT_PER_SERVER) { - ATOMIC_DEC(&ls_cnt_); - ret = OB_TOO_MANY_TENANT_LS; - LOG_WARN("too many lss of a tenant", K(ret), K(cnt), - LITERAL_K(OB_MAX_LS_NUM_PER_TENANT_PER_SERVER)); - } else if (OB_FAIL(ls.get_ref_mgr().inc(ObLSGetMod::TXSTORAGE_MOD))) { + if (OB_FAIL(ls.get_ref_mgr().inc(ObLSGetMod::TXSTORAGE_MOD))) { ATOMIC_DEC(&ls_cnt_); LOG_WARN("ls inc ref fail", K(ret), K(ls_id)); } else { diff --git a/src/storage/tx_storage/ob_ls_service.cpp b/src/storage/tx_storage/ob_ls_service.cpp index bc0beba0ff..9d4c284f44 100644 --- a/src/storage/tx_storage/ob_ls_service.cpp +++ b/src/storage/tx_storage/ob_ls_service.cpp @@ -281,12 +281,19 @@ int ObLSService::start() int ObLSService::check_tenant_ls_num_() { int ret = OB_SUCCESS; + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); const int64_t normal_ls_count = ls_map_.get_ls_count(); const int64_t removeing_ls_count = ATOMIC_LOAD(&safe_ls_destroy_task_cnt_); const int64_t tenant_memory = lib::get_tenant_memory_limit(MTL_ID()); - const int64_t tenant_max_ls_limit = (tenant_memory > SMALL_TENANT_MEMORY_LIMIT ? - OB_MAX_LS_NUM_PER_TENANT_PER_SERVER : - OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT); + int64_t tenant_max_ls_limit = OB_MAX(tenant_memory - SMALL_TENANT_MEMORY_LIMIT, 0) / TENANT_MEMORY_PER_LS_NEED + + OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT; + // the max ls limit should not greater than OB_MAX_LS_NUM_PER_TENANT_PER_SERVER + tenant_max_ls_limit = OB_MIN(tenant_max_ls_limit, OB_MAX_LS_NUM_PER_TENANT_PER_SERVER); + // the config priority is higher than the calculate result. + if (OB_LIKELY(tenant_config.is_valid())) { + tenant_max_ls_limit = (tenant_config->_max_ls_cnt_per_server != 0 ? + tenant_config->_max_ls_cnt_per_server : tenant_max_ls_limit); + } if (normal_ls_count + removeing_ls_count + 1 > tenant_max_ls_limit) { ret = OB_TOO_MANY_TENANT_LS; LOG_WARN("too many ls of a tenant", K(ret), K(normal_ls_count), K(removeing_ls_count), diff --git a/src/storage/tx_storage/ob_ls_service.h b/src/storage/tx_storage/ob_ls_service.h index 40c2df2ddd..a5836be945 100644 --- a/src/storage/tx_storage/ob_ls_service.h +++ b/src/storage/tx_storage/ob_ls_service.h @@ -47,8 +47,8 @@ struct ObLSMeta; class ObLSService { static const int64_t DEFAULT_LOCK_TIMEOUT = 60_s; - static const int64_t SMALL_TENANT_MEMORY_LIMIT = 5 * 1024 * 1024 * 1024L; // 5G - static const int64_t OB_MAX_LS_NUM_PER_TENANT_PER_SERVER_FOR_SMALL_TENANT = 8; + static const int64_t SMALL_TENANT_MEMORY_LIMIT = 4 * 1024 * 1024 * 1024L; // 4G + static const int64_t TENANT_MEMORY_PER_LS_NEED = 200 * 1024 * 1024L; // 200MB public: ObLSService(); ~ObLSService(); diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result index ed5e3dcfc2..72731072ff 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/all_virtual_sys_parameter_stat.result @@ -311,6 +311,7 @@ _log_writer_parallelism _ls_gc_wait_readonly_tx_time _ls_migration_wait_completing_timeout _max_elr_dependent_trx_count +_max_ls_cnt_per_server _max_malloc_sample_interval _max_rpc_packet_size _max_schema_slot_num