hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Physical Standby] Fix the gc strategy of the LogFetcher

Browse Source
This commit is contained in:
obdev
2023-05-08 04:40:31 +00:00
committed by ob-robot
parent 5cec66d5f1
commit d71599deb3
10 changed files with 44 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
deps/oblib/src/lib/objectpool/ob_small_obj_pool.h vendored
View File

@ -56,6 +56,8 @@ public:
int64_t get_free_count() const { return free_count_; }
int64_t get_alloc_count() const { return alloc_count_; }
int64_t get_fixed_count() const { return fixed_count_; }
int64_t get_cached_total_count() const { return free_list_.get_total(); }
int64_t get_cached_free_count() const { return free_list_.get_free(); }
public:
int init(const int64_t fixed_count = DEFAULT_FIXED_COUNT,

25
src/logservice/logfetcher/ob_log_fetch_log_rpc.cpp
View File

@ -1349,20 +1349,22 @@ int FetchLogARpcResult::set(const obrpc::ObRpcResultCode &rcode,
////////////////////////////// FetchLogARpcResult Object Pool //////////////////////////////
int FetchLogARpcResultPool::init(const int64_t cached_obj_count)
int FetchLogARpcResultPool::init(const uint64_t tenant_id, const int64_t cached_obj_count)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(inited_)) {
LOG_ERROR("init twice");
ret = OB_INIT_TWICE;
LOG_ERROR("init twice");
} else if (OB_FAIL(pool_.init(cached_obj_count,
ObModIds::OB_LOG_FETCH_LOG_ARPC_RESULT,
OB_SERVER_TENANT_ID,
tenant_id,
DEFAULT_RESULT_POOL_BLOCK_SIZE))) {
LOG_ERROR("init result obj pool fail", KR(ret), K(cached_obj_count));
LOG_ERROR("init result obj pool fail", KR(ret), K(tenant_id), K(cached_obj_count));
} else {
inited_ = true;
}
return ret;
}
@ -1374,14 +1376,15 @@ void FetchLogARpcResultPool::destroy()
void FetchLogARpcResultPool::print_stat()
{
int64_t alloc_count = pool_.get_alloc_count();
int64_t free_count = pool_.get_free_count();
int64_t fixed_count = pool_.get_fixed_count();
int64_t used_count = alloc_count - free_count;
int64_t dynamic_count = (alloc_count > fixed_count) ? alloc_count - fixed_count : 0;
const int64_t alloc_count = pool_.get_alloc_count();
const int64_t free_count = pool_.get_free_count();
const int64_t fixed_count = pool_.get_fixed_count();
const int64_t used_count = alloc_count - free_count;
const int64_t dynamic_count = (alloc_count > fixed_count) ? alloc_count - fixed_count : 0;
const int64_t cached_total_count = pool_.get_cached_total_count();
_LOG_INFO("[STAT] [RPC_RESULT_POOL] USED=%ld FREE=%ld FIXED=%ld DYNAMIC=%ld",
used_count, free_count, fixed_count, dynamic_count);
_LOG_INFO("[STAT] [RPC_RESULT_POOL] USED=%ld ALLOC=%ld FREE=%ld FIXED=%ld DYNAMIC=%ld CACHED_TOTAL_COUNT=%ld",
used_count, alloc_count, free_count, fixed_count, dynamic_count, cached_total_count);
}
int FetchLogARpcResultPool::alloc(FetchLogARpcResult *&result)

6
src/logservice/logfetcher/ob_log_fetch_log_rpc.h
View File

@ -500,16 +500,14 @@ class FetchLogARpcResultPool : public IFetchLogARpcResultPool
{
typedef common::ObSmallObjPool<FetchLogARpcResult> ResultPool;
// 16M
//static const int64_t DEFAULT_RESULT_POOL_BLOCK_SIZE = 1L << 24;
// 32M
static const int64_t DEFAULT_RESULT_POOL_BLOCK_SIZE = 1L << 25;
static const int64_t DEFAULT_RESULT_POOL_BLOCK_SIZE = (1 << 24L) + (1 << 20L);
public:
FetchLogARpcResultPool() : inited_(false), pool_() {}
virtual ~FetchLogARpcResultPool() { destroy(); }
public:
int init(const int64_t cached_obj_count);
int init(const uint64_t tenant_id, const int64_t cached_obj_count);
void destroy();
void print_stat();

9
src/logservice/logfetcher/ob_log_fetch_stream_container_mgr.cpp
View File

@ -37,7 +37,9 @@ ObFsContainerMgr::~ObFsContainerMgr()
destroy();
}
int ObFsContainerMgr::init(const int64_t svr_stream_cached_count,
int ObFsContainerMgr::init(
const uint64_t tenant_id,
const int64_t svr_stream_cached_count,
const int64_t fetch_stream_cached_count,
const int64_t rpc_result_cached_count,
IObLogRpc &rpc,
@ -59,8 +61,8 @@ int ObFsContainerMgr::init(const int64_t svr_stream_cached_count,
LOG_ERROR("init FetchStreamContainer pool fail", KR(ret));
} else if (OB_FAIL(fs_pool_.init(fetch_stream_cached_count))) {
LOG_ERROR("init fetch stream pool fail", KR(ret), K(fetch_stream_cached_count));
} else if (OB_FAIL(rpc_result_pool_.init(rpc_result_cached_count))) {
LOG_ERROR("init rpc result pool fail", KR(ret), K(rpc_result_cached_count));
} else if (OB_FAIL(rpc_result_pool_.init(tenant_id, rpc_result_cached_count))) {
LOG_ERROR("init rpc result pool fail", KR(ret), K(tenant_id), K(rpc_result_cached_count));
} else {
rpc_ = &rpc;
stream_worker_ = &stream_worker;
@ -84,6 +86,7 @@ void ObFsContainerMgr::destroy()
(void)fsc_map_.destroy();
fsc_pool_.destroy();
fs_pool_.destroy();
rpc_result_pool_.destroy();
}
}

4
src/logservice/logfetcher/ob_log_fetch_stream_container_mgr.h
View File

@ -49,7 +49,9 @@ class ObFsContainerMgr : public IObFsContainerMgr
public:
ObFsContainerMgr();
virtual ~ObFsContainerMgr();
int init(const int64_t svr_stream_cached_count,
int init(
const uint64_t tenant_id,
const int64_t svr_stream_cached_count,
const int64_t fetch_stream_cached_count,
const int64_t rpc_result_cached_count,
IObLogRpc &rpc,

1
src/logservice/logfetcher/ob_log_fetcher.cpp
View File

@ -157,6 +157,7 @@ int ObLogFetcher::init(
*err_handler))) {
LOG_ERROR("init stream worker fail", KR(ret));
} else if (OB_FAIL(fs_container_mgr_.init(
tenant_id,
cfg.svr_stream_cached_count,
cfg.fetch_stream_cached_count,
cfg.rpc_result_cached_count,

2
src/logservice/logfetcher/ob_log_fetcher_dead_pool.cpp
View File

@ -160,7 +160,7 @@ void ObLogFetcherDeadPool::handle(void *data, volatile bool &stop_flag)
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("ls_fetch_ctx is nullptt", KR(ret), K(thread_index), K(get_thread_cnt()));
} else {
LOG_INFO("fetcher dead pool thread start", K(thread_index));
LOG_INFO("fetcher dead pool thread start", K(thread_index), "tls_id", ls_fetch_ctx->get_tls_id());
if (OB_FAIL(handle_task_list_(thread_index, *ls_fetch_ctx, stop_flag))) {
if (OB_IN_STOP_STATE != ret) {

16
src/logservice/logfetcher/ob_log_fetcher_idle_pool.cpp
View File

@ -167,8 +167,7 @@ void ObLogFetcherIdlePool::handle(void *data, volatile bool &stop_flag)
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("ls_fetch_ctx is nullptt", KR(ret), K(thread_index), K(get_thread_cnt()));
} else {
// TODO remvoe
LOG_INFO("fetcher idle pool thread start", K(thread_index));
LOG_INFO("fetcher idle pool thread start", K(thread_index), "tls_id", ls_fetch_ctx->get_tls_id());
if (OB_FAIL(do_request_(thread_index, *ls_fetch_ctx))) {
if (OB_IN_STOP_STATE != ret) {
@ -199,8 +198,10 @@ int ObLogFetcherIdlePool::do_request_(const int64_t thread_index, LSFetchCtx &ls
ret = OB_INVALID_ERROR;
} else {
bool need_dispatch = false;
int64_t retry_count = 0;
const int64_t MAX_RETRY_COUNT = 5;
while (OB_SUCC(ret) && ! need_dispatch) {
while (OB_SUCC(ret) && ! need_dispatch && (retry_count < MAX_RETRY_COUNT)) {
if (OB_FAIL(handle_task_(&ls_fetch_ctx, need_dispatch))) {
LOG_ERROR("handle task fail", KR(ret), K(ls_fetch_ctx));
} else if (need_dispatch) {
@ -216,6 +217,13 @@ int ObLogFetcherIdlePool::do_request_(const int64_t thread_index, LSFetchCtx &ls
} else {
// sleep
ob_usleep(IDLE_WAIT_TIME);
++retry_count;
if (MAX_RETRY_COUNT == retry_count) {
if (OB_FAIL(push(&ls_fetch_ctx))) {
LOG_ERROR("ObLogFetcherIdlePool push failed", KR(ret));
}
}
}
} // while
}
@ -239,7 +247,7 @@ int ObLogFetcherIdlePool::handle_task_(LSFetchCtx *task, bool &need_dispatch)
} else if (OB_UNLIKELY(task->is_discarded())) {
// If a task is deleted, assign it directly and recycle it during the assignment process
need_dispatch = true;
LOG_DEBUG("[STAT] [IDLE_POOL] [RECYCLE_FETCH_TASK]", K(task), KPC(task));
LOG_INFO("[STAT] [IDLE_POOL] [RECYCLE_FETCH_TASK]", K(task), KPC(task));
} else if (OB_ISNULL(cfg_)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("invalid ObLogFetcherConfig", KR(ret), K(cfg_));

2
src/logservice/logfetcher/ob_log_rpc.cpp
View File

@ -51,7 +51,7 @@
int64_t max_rpc_proc_time = \
ATOMIC_LOAD(&ObLogRpc::g_rpc_process_handler_time_upper_limit); \
proxy.set_server((SVR)); \
if (OB_FAIL(proxy.dst_cluster_id(cluster_id_).by(tenant_id).trace_time(true).timeout((TIMEOUT))\
if (OB_FAIL(proxy.dst_cluster_id(cluster_id_).by(tenant_id).group_id(share::OBCG_CDCSERVICE).trace_time(true).timeout((TIMEOUT))\
.max_process_handler_time(static_cast<int32_t>(max_rpc_proc_time))\
.RPC((REQ), (ARG)))) { \
LOG_ERROR("rpc fail: " #RPC, "tenant_id", tenant_id, "svr", (SVR), "rpc_ret", ret, \

4
src/logservice/logfetcher/ob_ls_worker.cpp
View File

@ -198,10 +198,10 @@ int ObLSWorker::dispatch_fetch_task(LSFetchCtx &task, const char *dispatch_reaso
}
// Recycle deleted partitions and add them to DEAD POOL
else if (OB_UNLIKELY(task.is_discarded())) {
LOG_DEBUG("[STAT] [STREAM_WORKER] [RECYCLE_FETCH_TASK]", "task", &task, K(task));
LOG_INFO("[STAT] [STREAM_WORKER] [RECYCLE_FETCH_TASK]", "task", &task, K(task));
if (OB_FAIL(dead_pool_->push(&task))) {
LOG_DEBUG("push task into dead pool fail", KR(ret), K(task));
LOG_ERROR("push task into dead pool fail", KR(ret), K(task));
}
} else if (is_integrated_fetching_mode(task.get_fetching_mode())) {
ObAddr request_svr;