hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

273
deps/oblib/src/common/ob_queue_thread.cpp vendored
View File

@ -15,9 +15,13 @@
#include "common/ob_queue_thread.h"
#include "lib/utility/utility.h"
namespace oceanbase {
namespace common {
ObCond::ObCond(const int64_t spin_wait_num) : spin_wait_num_(spin_wait_num), bcond_(false), last_waked_time_(0)
namespace oceanbase
{
namespace common
{
ObCond::ObCond(const int64_t spin_wait_num) : spin_wait_num_(spin_wait_num),
bcond_(false),
last_waked_time_(0)
{
pthread_mutex_init(&mutex_, NULL);
if (0 != pthread_cond_init(&cond_, NULL)) {
@ -46,7 +50,7 @@ int ObCond::wait()
int ret = OB_SUCCESS;
bool need_wait = true;
if ((last_waked_time_ + BUSY_INTERVAL) > ::oceanbase::common::ObTimeUtility::current_time()) {
// return OB_SUCCESS;
//return OB_SUCCESS;
for (int64_t i = 0; i < spin_wait_num_; i++) {
if (true == ATOMIC_CAS(&bcond_, true, false)) {
need_wait = false;
@ -107,10 +111,13 @@ int ObCond::timedwait(const int64_t time_us)
////////////////////////////////////////////////////////////////////////////////////////////////////
S2MQueueThread::S2MQueueThread()
: thread_num_(0), thread_conf_iter_(0), thread_conf_lock_(), queued_num_(), queue_rebalance_(false)
S2MQueueThread::S2MQueueThread() : thread_num_(0),
thread_conf_iter_(0),
thread_conf_lock_(),
queued_num_(),
queue_rebalance_(false)
{
memset((void*)each_queue_len_, 0, sizeof(each_queue_len_));
memset((void *)each_queue_len_, 0, sizeof(each_queue_len_));
}
S2MQueueThread::~S2MQueueThread()
@ -118,18 +125,22 @@ S2MQueueThread::~S2MQueueThread()
destroy();
}
int S2MQueueThread::init(
const int64_t thread_num, const int64_t task_num_limit, const bool queue_rebalance, const bool dynamic_rebalance)
int S2MQueueThread::init(const int64_t thread_num, const int64_t task_num_limit,
const bool queue_rebalance, const bool dynamic_rebalance)
{
int ret = OB_SUCCESS;
queue_rebalance_ = queue_rebalance;
if (0 != thread_num_) {
ret = OB_INIT_TWICE;
} else if (0 >= thread_num || MAX_THREAD_NUM < thread_num || 0 >= task_num_limit) {
_OB_LOG(WARN, "invalid param, thread_num=%ld task_num_limit=%ld", thread_num, task_num_limit);
} else if (0 >= thread_num
|| MAX_THREAD_NUM < thread_num
|| 0 >= task_num_limit) {
_OB_LOG(WARN, "invalid param, thread_num=%ld task_num_limit=%ld",
thread_num, task_num_limit);
ret = OB_INVALID_ARGUMENT;
} else if (OB_SUCCESS != (ret = launch_thread_(thread_num, task_num_limit))) {
_OB_LOG(WARN, "launch thread fail, ret=%d thread_num=%ld task_num_limit=%ld", ret, thread_num, task_num_limit);
_OB_LOG(WARN, "launch thread fail, ret=%d thread_num=%ld task_num_limit=%ld", ret, thread_num,
task_num_limit);
} else if (OB_SUCCESS != (ret = balance_filter_.init(thread_num_, dynamic_rebalance))) {
_OB_LOG(WARN, "init balance_filter fail, ret=%d thread_num=%ld", ret, thread_num_);
} else {
@ -146,14 +157,14 @@ int S2MQueueThread::init(
void S2MQueueThread::destroy()
{
for (int64_t i = 0; i < thread_num_; i++) {
ThreadConf& tc = thread_conf_array_[i];
ThreadConf &tc = thread_conf_array_[i];
tc.run_flag = false;
tc.stop_flag = true;
tc.queue_cond.signal();
pthread_join(tc.pd, NULL);
}
for (int64_t i = 0; i < thread_num_; i++) {
ThreadConf& tc = thread_conf_array_[i];
ThreadConf &tc = thread_conf_array_[i];
tc.high_prio_task_queue.destroy();
tc.spec_task_queue.destroy();
tc.comm_task_queue.destroy();
@ -180,7 +191,7 @@ int S2MQueueThread::wakeup()
return err;
}
int S2MQueueThread::push(void* task, const int64_t prio)
int S2MQueueThread::push(void *task, const int64_t prio)
{
int ret = OB_SUCCESS;
int queue_id = -1;
@ -191,7 +202,7 @@ int S2MQueueThread::push(void* task, const int64_t prio)
} else {
RDLockGuard guard(thread_conf_lock_);
uint64_t i = ATOMIC_AAF(&thread_conf_iter_, 1);
ThreadConf& tc = thread_conf_array_[i % thread_num_];
ThreadConf &tc = thread_conf_array_[i % thread_num_];
switch (prio) {
case HIGH_PRIV:
ret = tc.high_prio_task_queue.push(task);
@ -224,7 +235,7 @@ int S2MQueueThread::push(void* task, const int64_t prio)
return ret;
}
int S2MQueueThread::push(void* task, const uint64_t task_sign, const int64_t prio)
int S2MQueueThread::push(void *task, const uint64_t task_sign, const int64_t prio)
{
int ret = OB_SUCCESS;
if (OB_INVALID_ID == task_sign) {
@ -236,8 +247,9 @@ int S2MQueueThread::push(void* task, const uint64_t task_sign, const int64_t pri
} else {
uint64_t filted_task_sign = balance_filter_.filt(task_sign);
RDLockGuard guard(thread_conf_lock_);
ThreadConf& tc = thread_conf_array_[filted_task_sign % thread_num_];
_OB_LOG(TRACE, "req_sign=%lu,%lu hash=%lu", filted_task_sign % thread_num_, filted_task_sign, task_sign);
ThreadConf &tc = thread_conf_array_[filted_task_sign % thread_num_];
_OB_LOG(TRACE, "req_sign=%lu,%lu hash=%lu", filted_task_sign % thread_num_, filted_task_sign,
task_sign);
if (OB_SUCCESS != (ret = tc.spec_task_queue.push(task))) {
//_OB_LOG(WARN, "push task to queue fail, ret=%d", ret);
if (OB_SIZE_OVERFLOW == ret) {
@ -252,7 +264,7 @@ int S2MQueueThread::push(void* task, const uint64_t task_sign, const int64_t pri
return ret;
}
int S2MQueueThread::set_prio_quota(v4si& quota)
int S2MQueueThread::set_prio_quota(v4si &quota)
{
int ret = OB_SUCCESS;
if (0 >= thread_num_) {
@ -260,7 +272,7 @@ int S2MQueueThread::set_prio_quota(v4si& quota)
} else {
WRLockGuard guard(thread_conf_lock_);
for (int64_t i = 0; i < thread_num_; i++) {
ThreadConf& tc = thread_conf_array_[i];
ThreadConf &tc = thread_conf_array_[i];
tc.scheduler_.set_quota(quota);
}
}
@ -293,8 +305,10 @@ int S2MQueueThread::sub_thread(const int64_t thread_num)
ret = OB_NOT_INIT;
} else {
WRLockGuard guard(thread_conf_lock_);
if (0 >= thread_num || thread_num >= thread_num_) {
_OB_LOG(WARN, "invalid param, thread_num=%ld thread_num_=%ld", thread_num, thread_num_);
if (0 >= thread_num
|| thread_num >= thread_num_) {
_OB_LOG(WARN, "invalid param, thread_num=%ld thread_num_=%ld",
thread_num, thread_num_);
ret = OB_INVALID_ARGUMENT;
} else {
prev_thread_num = thread_num_;
@ -305,7 +319,7 @@ int S2MQueueThread::sub_thread(const int64_t thread_num)
if (OB_SUCC(ret)) {
for (int64_t i = cur_thread_num; i < prev_thread_num; i++) {
balance_filter_.sub_thread(1);
ThreadConf& tc = thread_conf_array_[i];
ThreadConf &tc = thread_conf_array_[i];
tc.run_flag = false;
tc.stop_flag = true;
tc.queue_cond.signal();
@ -320,18 +334,18 @@ int S2MQueueThread::sub_thread(const int64_t thread_num)
return ret;
}
void* S2MQueueThread::rebalance_(int64_t& idx, const ThreadConf& cur_thread)
void *S2MQueueThread::rebalance_(int64_t &idx, const ThreadConf &cur_thread)
{
void* ret = NULL;
void *ret = NULL;
// Note: debug only
static __thread int64_t rebalance_counter = 0;
RLOCAL(int64_t, rebalance_counter);
for (uint64_t i = 1; (int64_t)i <= thread_num_; i++) {
RDLockGuard guard(thread_conf_lock_);
uint64_t balance_idx = (cur_thread.index + i) % thread_num_;
ThreadConf& tc = thread_conf_array_[balance_idx];
ThreadConf &tc = thread_conf_array_[balance_idx];
int64_t try_queue_idx = -1;
if (tc.using_flag) { //&& (tc.last_active_time + THREAD_BUSY_TIME_LIMIT) <
//::oceanbase::common::ObTimeUtility::current_time())
if (tc.using_flag
) { //&& (tc.last_active_time + THREAD_BUSY_TIME_LIMIT) < ::oceanbase::common::ObTimeUtility::current_time())
continue;
}
if (NULL == ret) {
@ -347,10 +361,8 @@ void* S2MQueueThread::rebalance_(int64_t& idx, const ThreadConf& cur_thread)
each_queue_len_[idx].inc(-1);
if (0 == (rebalance_counter++ % 10000)) {
_OB_LOG(INFO,
"task has been rebalance between threads rebalance_counter=%ld cur_idx=%ld balance_idx=%ld",
rebalance_counter,
cur_thread.index,
balance_idx);
"task has been rebalance between threads rebalance_counter=%ld cur_idx=%ld balance_idx=%ld",
*(&rebalance_counter), cur_thread.index, balance_idx);
}
break;
}
@ -363,7 +375,7 @@ int S2MQueueThread::launch_thread_(const int64_t thread_num, const int64_t task_
int ret = OB_SUCCESS;
int64_t thread_num2launch = std::min(MAX_THREAD_NUM - thread_num_, thread_num);
for (int64_t i = 0; OB_SUCC(ret) && i < thread_num2launch; i++) {
ThreadConf& tc = thread_conf_array_[thread_num_];
ThreadConf &tc = thread_conf_array_[thread_num_];
tc.index = thread_num_;
tc.run_flag = true;
tc.stop_flag = false;
@ -396,32 +408,33 @@ int S2MQueueThread::launch_thread_(const int64_t thread_num, const int64_t task_
tc.low_prio_task_queue.destroy();
break;
}
// cpu_set_t cpu_set;
// CPU_ZERO(&cpu_set);
// CPU_SET(thread_num_ % get_cpu_num(), &cpu_set);
// tmp_ret = pthread_setaffinity_np(tc.pd, sizeof(cpu_set), &cpu_set);
//cpu_set_t cpu_set;
//CPU_ZERO(&cpu_set);
//CPU_SET(thread_num_ % get_cpu_num(), &cpu_set);
//tmp_ret = pthread_setaffinity_np(tc.pd, sizeof(cpu_set), &cpu_set);
_OB_LOG(INFO, "create thread succ, index=%ld tmp_ret=%d", thread_num_, tmp_ret);
thread_num_ += 1;
}
return ret;
}
void* S2MQueueThread::thread_func_(void* data)
void *S2MQueueThread::thread_func_(void *data)
{
ThreadConf* const tc = (ThreadConf*)data;
if (NULL == tc || NULL == tc->host) {
ThreadConf *const tc = (ThreadConf *)data;
if (NULL == tc
|| NULL == tc->host) {
_OB_LOG(WARN, "thread_func param null pointer");
} else {
tc->host->thread_index() = tc->index;
void* pdata = tc->host->on_begin();
void *pdata = tc->host->on_begin();
bool last_rebalance_got = false;
while (tc->run_flag)
// && (0 != tc->high_prio_task_queue.get_total()
// || 0 != tc->spec_task_queue.get_total()
// || 0 != tc->comm_task_queue.get_total()
// || 0 != tc->low_prio_task_queue.get_total()))
// && (0 != tc->high_prio_task_queue.get_total()
// || 0 != tc->spec_task_queue.get_total()
// || 0 != tc->comm_task_queue.get_total()
// || 0 != tc->low_prio_task_queue.get_total()))
{
void* task = NULL;
void *task = NULL;
int64_t start_time = ::oceanbase::common::ObTimeUtility::current_time();
int64_t idx = -1;
tc->using_flag = true;
@ -450,7 +463,8 @@ void* S2MQueueThread::thread_func_(void* data)
idx = 3;
}
break;
default:;
default:
;
};
if (NULL == task) {
IGNORE_RETURN tc->high_prio_task_queue.pop(task);
@ -476,13 +490,14 @@ void* S2MQueueThread::thread_func_(void* data)
idx = 3;
}
}
tc->using_flag = false; // not need strict consist, so do not use barrier
tc->using_flag = false; // not need strict consist, so do not use barrier
if (idx >= 0 && idx < QUEUE_COUNT) {
tc->host->each_queue_len_[idx].inc(-1);
}
if (NULL != task ||
(tc->host->queue_rebalance_ && (last_rebalance_got || TC_REACH_TIME_INTERVAL(QUEUE_WAIT_TIME)) &&
(last_rebalance_got = (NULL != (task = tc->host->rebalance_(idx, *tc)))))) {
if (NULL != task
|| (tc->host->queue_rebalance_
&& (last_rebalance_got || TC_REACH_TIME_INTERVAL(QUEUE_WAIT_TIME))
&& (last_rebalance_got = (NULL != (task = tc->host->rebalance_(idx, *tc)))))) {
tc->host->queued_num_.inc(-1);
tc->host->handle_with_stopflag(task, pdata, tc->stop_flag);
tc->last_active_time = ::oceanbase::common::ObTimeUtility::current_time();
@ -490,11 +505,12 @@ void* S2MQueueThread::thread_func_(void* data)
tc->queue_cond.timedwait(QUEUE_WAIT_TIME);
}
tc->host->on_iter();
v4si queue_len = {
tc->high_prio_task_queue.get_total(),
tc->spec_task_queue.get_total(),
tc->comm_task_queue.get_total(),
tc->low_prio_task_queue.get_total(),
v4si queue_len =
{
tc->high_prio_task_queue.get_total(),
tc->spec_task_queue.get_total(),
tc->comm_task_queue.get_total(),
tc->low_prio_task_queue.get_total(),
};
tc->scheduler_.update(idx, ::oceanbase::common::ObTimeUtility::current_time() - start_time, queue_len);
}
@ -503,16 +519,9 @@ void* S2MQueueThread::thread_func_(void* data)
return NULL;
}
int64_t& S2MQueueThread::thread_index()
{
// Note: debug only right now
static __thread int64_t index = 0;
return index;
}
int64_t S2MQueueThread::get_thread_index() const
{
int64_t ret = const_cast<S2MQueueThread&>(*this).thread_index();
int64_t ret = const_cast<S2MQueueThread &>(*this).thread_index();
return ret;
}
@ -520,22 +529,23 @@ int64_t S2MQueueThread::get_thread_index() const
const int64_t M2SQueueThread::QUEUE_WAIT_TIME = 100 * 1000;
M2SQueueThread::M2SQueueThread()
: inited_(false),
pd_(0),
run_flag_(true),
queue_cond_(),
task_queue_(),
idle_interval_(INT64_MAX),
last_idle_time_(0)
{}
M2SQueueThread::M2SQueueThread() : inited_(false),
pd_(0),
run_flag_(true),
queue_cond_(),
task_queue_(),
idle_interval_(INT64_MAX),
last_idle_time_(0)
{
}
M2SQueueThread::~M2SQueueThread()
{
destroy();
}
int M2SQueueThread::init(const int64_t task_num_limit, const int64_t idle_interval)
int M2SQueueThread::init(const int64_t task_num_limit,
const int64_t idle_interval)
{
int ret = OB_SUCCESS;
int tmp_ret = 0;
@ -572,7 +582,7 @@ void M2SQueueThread::destroy()
inited_ = false;
}
int M2SQueueThread::push(void* task)
int M2SQueueThread::push(void *task)
{
int ret = OB_SUCCESS;
if (!inited_) {
@ -597,17 +607,17 @@ int64_t M2SQueueThread::get_queued_num() const
return task_queue_.get_total();
}
void* M2SQueueThread::thread_func_(void* data)
void *M2SQueueThread::thread_func_(void *data)
{
M2SQueueThread* const host = (M2SQueueThread*)data;
M2SQueueThread *const host = (M2SQueueThread *)data;
if (NULL == host) {
_OB_LOG(WARN, "thread_func param null pointer");
} else {
void* pdata = host->on_begin();
void *pdata = host->on_begin();
while (host->run_flag_)
//&& 0 != host->task_queue_.get_total())
//&& 0 != host->task_queue_.get_total())
{
void* task = NULL;
void *task = NULL;
IGNORE_RETURN host->task_queue_.pop(task);
if (NULL != task) {
host->handle(task, pdata);
@ -623,105 +633,6 @@ void* M2SQueueThread::thread_func_(void* data)
return NULL;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
SeqQueueThread::SeqQueueThread()
: inited_(false), pd_(0), run_flag_(true), idle_interval_(INT64_MAX), last_idle_time_(0)
{}
SeqQueueThread::~SeqQueueThread()
{
destroy();
}
int SeqQueueThread::init(const int64_t task_num_limit, const int64_t idle_interval)
{
int ret = OB_SUCCESS;
int tmp_ret = 0;
run_flag_ = true;
if (inited_) {
ret = OB_INIT_TWICE;
} else if (OB_SUCCESS != (ret = task_queue_.init(task_num_limit))) {
_OB_LOG(WARN, "task_queue init fail, ret=%d task_num_limit=%ld", ret, task_num_limit);
} else if (OB_FAIL(task_queue_.start(1))) {
_OB_LOG(ERROR, "task_queue_.start(1)=>%d", ret);
} else if (0 != (tmp_ret = pthread_create(&pd_, NULL, thread_func_, this))) {
ret = OB_ERR_UNEXPECTED;
_OB_LOG(WARN, "pthread_create fail, ret=%d", tmp_ret);
} else {
inited_ = true;
idle_interval_ = idle_interval;
last_idle_time_ = 0;
}
if (OB_FAIL(ret)) {
destroy();
}
return ret;
}
void SeqQueueThread::destroy()
{
if (0 != pd_) {
run_flag_ = false;
pthread_join(pd_, NULL);
pd_ = 0;
}
inited_ = false;
}
int SeqQueueThread::push(void* task)
{
int ret = OB_SUCCESS;
int64_t seq = 0;
if (!inited_) {
ret = OB_NOT_INIT;
} else if (NULL == task) {
ret = OB_INVALID_ARGUMENT;
} else if (0 >= (seq = get_seq(task))) {
ret = OB_ERR_UNEXPECTED;
_OB_LOG(WARN, "get_seq(task[%p])>%ld", task, seq);
} else {
ret = task_queue_.add(seq, task);
if (OB_FAIL(ret)) {
_OB_LOG(ERROR, "task_queue_.add(seq=%ld, task=%p)=>%d", seq, task, ret);
}
}
if (OB_FAIL(ret)) {
on_push_fail(task);
}
return ret;
}
int64_t SeqQueueThread::get_queued_num() const
{
return task_queue_.next_is_ready() ? 1 : 0;
}
void* SeqQueueThread::thread_func_(void* data)
{
int err = OB_SUCCESS;
SeqQueueThread* const host = (SeqQueueThread*)data;
if (NULL == host) {
_OB_LOG(WARN, "thread_func param null pointer");
} else {
void* pdata = host->on_begin();
int64_t seq = 0;
while (host->run_flag_) {
void* task = NULL;
if (OB_SUCCESS != (err = host->task_queue_.get(seq, task, QUEUE_WAIT_TIME)) && OB_EAGAIN != err) {
_OB_LOG(ERROR, "get(seq=%ld)=>%d", seq, err);
break;
} else if (OB_SUCCESS == err) {
host->handle(task, pdata);
} else if ((host->last_idle_time_ + host->idle_interval_) <= ::oceanbase::common::ObTimeUtility::current_time()) {
host->on_idle();
host->last_idle_time_ = ::oceanbase::common::ObTimeUtility::current_time();
}
}
host->on_end(pdata);
}
return NULL;
}
} // namespace common
} // namespace oceanbase