hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
deps/oblib/src/lib/queue/ob_dedup_queue.cpp vendored
View File

@ -13,53 +13,66 @@
#include "lib/queue/ob_dedup_queue.h"
#include "lib/thread/ob_thread_name.h"
namespace oceanbase {
namespace common {
#define ERASE_FROM_LIST(head, tail, node) \
do { \
if (NULL == node->get_prev()) { \
head = node->get_next(); \
} else { \
namespace oceanbase
{
namespace common
{
#define ERASE_FROM_LIST(head, tail, node) \
do \
{ \
if (NULL == node->get_prev()) \
{ \
head = node->get_next(); \
} \
else \
{ \
node->get_prev()->set_next(node->get_next()); \
} \
if (NULL == node->get_next()) { \
tail = node->get_prev(); \
} else { \
} \
if (NULL == node->get_next()) \
{ \
tail = node->get_prev(); \
} \
else \
{ \
node->get_next()->set_prev(node->get_prev()); \
} \
node->set_prev(NULL); \
node->set_next(NULL); \
} while (0)
} \
node->set_prev(NULL); \
node->set_next(NULL); \
} while(0)
ObDedupQueue::ObDedupQueue()
: is_inited_(false),
thread_num_(DEFAULT_THREAD_NUM),
work_thread_num_(DEFAULT_THREAD_NUM),
thread_dead_threshold_(DEFALT_THREAD_DEAD_THRESHOLD),
hash_allocator_(allocator_),
gc_queue_head_(NULL),
gc_queue_tail_(NULL),
thread_name_(nullptr)
{}
ObDedupQueue::ObDedupQueue() : is_inited_(false),
thread_num_(DEFAULT_THREAD_NUM),
work_thread_num_(DEFAULT_THREAD_NUM),
thread_dead_threshold_(DEFALT_THREAD_DEAD_THRESHOLD),
hash_allocator_(allocator_),
gc_queue_head_(NULL),
gc_queue_tail_(NULL),
thread_name_(nullptr)
{
}
ObDedupQueue::~ObDedupQueue()
{
destroy();
}
int ObDedupQueue::init(int32_t thread_num /*= DEFAULT_THREAD_NUM*/, const char* thread_name /*= NULL*/,
const int64_t queue_size /*= TASK_QUEUE_SIZE*/, const int64_t task_map_size /*= TASK_MAP_SIZE*/,
const int64_t total_mem_limit /*= TOTAL_LIMIT*/, const int64_t hold_mem_limit /*= HOLD_LIMIT*/,
const int64_t page_size /*= PAGE_SIZE*/)
int ObDedupQueue::init(int32_t thread_num /*= DEFAULT_THREAD_NUM*/,
const char* thread_name /*= NULL*/,
const int64_t queue_size /*= TASK_QUEUE_SIZE*/,
const int64_t task_map_size /*= TASK_MAP_SIZE*/,
const int64_t total_mem_limit /*= TOTAL_LIMIT*/,
const int64_t hold_mem_limit /*= HOLD_LIMIT*/,
const int64_t page_size /*= PAGE_SIZE*/)
{
int ret = OB_SUCCESS;
if (is_inited_) {
ret = OB_INIT_TWICE;
} else if (thread_num <= 0 || thread_num > MAX_THREAD_NUM || total_mem_limit <= 0 || hold_mem_limit <= 0 ||
page_size <= 0) {
} else if (thread_num <= 0 || thread_num > MAX_THREAD_NUM
|| total_mem_limit <= 0 || hold_mem_limit <= 0
|| page_size <= 0) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(
WARN, "invalid argument", K(thread_num), K(queue_size), K(total_mem_limit), K(hold_mem_limit), K(page_size));
COMMON_LOG(WARN, "invalid argument", K(thread_num), K(queue_size),
K(total_mem_limit), K(hold_mem_limit), K(page_size));
} else if (OB_FAIL(task_queue_sync_.init(ObWaitEventIds::DEDUP_QUEUE_COND_WAIT))) {
COMMON_LOG(WARN, "fail to init task queue sync cond, ", K(ret));
} else if (OB_FAIL(work_thread_sync_.init(ObWaitEventIds::DEFAULT_COND_WAIT))) {
@ -70,9 +83,10 @@ int ObDedupQueue::init(int32_t thread_num /*= DEFAULT_THREAD_NUM*/, const char*
work_thread_num_ = thread_num;
set_thread_count(thread_num);
if (OB_SUCCESS != (ret = allocator_.init(total_mem_limit, hold_mem_limit, page_size))) {
COMMON_LOG(WARN, "allocator init fail", K(total_mem_limit), K(hold_mem_limit), K(page_size), K(ret));
} else if (OB_SUCCESS !=
(ret = task_map_.create(task_map_size, &hash_allocator_, ObModIds::OB_HASH_BUCKET_TASK_MAP))) {
COMMON_LOG(WARN, "allocator init fail", K(total_mem_limit), K(hold_mem_limit), K(page_size),
K(ret));
} else if (OB_SUCCESS != (ret = task_map_.create(task_map_size, &hash_allocator_,
ObModIds::OB_HASH_BUCKET_TASK_MAP))) {
COMMON_LOG(WARN, "task_map create fail", K(ret));
} else if (OB_SUCCESS != (ret = task_queue_.init(queue_size))) {
COMMON_LOG(WARN, "task_queue init fail", K(ret));
@ -85,17 +99,9 @@ int ObDedupQueue::init(int32_t thread_num /*= DEFAULT_THREAD_NUM*/, const char*
if (OB_FAIL(ret)) {
destroy();
} else {
COMMON_LOG(INFO,
"init dedup-queue:",
K(thread_num),
K(queue_size),
K(task_map_size),
K(total_mem_limit),
K(hold_mem_limit),
K(page_size),
KP(this),
"lbt",
lbt());
COMMON_LOG(INFO, "init dedup-queue:",
K(thread_num), K(queue_size), K(task_map_size), K(total_mem_limit), K(hold_mem_limit),
K(page_size), KP(this), "lbt", lbt());
}
return ret;
@ -103,13 +109,13 @@ int ObDedupQueue::init(int32_t thread_num /*= DEFAULT_THREAD_NUM*/, const char*
void ObDedupQueue::destroy()
{
lib::CoKThread::stop();
lib::CoKThread::wait();
lib::CoKThread::destroy();
lib::Threads::stop();
lib::Threads::wait();
lib::Threads::destroy();
IObDedupTask* iter = gc_queue_head_;
IObDedupTask *iter = gc_queue_head_;
while (NULL != iter) {
IObDedupTask* next = iter->get_next();
IObDedupTask *next = iter->get_next();
destroy_task_(iter);
iter = next;
}
@ -141,23 +147,7 @@ int ObDedupQueue::set_thread_dead_threshold(const int64_t thread_dead_threshold)
return ret;
}
int ObDedupQueue::set_work_thread_num(const int32_t work_thread_num)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (0 >= work_thread_num) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "work_thread_num is not valid. ", K(ret), K(work_thread_num));
} else {
ObThreadCondGuard cond_guard(work_thread_sync_);
work_thread_num_ = work_thread_num;
(void)work_thread_sync_.broadcast();
}
return ret;
}
int ObDedupQueue::add_task(const IObDedupTask& task)
int ObDedupQueue::add_task(const IObDedupTask &task)
{
int ret = OB_SUCCESS;
@ -192,15 +182,15 @@ int ObDedupQueue::add_task(const IObDedupTask& task)
return ret;
}
IObDedupTask* ObDedupQueue::copy_task_(const IObDedupTask& task)
IObDedupTask *ObDedupQueue::copy_task_(const IObDedupTask &task)
{
IObDedupTask* ret = NULL;
IObDedupTask *ret = NULL;
if (IS_NOT_INIT) {
COMMON_LOG(WARN, "ObDedupQueue is not inited");
} else {
int64_t deep_copy_size = task.get_deep_copy_size();
char* memory = NULL;
if (NULL == (memory = (char*)allocator_.alloc(deep_copy_size))) {
char *memory = NULL;
if (NULL == (memory = (char *)allocator_.alloc(deep_copy_size))) {
COMMON_LOG(WARN, "alloc memory fail", K(deep_copy_size), K(task.get_type()));
} else if (NULL == (ret = task.deep_copy(memory, deep_copy_size))) {
COMMON_LOG(WARN, "deep copy task object fail", K(deep_copy_size), KP(memory));
@ -218,14 +208,14 @@ IObDedupTask* ObDedupQueue::copy_task_(const IObDedupTask& task)
return ret;
}
void ObDedupQueue::destroy_task_(IObDedupTask* task)
void ObDedupQueue::destroy_task_(IObDedupTask *task)
{
if (IS_NOT_INIT) {
COMMON_LOG(WARN, "ObDedupQueue is not inited");
} else if (OB_ISNULL(task)) {
COMMON_LOG(WARN, "invalid argument");
} else {
char* memory = task->get_memory_ptr();
char *memory = task->get_memory_ptr();
task->~IObDedupTask();
if (NULL != memory) {
allocator_.free(memory);
@ -234,21 +224,22 @@ void ObDedupQueue::destroy_task_(IObDedupTask* task)
}
}
int ObDedupQueue::map_callback_(const IObDedupTask& task, TaskMapKVPair& kvpair)
int ObDedupQueue::map_callback_(const IObDedupTask &task, TaskMapKVPair &kvpair)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
IObDedupTask* task2remove = NULL;
IObDedupTask* task2add = NULL;
if (kvpair.first != kvpair.second || NULL == (task2remove = kvpair.second)) {
IObDedupTask *task2remove = NULL;
IObDedupTask *task2add = NULL;
if (kvpair.first != kvpair.second
|| NULL == (task2remove = kvpair.second)) {
COMMON_LOG(WARN, "unexpected key null pointer", K(kvpair.first), K(kvpair.second));
ret = OB_ERR_UNEXPECTED;
} else if (0 != task2remove->trylock()) {
ret = OB_EAGAIN;
} else if (!task2remove->is_process_done() ||
::oceanbase::common::ObTimeUtility::current_time() < task2remove->get_abs_expired_time()) {
} else if (!task2remove->is_process_done()
|| ::oceanbase::common::ObTimeUtility::current_time() < task2remove->get_abs_expired_time()) {
task2remove->unlock();
ret = OB_EAGAIN;
} else if (NULL == (task2add = copy_task_(task))) {
@ -274,13 +265,13 @@ int ObDedupQueue::map_callback_(const IObDedupTask& task, TaskMapKVPair& kvpair)
return ret;
}
int ObDedupQueue::add_task_(const IObDedupTask& task)
int ObDedupQueue::add_task_(const IObDedupTask &task)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else {
IObDedupTask* task2add = NULL;
IObDedupTask *task2add = NULL;
if (NULL == (task2add = copy_task_(task))) {
COMMON_LOG(WARN, "copy task fail");
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -318,15 +309,17 @@ int ObDedupQueue::add_task_(const IObDedupTask& task)
bool ObDedupQueue::gc_()
{
bool bret = false;
IObDedupTask* task_list[GC_BATCH_NUM];
IObDedupTask *task_list[GC_BATCH_NUM];
int64_t task_list_size = 0;
if (0 == gc_queue_sync_.trylock()) {
bret = true;
IObDedupTask* iter = gc_queue_head_;
while (NULL != iter && GC_BATCH_NUM > task_list_size) {
IObDedupTask* next = iter->get_next();
if (iter->is_process_done() &&
::oceanbase::common::ObTimeUtility::current_time() > iter->get_abs_expired_time() && 0 == iter->trylock()) {
IObDedupTask *iter = gc_queue_head_;
while (NULL != iter
&& GC_BATCH_NUM > task_list_size) {
IObDedupTask *next = iter->get_next();
if (iter->is_process_done()
&& ::oceanbase::common::ObTimeUtility::current_time() > iter->get_abs_expired_time()
&& 0 == iter->trylock()) {
task_list[task_list_size++] = iter;
ERASE_FROM_LIST(gc_queue_head_, gc_queue_tail_, iter);
}
@ -356,14 +349,14 @@ void ObDedupQueue::run1()
{
int tmp_ret = OB_SUCCESS;
int64_t thread_pos = (int64_t)get_thread_idx();
ThreadMeta& thread_meta = thread_metas_[thread_pos];
ThreadMeta &thread_meta = thread_metas_[thread_pos];
thread_meta.init();
COMMON_LOG(INFO, "dedup queue thread start", KP(this));
if (OB_NOT_NULL(thread_name_)) {
lib::set_thread_name(thread_name_);
}
while (!has_set_stop()) {
IObDedupTask* task2process = NULL;
while (!has_set_stop() && !(OB_NOT_NULL(&lib::Thread::current()) ? lib::Thread::current().has_set_stop() : false)) {
IObDedupTask *task2process = NULL;
if (thread_pos < work_thread_num_) {
if (OB_SUCCESS != (tmp_ret = task_queue_.pop(task2process)) && OB_UNLIKELY(tmp_ret != OB_ENTRY_NOT_EXIST)) {
COMMON_LOG(WARN, "task_queue_.pop error", K(tmp_ret), K(task2process));
@ -391,7 +384,8 @@ void ObDedupQueue::run1()
bool gc_done = gc_();
thread_meta.on_gc_end();
if ((NULL != gc_queue_head_ && gc_done) || NULL != task2process) {
if ((NULL != gc_queue_head_ && gc_done)
|| NULL != task2process) {
// need not wait
} else {
ObThreadCondGuard guard(task_queue_sync_);
@ -415,5 +409,5 @@ void ObDedupQueue::run1()
}
}
}
} // namespace common
} // namespace oceanbase
} // namespace common
} // namespace oceanbase