hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

patch 4.0

Browse Source
This commit is contained in:
wangzelin.wzl
2022-10-24 10:34:53 +08:00
parent 4ad6e00ec3
commit 93a1074b0c
10533 changed files with 2588271 additions and 2299373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

322
unittest/share/cache/ob_cache_test_utils.h vendored
View File

@ -13,21 +13,28 @@
#ifndef OCEANBASE_SHARE_OB_CACHE_TEST_UTILS_H_
#define OCEANBASE_SHARE_OB_CACHE_TEST_UTILS_H_
#include "share/cache/ob_kv_storecache.h"
#include "share/ob_define.h"
#include "share/ob_thread_pool.h"
#include "lib/atomic/ob_atomic.h"
#include "lib/queue/ob_lighty_queue.h"
#include "lib/alloc/ob_malloc_allocator.h"
namespace oceanbase {
#include "share/cache/ob_kvcache_hazard_version.h"
namespace oceanbase
{
using namespace lib;
namespace common {
namespace common
{
template <int64_t SIZE>
struct TestKVCacheKey : public ObIKVCacheKey {
TestKVCacheKey(void) : v_(0), tenant_id_(0)
template<int64_t SIZE>
struct TestKVCacheKey: public ObIKVCacheKey
{
TestKVCacheKey(void)
: v_(0), tenant_id_(0)
{
memset(buf_, 0, sizeof(buf_));
}
virtual bool operator==(const ObIKVCacheKey& other) const;
virtual bool operator ==(const ObIKVCacheKey &other) const;
virtual uint64_t get_tenant_id() const
{
return tenant_id_;
@ -40,15 +47,17 @@ struct TestKVCacheKey : public ObIKVCacheKey {
{
return sizeof(*this);
}
virtual int deep_copy(char* buf, const int64_t buf_len, ObIKVCacheKey*& key) const;
virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const;
uint64_t v_;
uint64_t tenant_id_;
char buf_[SIZE > sizeof(v_) ? SIZE - sizeof(v_) : 0];
};
template <int64_t SIZE>
struct TestKVCacheValue : public ObIKVCacheValue {
TestKVCacheValue(void) : v_(0)
template<int64_t SIZE>
struct TestKVCacheValue: public ObIKVCacheValue
{
TestKVCacheValue(void)
: v_(0)
{
memset(buf_, 0, sizeof(buf_));
}
@ -56,23 +65,24 @@ struct TestKVCacheValue : public ObIKVCacheValue {
{
return sizeof(*this);
}
virtual int deep_copy(char* buf, const int64_t buf_len, ObIKVCacheValue*& value) const;
virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const;
uint64_t v_;
char buf_[SIZE > sizeof(v_) ? SIZE - sizeof(v_) : 0];
};
template <int64_t SIZE>
bool TestKVCacheKey<SIZE>::operator==(const ObIKVCacheKey& other) const
template<int64_t SIZE>
bool TestKVCacheKey<SIZE>::operator ==(const ObIKVCacheKey &other) const
{
const TestKVCacheKey& other_key = reinterpret_cast<const TestKVCacheKey&>(other);
return v_ == other_key.v_ && tenant_id_ == other_key.tenant_id_;
const TestKVCacheKey &other_key = reinterpret_cast<const TestKVCacheKey &>(other);
return v_ == other_key.v_ && tenant_id_ == other_key.tenant_id_;
}
template <int64_t SIZE>
int TestKVCacheKey<SIZE>::deep_copy(char* buf, const int64_t buf_len, ObIKVCacheKey*& key) const
template<int64_t SIZE>
int TestKVCacheKey<SIZE>::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const
{
int ret = OB_SUCCESS;
TestKVCacheKey<SIZE>* pkey = NULL;
TestKVCacheKey<SIZE> *pkey = NULL;
if (NULL == buf || buf_len < size()) {
ret = OB_INVALID_ARGUMENT;
} else {
@ -84,11 +94,14 @@ int TestKVCacheKey<SIZE>::deep_copy(char* buf, const int64_t buf_len, ObIKVCache
return ret;
}
template <int64_t SIZE>
int TestKVCacheValue<SIZE>::deep_copy(char* buf, const int64_t buf_len, ObIKVCacheValue*& value) const
template<int64_t SIZE>
int TestKVCacheValue<SIZE>::deep_copy(
char *buf,
const int64_t buf_len,
ObIKVCacheValue *&value) const
{
int ret = OB_SUCCESS;
TestKVCacheValue<SIZE>* pvalue = NULL;
TestKVCacheValue<SIZE> *pvalue = NULL;
if (NULL == buf || buf_len < size()) {
ret = OB_INVALID_ARGUMENT;
} else {
@ -99,34 +112,36 @@ int TestKVCacheValue<SIZE>::deep_copy(char* buf, const int64_t buf_len, ObIKVCac
return ret;
}
struct AllocBuf {
void* ptr_;
AllocBuf* next_;
struct AllocBuf
{
void *ptr_;
AllocBuf *next_;
};
class ObCacheTestTask;
class ObICacheTestStat {
class ObICacheTestStat
{
public:
virtual void add_task(ObCacheTestTask* task) = 0;
virtual ObCacheTestTask* pop_oppo_task(ObCacheTestTask* task) = 0;
virtual void add_task(ObCacheTestTask *task) = 0;
virtual ObCacheTestTask *pop_oppo_task(ObCacheTestTask *task) = 0;
virtual void inc_fail_count() = 0;
};
class ObCacheTestTask {
class ObCacheTestTask
{
public:
ObCacheTestTask(const int64_t tenant_id, const bool is_alloc, const int64_t alloc_size, const int64_t alloc_count,
ObICacheTestStat* stat)
: tenant_id_(tenant_id),
is_alloc_(is_alloc),
alloc_size_(alloc_size),
alloc_count_(alloc_count),
next_(NULL),
stat_(stat),
alloc_list_(NULL)
{}
ObCacheTestTask(const int64_t tenant_id, const bool is_alloc,
const int64_t alloc_size, const int64_t alloc_count,
ObICacheTestStat *stat)
: tenant_id_(tenant_id), is_alloc_(is_alloc),
alloc_size_(alloc_size), alloc_count_(alloc_count),
next_(NULL), stat_(stat), alloc_list_(NULL)
{
}
virtual ~ObCacheTestTask()
{}
virtual ~ObCacheTestTask() {}
virtual int process()
{
@ -134,9 +149,9 @@ public:
if (is_alloc_) {
ObMemAttr attr;
attr.tenant_id_ = tenant_id_;
attr.label_ = 1; // xxx
attr.label_ = "CacheTestTask"; //xxx
for (int64_t i = 0; i < alloc_count_; ++i) {
void* ptr = ob_malloc(alloc_size_, attr);
void *ptr = ob_malloc(alloc_size_, attr);
if (NULL == ptr) {
ret = OB_ALLOCATE_MEMORY_FAILED;
COMMON_LOG(WARN, "ob_malloc failed", K(ret), K_(alloc_size));
@ -152,14 +167,14 @@ public:
stat_->add_task(this);
}
} else {
ObCacheTestTask* oppo_task = stat_->pop_oppo_task(this);
ObCacheTestTask *oppo_task = stat_->pop_oppo_task(this);
if (NULL == oppo_task) {
ret = OB_ERR_UNEXPECTED;
COMMON_LOG(WARN, "oppo_task not exist", K(ret), "this", *this);
} else {
int64_t free_count = 0;
AllocBuf* buf = oppo_task->alloc_list_;
AllocBuf* next = NULL;
AllocBuf *buf = oppo_task->alloc_list_;
AllocBuf *next = NULL;
while (NULL != buf) {
next = buf->next_;
ob_free(buf->ptr_);
@ -185,61 +200,50 @@ public:
return ret;
}
void set_next(ObCacheTestTask* task)
void set_next(ObCacheTestTask *task) { next_ = task; }
void append_alloc_list(void *ptr)
{
next_ = task;
}
void append_alloc_list(void* ptr)
{
AllocBuf* buf = new (ptr) AllocBuf();
AllocBuf *buf = new (ptr) AllocBuf();
buf->ptr_ = ptr;
buf->next_ = alloc_list_;
alloc_list_ = buf;
}
TO_STRING_KV(K_(tenant_id), K_(is_alloc), K_(alloc_size), K_(alloc_count));
public:
uint64_t tenant_id_;
bool is_alloc_; // alloc or free
int64_t alloc_size_;
int64_t alloc_count_;
ObCacheTestTask* next_;
ObICacheTestStat* stat_;
AllocBuf* alloc_list_;
ObCacheTestTask *next_;
ObICacheTestStat *stat_;
AllocBuf *alloc_list_;
};
class CacheTestStat : public ObICacheTestStat {
class CacheTestStat : public ObICacheTestStat
{
public:
CacheTestStat() : fail_count_(0), task_list_(NULL)
{}
virtual ~CacheTestStat()
{}
CacheTestStat() : fail_count_(0), task_list_(NULL) {}
virtual ~CacheTestStat() {}
virtual void inc_fail_count()
{
ATOMIC_AAF(&fail_count_, 1);
}
virtual int64_t get_fail_count() const
{
return ATOMIC_LOAD(&fail_count_);
}
virtual void add_task(ObCacheTestTask* task)
virtual void inc_fail_count() { ATOMIC_AAF(&fail_count_, 1); }
virtual int64_t get_fail_count() const { return ATOMIC_LOAD(&fail_count_); }
virtual void add_task(ObCacheTestTask *task)
{
if (NULL != task) {
task->next_ = task_list_;
task_list_ = task;
}
}
virtual ObCacheTestTask* pop_oppo_task(ObCacheTestTask* task)
virtual ObCacheTestTask *pop_oppo_task(ObCacheTestTask *task)
{
ObCacheTestTask* oppo_task = NULL;
ObCacheTestTask* prev_task = NULL;
ObCacheTestTask* cur_task = NULL;
ObCacheTestTask *oppo_task = NULL;
ObCacheTestTask *prev_task = NULL;
ObCacheTestTask *cur_task = NULL;
if (NULL != task && !task->is_alloc_) {
cur_task = task_list_;
while (NULL != cur_task) {
if (cur_task->is_alloc_ && cur_task->tenant_id_ == task->tenant_id_ &&
cur_task->alloc_size_ == task->alloc_size_ && cur_task->alloc_count_ == task->alloc_count_) {
if (cur_task->is_alloc_ && cur_task->tenant_id_ == task->tenant_id_
&& cur_task->alloc_size_ == task->alloc_size_ && cur_task->alloc_count_ == task->alloc_count_) {
oppo_task = cur_task;
// delete it from list
if (NULL != prev_task) {
@ -257,18 +261,16 @@ public:
}
return oppo_task;
}
private:
int64_t fail_count_;
ObCacheTestTask* task_list_;
ObCacheTestTask *task_list_;
};
class ObAllocatorStress : public share::ObThreadPool {
class ObAllocatorStress : public share::ObThreadPool
{
public:
ObAllocatorStress() : inited_(false), stat_(), queue_()
{}
virtual ~ObAllocatorStress()
{}
ObAllocatorStress() : inited_(false), stat_(), queue_() {}
virtual ~ObAllocatorStress() {}
int init()
{
@ -287,7 +289,7 @@ public:
virtual void run1()
{
UNUSED(arg);
// UNUSED(arg);
int ret = OB_SUCCESS;
COMMON_LOG(INFO, "allocator stress thread start");
if (!inited_) {
@ -296,7 +298,7 @@ public:
} else {
// will process all task before exit
while (!has_set_stop() || OB_ENTRY_NOT_EXIST != ret) {
ObCacheTestTask* task = NULL;
ObCacheTestTask *task = NULL;
if (OB_FAIL(pop(task))) {
if (OB_ENTRY_NOT_EXIST != ret) {
COMMON_LOG(WARN, "pop task failed", K(ret));
@ -318,17 +320,17 @@ public:
COMMON_LOG(INFO, "allocator stress thread end");
}
int add_task(const ObCacheTestTask& task)
int add_task(const ObCacheTestTask &task)
{
int ret = OB_SUCCESS;
const int64_t buf_size = sizeof(task);
void* ptr = NULL;
void *ptr = NULL;
if (NULL == (ptr = ob_malloc(buf_size))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
COMMON_LOG(WARN, "ob_malloc failed", K(ret), K(buf_size));
} else {
ObCacheTestTask* copy_task =
new (ptr) ObCacheTestTask(task.tenant_id_, task.is_alloc_, task.alloc_size_, task.alloc_count_, task.stat_);
ObCacheTestTask *copy_task = new (ptr) ObCacheTestTask(
task.tenant_id_, task.is_alloc_, task.alloc_size_, task.alloc_count_, task.stat_);
if (OB_FAIL(queue_.push(copy_task))) {
COMMON_LOG(WARN, "push task failed", K(ret));
}
@ -336,14 +338,14 @@ public:
return ret;
}
int pop(ObCacheTestTask*& task)
int pop(ObCacheTestTask *&task)
{
int ret = OB_SUCCESS;
void* vp = NULL;
void *vp = NULL;
const int64_t timeout = 1000 * 1000;
if (!inited_) {
ret = OB_NOT_INIT;
COMMON_LOG(WARN, "not init", K(ret));
COMMON_LOG(WARN,"not init", K(ret));
} else {
ret = queue_.pop(vp, timeout);
if (OB_FAIL(ret)) {
@ -351,37 +353,30 @@ public:
COMMON_LOG(WARN, "queue pop failed", K(ret));
}
} else {
task = static_cast<ObCacheTestTask*>(vp);
task = static_cast<ObCacheTestTask *>(vp);
}
}
return ret;
}
int64_t get_fail_count()
{
return stat_.get_fail_count();
}
CacheTestStat* get_stat()
{
return &stat_;
}
int64_t get_fail_count() { return stat_.get_fail_count(); }
CacheTestStat *get_stat() { return &stat_; }
private:
bool inited_;
CacheTestStat stat_;
common::LightyQueue queue_;
common::ObLightyQueue queue_;
};
template <int64_t K_SIZE, int64_t V_SIZE>
class ObCacheStress : public share::ObThreadPool {
template<int64_t K_SIZE, int64_t V_SIZE>
class ObCacheStress : public share::ObThreadPool
{
public:
typedef TestKVCacheKey<K_SIZE> TestKey;
typedef TestKVCacheValue<V_SIZE> TestValue;
ObCacheStress() : inited_(false), tenant_id_(OB_INVALID_ID), put_count_(0), fail_count_(0), cache_()
{}
virtual ~ObCacheStress()
{}
ObCacheStress() : inited_(false), tenant_id_(OB_INVALID_ID),
put_count_(0), fail_count_(0), cache_() {}
virtual ~ObCacheStress() {}
int init(const uint64_t tenant_id, int64_t index)
{
@ -402,7 +397,7 @@ public:
virtual void run1()
{
UNUSED(arg);
// UNUSED(arg);
int ret = OB_SUCCESS;
COMMON_LOG(INFO, "cache stress thread start");
TestKey key;
@ -417,7 +412,7 @@ public:
if (OB_FAIL(cache_.put(key, value))) {
COMMON_LOG(WARN, "cache put failed", K(ret));
} else {
const TestValue* get_value = NULL;
const TestValue *get_value = NULL;
ObKVCacheHandle handle;
if (OB_FAIL(cache_.get(key, get_value, handle))) {
COMMON_LOG(WARN, "cache get failed", K(ret));
@ -432,19 +427,9 @@ public:
COMMON_LOG(INFO, "cache stress thread exit");
}
uint64_t get_tenant_id() const
{
return tenant_id_;
}
int64_t get_put_count() const
{
return put_count_;
}
int64_t get_fail_count() const
{
return fail_count_;
}
uint64_t get_tenant_id() const { return tenant_id_; }
int64_t get_put_count() const { return put_count_; }
int64_t get_fail_count() const { return fail_count_; }
private:
bool inited_;
uint64_t tenant_id_;
@ -453,15 +438,18 @@ private:
ObKVCache<TestKey, TestValue> cache_;
};
template <int64_t K_SIZE, int64_t V_SIZE>
class ObCacheGetStress : public share::ObThreadPool {
template<int64_t K_SIZE, int64_t V_SIZE>
class ObCacheGetStress : public share::ObThreadPool
{
public:
typedef TestKVCacheKey<K_SIZE> TestKey;
typedef TestKVCacheValue<V_SIZE> TestValue;
ObCacheGetStress()
: inited_(false), tenant_id_(OB_INVALID_ID), kv_cnt_(0), hit_cnt_(0), total_cnt_(0), fail_cnt_(0), cache_()
{}
: inited_(false), tenant_id_(OB_INVALID_ID),
kv_cnt_(0), hit_cnt_(0), total_cnt_(0), fail_cnt_(0), cache_()
{
}
int init(const uint64_t tenant_id, const int64_t kv_cnt)
{
@ -479,8 +467,7 @@ public:
TestKey key;
TestValue value;
for (int64_t i = 0; OB_SUCC(ret) && i < kv_cnt; ++i) {
key.tenant_id_ = tenant_id;
;
key.tenant_id_ = tenant_id;;
key.v_ = i;
if (OB_FAIL(cache_.put(key, value))) {
COMMON_LOG(WARN, "put failed", K(ret));
@ -506,7 +493,8 @@ public:
virtual void run1()
{
int64_t thread_id = (int64_t)(arg);
// int64_t thread_id = (int64_t)(arg);
int64_t thread_id = (int64_t)(this->get_thread_idx());
if (0 == thread_id) {
do_monitor();
} else {
@ -514,7 +502,7 @@ public:
}
}
ObKVCache<TestKey, TestValue>& get_cache()
ObKVCache<TestKey, TestValue> &get_cache()
{
return cache_;
}
@ -523,7 +511,7 @@ public:
int ret = OB_SUCCESS;
int64_t hit_cnt = 0;
TestKey key;
const TestValue* pvalue = NULL;
const TestValue *pvalue = NULL;
ObKVCacheHandle handle;
for (int64_t i = 0; i < kv_cnt_; ++i) {
key.tenant_id_ = tenant_id_;
@ -538,7 +526,6 @@ public:
}
return (double)hit_cnt / (double)kv_cnt_;
}
private:
void do_monitor()
{
@ -560,7 +547,7 @@ private:
{
int ret = OB_SUCCESS;
TestKey key;
const TestValue* pvalue = NULL;
const TestValue *pvalue = NULL;
ObKVCacheHandle handle;
while (!has_set_stop()) {
for (int64_t i = 0; i < kv_cnt_ && !has_set_stop(); ++i) {
@ -592,16 +579,16 @@ private:
ObKVCache<TestKey, TestValue> cache_;
};
template <int64_t K_SIZE, int64_t V_SIZE>
class ObWorkingSetStress : public share::ObThreadPool {
template<int64_t K_SIZE, int64_t V_SIZE>
class ObWorkingSetStress : public share::ObThreadPool
{
public:
typedef TestKVCacheKey<K_SIZE> TestKey;
typedef TestKVCacheValue<V_SIZE> TestValue;
ObWorkingSetStress() : inited_(false), tenant_id_(OB_INVALID_ID), put_count_(0), fail_count_(0)
{}
virtual ~ObWorkingSetStress()
{}
ObWorkingSetStress() : inited_(false), tenant_id_(OB_INVALID_ID),
put_count_(0), fail_count_(0) {}
virtual ~ObWorkingSetStress() {}
int init(const uint64_t tenant_id, const bool only_put)
{
@ -624,8 +611,7 @@ public:
return ret;
}
int init(
const uint64_t tenant_id, ObKVCache<TestKey, TestValue>& cache, const bool use_ws, const int64_t start_key = 0)
int init(const uint64_t tenant_id, ObKVCache<TestKey, TestValue> &cache, const bool use_ws, const int64_t start_key = 0)
{
int ret = OB_SUCCESS;
if (inited_) {
@ -655,8 +641,9 @@ public:
virtual void run1()
{
const int64_t thread_id = (int64_t)(arg);
const int64_t count = _threadCount;
// const int64_t thread_id = (int64_t)(arg);
const int64_t thread_id = this->get_thread_idx();
const int64_t count = this->get_thread_count();
int64_t put_count = 0;
int64_t fail_count = 0;
int ret = OB_SUCCESS;
@ -673,7 +660,7 @@ public:
if (OB_FAIL(pcache_->put(key, value))) {
COMMON_LOG(WARN, "cache put failed", K(ret));
} else if (!only_put_) {
const TestValue* get_value = NULL;
const TestValue *get_value = NULL;
ObKVCacheHandle handle;
if (OB_FAIL(pcache_->get(key, get_value, handle))) {
if (OB_ENTRY_NOT_EXIST == ret) {
@ -688,6 +675,7 @@ public:
++fail_count;
ObMallocAllocator::get_instance()->print_tenant_memory_usage(tenant_id_);
ObMallocAllocator::get_instance()->print_tenant_ctx_memory_usage(tenant_id_);
}
}
}
@ -696,40 +684,34 @@ public:
COMMON_LOG(INFO, "working set stress thread exit");
}
uint64_t get_tenant_id() const
{
return tenant_id_;
}
int64_t get_put_count() const
{
return put_count_;
}
int64_t get_fail_count() const
{
return fail_count_;
}
int64_t get_used() const
{
return ws_.working_set_->get_used();
}
int64_t get_limit() const
{
return ws_.working_set_->get_limit();
}
uint64_t get_tenant_id() const { return tenant_id_; }
int64_t get_put_count() const { return put_count_; }
int64_t get_fail_count() const { return fail_count_; }
int64_t get_used() const { return ws_.working_set_->get_used(); }
int64_t get_limit() const { return ws_.working_set_->get_limit(); }
private:
bool inited_;
uint64_t tenant_id_;
int64_t put_count_;
int64_t fail_count_;
bool only_put_;
ObIKVCache<TestKey, TestValue>* pcache_;
ObIKVCache<TestKey, TestValue> *pcache_;
ObKVCache<TestKey, TestValue> cache_;
ObCacheWorkingSet<TestKey, TestValue> ws_;
int64_t start_key_;
};
} // end namespace common
} // end namespace oceanbase
class TestNode : public KVCacheHazardNode{
public:
TestNode()
: id_(0)
{}
virtual void retire() {COMMON_LOG(INFO, "TestNode(HazardNode) retire", K(this->get_version()), K(id_));}
private:
int64_t id_;
};
#endif // OCEANBASE_SHARE_OB_CACHE_TEST_UTILS_H_
}//end namespace common
}//end namespace oceanbase
#endif //OCEANBASE_SHARE_OB_CACHE_TEST_UTILS_H_